From 900fe846957c14b049e9fd45374ba016619f0af8 Mon Sep 17 00:00:00 2001 From: Ram Adithya Date: Fri, 3 Apr 2026 10:43:35 -0700 Subject: [PATCH 01/20] Combined two demo python notebooks into one --- demo/ParaFrame.ipynb | 1678 -------------------------------- demo/demo_python.ipynb | 2091 ++++++++++++++++++++++++++++++++++------ 2 files changed, 1821 insertions(+), 1948 deletions(-) delete mode 100644 demo/ParaFrame.ipynb diff --git a/demo/ParaFrame.ipynb b/demo/ParaFrame.ipynb deleted file mode 100644 index 5cd5fbf..0000000 --- a/demo/ParaFrame.ipynb +++ /dev/null @@ -1,1678 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "0", - "metadata": {}, - "source": [ - "# ParaFrame Demo\n", - "\n", - "This notebook demonstrates the current `hallmark` ParaFrame API.\n", - "\n", - "`ParaFrame` is a Pandas DataFrame subclass that discovers and parses files whose names follow a Python format string pattern. It extracts named parameters directly into DataFrame columns, making it easy to filter and select files across large parameter surveys.\n", - "\n", - "Key components:\n", - "- **`.hallmark.yaml`** — config file that lives alongside your data, declaring format strings and optional regex encodings for special filename conventions\n", - "- **`ParaFrame.parse()`** — discovers matching files and returns a DataFrame\n", - "- **`ParaFrame.__call__()`** — filters the DataFrame by column values" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "1", - "metadata": { - "execution": { - "iopub.execute_input": "2026-03-17T01:36:44.420836Z", - "iopub.status.busy": "2026-03-17T01:36:44.420675Z", - "iopub.status.idle": "2026-03-17T01:36:44.495945Z", - "shell.execute_reply": "2026-03-17T01:36:44.495136Z" - } - }, - "outputs": [], - "source": [ - "# auto reload modules when they change\n", - "%load_ext autoreload\n", - "%autoreload 2" - ] - }, - { - "cell_type": "markdown", - "id": "2", - "metadata": {}, - "source": [ - "## 1. The `.hallmark.yaml` Config File\n", - "\n", - "Before calling `ParaFrame.parse()`, a `.hallmark.yaml` must exist in your data directory. It declares:\n", - "- `fmt`: the Python format string for your file naming convention\n", - "- `encoding` (optional): regex patterns for parameters that use special filename conventions\n", - "\n", - "Here is the `.hallmark.yaml` used in this demo:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3", - "metadata": { - "execution": { - "iopub.execute_input": "2026-03-17T01:36:44.497846Z", - "iopub.status.busy": "2026-03-17T01:36:44.497728Z", - "iopub.status.idle": "2026-03-17T01:36:44.764964Z", - "shell.execute_reply": "2026-03-17T01:36:44.764361Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "data:\n", - "- fmt: '{mag:d}a{aspin}_w{win:d}.h5'\n", - " # path_to_fmt: m5/data\n", - " encoding:\n", - " aspin: m([0-9]+(\\.[0-9]+)?|\\.[0-9]+)\n", - "\n", - "- fmt: a_{a:d}/b_{b:d}.txt\n", - " # path_to_fmt: data\n", - "\n", - "- fmt: a{aspin}/b_{b:d}.txt\n", - " # path_to_fmt: data\n", - " encoding:\n", - " aspin: ''\n" - ] - } - ], - "source": [ - "! cat data/hallmark.yaml # check what fmts are present" - ] - }, - { - "cell_type": "markdown", - "id": "4", - "metadata": {}, - "source": [ - "## 2. Simple File Discovery (no encoding)\n", - "\n", - "Files structured as `a_{a:d}/b_{b:d}.txt` — no special encoding, so `encoding=False` (the default)." - ] - }, - { - "cell_type": "markdown", - "id": "5", - "metadata": {}, - "source": [ - "### Create sample data files" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "6", - "metadata": { - "execution": { - "iopub.execute_input": "2026-03-17T01:36:44.767668Z", - "iopub.status.busy": "2026-03-17T01:36:44.767500Z", - "iopub.status.idle": "2026-03-17T01:36:44.853271Z", - "shell.execute_reply": "2026-03-17T01:36:44.852483Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "b_10.txt\n", - "b_11.txt\n", - "b_12.txt\n", - "b_13.txt\n", - "\u001b[30m\u001b[47mb_14.txt\u001b[m\u001b[m\n" - ] - } - ], - "source": [ - "%%bash\n", - "# make some test files to parse\n", - "for a in {0..4}; do\n", - " mkdir -p \"data/a_$a\"\n", - " for b in {10..14}; do\n", - " touch \"data/a_$a/b_$b.txt\"\n", - " done\n", - "done\n", - "ls data/a_0/" - ] - }, - { - "cell_type": "markdown", - "id": "7", - "metadata": {}, - "source": [ - "### Parse with `repo_path`\n", - "\n", - "`repo_path` tells `ParaFrame` where to find `.hallmark.yaml` and where to search for files." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "8", - "metadata": { - "execution": { - "iopub.execute_input": "2026-03-17T01:36:44.855422Z", - "iopub.status.busy": "2026-03-17T01:36:44.855306Z", - "iopub.status.idle": "2026-03-17T01:36:45.283614Z", - "shell.execute_reply": "2026-03-17T01:36:45.283219Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Pattern: data/a_{a:d}/b_{b:d}.txt\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
pathab
0a_0/b_10.txt0.010.0
1a_0/b_11.txt0.011.0
2a_0/b_12.txt0.012.0
3a_0/b_13.txt0.013.0
4a_0/b_14.txt0.014.0
5a_1/b_10.txt1.010.0
6a_1/b_11.txt1.011.0
7a_1/b_12.txt1.012.0
8a_1/b_13.txt1.013.0
9a_1/b_14.txt1.014.0
10a_2/b_10.txt2.010.0
11a_2/b_11.txt2.011.0
12a_2/b_12.txt2.012.0
13a_2/b_13.txt2.013.0
14a_2/b_14.txt2.014.0
15a_3/b_10.txt3.010.0
16a_3/b_11.txt3.011.0
17a_3/b_12.txt3.012.0
18a_3/b_13.txt3.013.0
19a_3/b_14.txt3.014.0
20a_4/b_10.txt4.010.0
21a_4/b_11.txt4.011.0
22a_4/b_12.txt4.012.0
23a_4/b_13.txt4.013.0
24a_4/b_14.txt4.014.0
\n", - "
" - ], - "text/plain": [ - " path a b\n", - "0 a_0/b_10.txt 0.0 10.0\n", - "1 a_0/b_11.txt 0.0 11.0\n", - "2 a_0/b_12.txt 0.0 12.0\n", - "3 a_0/b_13.txt 0.0 13.0\n", - "4 a_0/b_14.txt 0.0 14.0\n", - "5 a_1/b_10.txt 1.0 10.0\n", - "6 a_1/b_11.txt 1.0 11.0\n", - "7 a_1/b_12.txt 1.0 12.0\n", - "8 a_1/b_13.txt 1.0 13.0\n", - "9 a_1/b_14.txt 1.0 14.0\n", - "10 a_2/b_10.txt 2.0 10.0\n", - "11 a_2/b_11.txt 2.0 11.0\n", - "12 a_2/b_12.txt 2.0 12.0\n", - "13 a_2/b_13.txt 2.0 13.0\n", - "14 a_2/b_14.txt 2.0 14.0\n", - "15 a_3/b_10.txt 3.0 10.0\n", - "16 a_3/b_11.txt 3.0 11.0\n", - "17 a_3/b_12.txt 3.0 12.0\n", - "18 a_3/b_13.txt 3.0 13.0\n", - "19 a_3/b_14.txt 3.0 14.0\n", - "20 a_4/b_10.txt 4.0 10.0\n", - "21 a_4/b_11.txt 4.0 11.0\n", - "22 a_4/b_12.txt 4.0 12.0\n", - "23 a_4/b_13.txt 4.0 13.0\n", - "24 a_4/b_14.txt 4.0 14.0" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from hallmark import ParaFrame\n", - "\n", - "# repo_path tells hallmark where to find .hallmark.yaml\n", - "pf = ParaFrame.parse(\"a_{a:d}/b_{b:d}.txt\", repo_path=\"data\")\n", - "pf" - ] - }, - { - "cell_type": "markdown", - "id": "9", - "metadata": {}, - "source": [ - "### Parse with `set_rel_yaml_path`\n", - "\n", - "Instead of passing `repo_path` every time, set it globally once." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "10", - "metadata": { - "execution": { - "iopub.execute_input": "2026-03-17T01:36:45.285073Z", - "iopub.status.busy": "2026-03-17T01:36:45.284949Z", - "iopub.status.idle": "2026-03-17T01:36:45.302480Z", - "shell.execute_reply": "2026-03-17T01:36:45.302088Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Pattern: data/a_{a:d}/b_{b:d}.txt\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
pathab
0a_0/b_10.txt0.010.0
1a_0/b_11.txt0.011.0
2a_0/b_12.txt0.012.0
3a_0/b_13.txt0.013.0
4a_0/b_14.txt0.014.0
5a_1/b_10.txt1.010.0
6a_1/b_11.txt1.011.0
7a_1/b_12.txt1.012.0
8a_1/b_13.txt1.013.0
9a_1/b_14.txt1.014.0
10a_2/b_10.txt2.010.0
11a_2/b_11.txt2.011.0
12a_2/b_12.txt2.012.0
13a_2/b_13.txt2.013.0
14a_2/b_14.txt2.014.0
15a_3/b_10.txt3.010.0
16a_3/b_11.txt3.011.0
17a_3/b_12.txt3.012.0
18a_3/b_13.txt3.013.0
19a_3/b_14.txt3.014.0
20a_4/b_10.txt4.010.0
21a_4/b_11.txt4.011.0
22a_4/b_12.txt4.012.0
23a_4/b_13.txt4.013.0
24a_4/b_14.txt4.014.0
\n", - "
" - ], - "text/plain": [ - " path a b\n", - "0 a_0/b_10.txt 0.0 10.0\n", - "1 a_0/b_11.txt 0.0 11.0\n", - "2 a_0/b_12.txt 0.0 12.0\n", - "3 a_0/b_13.txt 0.0 13.0\n", - "4 a_0/b_14.txt 0.0 14.0\n", - "5 a_1/b_10.txt 1.0 10.0\n", - "6 a_1/b_11.txt 1.0 11.0\n", - "7 a_1/b_12.txt 1.0 12.0\n", - "8 a_1/b_13.txt 1.0 13.0\n", - "9 a_1/b_14.txt 1.0 14.0\n", - "10 a_2/b_10.txt 2.0 10.0\n", - "11 a_2/b_11.txt 2.0 11.0\n", - "12 a_2/b_12.txt 2.0 12.0\n", - "13 a_2/b_13.txt 2.0 13.0\n", - "14 a_2/b_14.txt 2.0 14.0\n", - "15 a_3/b_10.txt 3.0 10.0\n", - "16 a_3/b_11.txt 3.0 11.0\n", - "17 a_3/b_12.txt 3.0 12.0\n", - "18 a_3/b_13.txt 3.0 13.0\n", - "19 a_3/b_14.txt 3.0 14.0\n", - "20 a_4/b_10.txt 4.0 10.0\n", - "21 a_4/b_11.txt 4.0 11.0\n", - "22 a_4/b_12.txt 4.0 12.0\n", - "23 a_4/b_13.txt 4.0 13.0\n", - "24 a_4/b_14.txt 4.0 14.0" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from hallmark import ParaFrame, set_rel_yaml_path\n", - "\n", - "# set once globally so you dont have to pass repo_path every time\n", - "set_rel_yaml_path(\"data\")\n", - "\n", - "pf = ParaFrame.parse(\"a_{a:d}/b_{b:d}.txt\")\n", - "pf" - ] - }, - { - "cell_type": "markdown", - "id": "11", - "metadata": {}, - "source": [ - "### Debug mode\n", - "\n", - "`debug=True` prints the resolved glob pattern and match count — useful when no files are found." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "12", - "metadata": { - "execution": { - "iopub.execute_input": "2026-03-17T01:36:45.303832Z", - "iopub.status.busy": "2026-03-17T01:36:45.303732Z", - "iopub.status.idle": "2026-03-17T01:36:45.316212Z", - "shell.execute_reply": "2026-03-17T01:36:45.315734Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Pattern: data/a_{a:d}/b_{b:d}.txt\n", - "0 data/a_{a:d}/b_{b:d}.txt () {}\n", - "1 data/a_{a:s}/b_{b:d}.txt () {'a': '*'}\n", - "2 data/a_{a:s}/b_{b:s}.txt () {'a': '*', 'b': '*'}\n", - "Pattern: \"data/a_*/b_*.txt\"\n", - "25 matches, e.g., \"data/a_0/b_10.txt\"\n" - ] - } - ], - "source": [ - "# debug=True shows the resolved glob pattern and how many files matched\n", - "pf = ParaFrame.parse(\"a_{a:d}/b_{b:d}.txt\", repo_path=\"data\", debug=True)" - ] - }, - { - "cell_type": "markdown", - "id": "13", - "metadata": {}, - "source": [ - "## 3. Filtering\n", - "\n", - "Call a `ParaFrame` like a function to filter by column values:\n", - "- **scalar** → exact match\n", - "- **list or tuple** → OR match\n", - "- **chaining** → AND logic" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "14", - "metadata": { - "execution": { - "iopub.execute_input": "2026-03-17T01:36:45.317526Z", - "iopub.status.busy": "2026-03-17T01:36:45.317420Z", - "iopub.status.idle": "2026-03-17T01:36:45.327893Z", - "shell.execute_reply": "2026-03-17T01:36:45.327195Z" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
pathab
0a_0/b_10.txt0.010.0
1a_0/b_11.txt0.011.0
2a_0/b_12.txt0.012.0
3a_0/b_13.txt0.013.0
4a_0/b_14.txt0.014.0
\n", - "
" - ], - "text/plain": [ - " path a b\n", - "0 a_0/b_10.txt 0.0 10.0\n", - "1 a_0/b_11.txt 0.0 11.0\n", - "2 a_0/b_12.txt 0.0 12.0\n", - "3 a_0/b_13.txt 0.0 13.0\n", - "4 a_0/b_14.txt 0.0 14.0" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Filter a == 0\n", - "pf(a=0)" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "15", - "metadata": { - "execution": { - "iopub.execute_input": "2026-03-17T01:36:45.329283Z", - "iopub.status.busy": "2026-03-17T01:36:45.329178Z", - "iopub.status.idle": "2026-03-17T01:36:45.338757Z", - "shell.execute_reply": "2026-03-17T01:36:45.338404Z" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
pathab
0a_0/b_10.txt0.010.0
1a_0/b_11.txt0.011.0
2a_0/b_12.txt0.012.0
3a_0/b_13.txt0.013.0
4a_0/b_14.txt0.014.0
5a_1/b_10.txt1.010.0
6a_1/b_11.txt1.011.0
7a_1/b_12.txt1.012.0
8a_1/b_13.txt1.013.0
9a_1/b_14.txt1.014.0
\n", - "
" - ], - "text/plain": [ - " path a b\n", - "0 a_0/b_10.txt 0.0 10.0\n", - "1 a_0/b_11.txt 0.0 11.0\n", - "2 a_0/b_12.txt 0.0 12.0\n", - "3 a_0/b_13.txt 0.0 13.0\n", - "4 a_0/b_14.txt 0.0 14.0\n", - "5 a_1/b_10.txt 1.0 10.0\n", - "6 a_1/b_11.txt 1.0 11.0\n", - "7 a_1/b_12.txt 1.0 12.0\n", - "8 a_1/b_13.txt 1.0 13.0\n", - "9 a_1/b_14.txt 1.0 14.0" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Filter a == 0 or a == 1\n", - "pf(a=[0, 1])" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "16", - "metadata": { - "execution": { - "iopub.execute_input": "2026-03-17T01:36:45.340105Z", - "iopub.status.busy": "2026-03-17T01:36:45.340030Z", - "iopub.status.idle": "2026-03-17T01:36:45.348202Z", - "shell.execute_reply": "2026-03-17T01:36:45.347785Z" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
pathab
0a_0/b_10.txt0.010.0
\n", - "
" - ], - "text/plain": [ - " path a b\n", - "0 a_0/b_10.txt 0.0 10.0" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Filter a == 0 AND b == 10\n", - "pf(a=0)(b=10)" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "17", - "metadata": { - "execution": { - "iopub.execute_input": "2026-03-17T01:36:45.349400Z", - "iopub.status.busy": "2026-03-17T01:36:45.349317Z", - "iopub.status.idle": "2026-03-17T01:36:45.358152Z", - "shell.execute_reply": "2026-03-17T01:36:45.357766Z" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
pathab
10a_2/b_10.txt2.010.0
11a_2/b_11.txt2.011.0
12a_2/b_12.txt2.012.0
13a_2/b_13.txt2.013.0
14a_2/b_14.txt2.014.0
15a_3/b_10.txt3.010.0
16a_3/b_11.txt3.011.0
17a_3/b_12.txt3.012.0
18a_3/b_13.txt3.013.0
19a_3/b_14.txt3.014.0
20a_4/b_10.txt4.010.0
21a_4/b_11.txt4.011.0
22a_4/b_12.txt4.012.0
23a_4/b_13.txt4.013.0
24a_4/b_14.txt4.014.0
\n", - "
" - ], - "text/plain": [ - " path a b\n", - "10 a_2/b_10.txt 2.0 10.0\n", - "11 a_2/b_11.txt 2.0 11.0\n", - "12 a_2/b_12.txt 2.0 12.0\n", - "13 a_2/b_13.txt 2.0 13.0\n", - "14 a_2/b_14.txt 2.0 14.0\n", - "15 a_3/b_10.txt 3.0 10.0\n", - "16 a_3/b_11.txt 3.0 11.0\n", - "17 a_3/b_12.txt 3.0 12.0\n", - "18 a_3/b_13.txt 3.0 13.0\n", - "19 a_3/b_14.txt 3.0 14.0\n", - "20 a_4/b_10.txt 4.0 10.0\n", - "21 a_4/b_11.txt 4.0 11.0\n", - "22 a_4/b_12.txt 4.0 12.0\n", - "23 a_4/b_13.txt 4.0 13.0\n", - "24 a_4/b_14.txt 4.0 14.0" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# use pandas masking directly for more complex conditions\n", - "pf[(2 <= pf.a) & (pf.a <= 4)]" - ] - }, - { - "cell_type": "markdown", - "id": "18", - "metadata": {}, - "source": [ - "### Using filtered paths in a workflow" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "19", - "metadata": { - "execution": { - "iopub.execute_input": "2026-03-17T01:36:45.359332Z", - "iopub.status.busy": "2026-03-17T01:36:45.359244Z", - "iopub.status.idle": "2026-03-17T01:36:45.366724Z", - "shell.execute_reply": "2026-03-17T01:36:45.366336Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Processing \"a_0/b_10.txt\"...\n", - "Processing \"a_0/b_11.txt\"...\n", - "Processing \"a_0/b_12.txt\"...\n", - "Processing \"a_0/b_13.txt\"...\n", - "Processing \"a_0/b_14.txt\"...\n", - "Processing \"a_1/b_10.txt\"...\n", - "Processing \"a_2/b_10.txt\"...\n", - "Processing \"a_3/b_10.txt\"...\n", - "Processing \"a_4/b_10.txt\"...\n" - ] - } - ], - "source": [ - "# pf(a=0, b=10) means a==0 OR b==10\n", - "for p in pf(a=0, b=10).path:\n", - " print(f'Processing \"{p}\"...')" - ] - }, - { - "cell_type": "markdown", - "id": "20", - "metadata": {}, - "source": [ - "## 4. Encoding — Special Filename Conventions\n", - "\n", - "Some datasets use non-standard filename conventions. For example, black hole spin values are stored as `m0.94` to represent `-0.94` (the `m` prefix means \"minus\").\n", - "\n", - "The `.hallmark.yaml` handles this with the `encoding` key, which holds a regex pattern. When `encoding=True` is passed, the regex substitution is applied to the filename before parsing.\n", - "\n", - "Relevant YAML entry:\n", - "```yaml\n", - "- fmt: '{mag:d}a{aspin}_w{win:d}.h5'\n", - " encoding:\n", - " aspin: m([0-9]+(\\.[0-9]+)?|\\.[0-9]+)\n", - "```" - ] - }, - { - "cell_type": "markdown", - "id": "21", - "metadata": {}, - "source": [ - "### Create sample spin data files" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "22", - "metadata": { - "execution": { - "iopub.execute_input": "2026-03-17T01:36:45.368197Z", - "iopub.status.busy": "2026-03-17T01:36:45.368111Z", - "iopub.status.idle": "2026-03-17T01:36:45.421524Z", - "shell.execute_reply": "2026-03-17T01:36:45.421043Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "data/1a0_w10.h5\n", - "data/1a0_w20.h5\n", - "data/1a0.5_w10.h5\n", - "data/1a0.5_w20.h5\n", - "data/1am0.5_w10.h5\n", - "data/1am0.5_w20.h5\n", - "data/1am0.94_w10.h5\n", - "data/1am0.94_w20.h5\n", - "data/2a0_w10.h5\n", - "data/2a0_w20.h5\n", - "data/2a0.5_w10.h5\n", - "data/2a0.5_w20.h5\n", - "data/2am0.5_w10.h5\n", - "data/2am0.5_w20.h5\n", - "data/2am0.94_w10.h5\n", - "data/2am0.94_w20.h5\n" - ] - } - ], - "source": [ - "%%bash\n", - "# spin values use m prefix to mean negative e.g. m0.94 = -0.94\n", - "for mag in 1 2; do\n", - " for aspin in 0 0.5 m0.5 m0.94; do\n", - " for win in 10 20; do\n", - " touch \"data/${mag}a${aspin}_w${win}.h5\"\n", - " done\n", - " done\n", - "done\n", - "ls data/*.h5" - ] - }, - { - "cell_type": "markdown", - "id": "23", - "metadata": {}, - "source": [ - "### Parse with `encoding=True`\n", - "\n", - "The `m` prefix is decoded by the regex: `m0.94` → `-0.94`, `m0.5` → `-0.5`." - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "24", - "metadata": { - "execution": { - "iopub.execute_input": "2026-03-17T01:36:45.422874Z", - "iopub.status.busy": "2026-03-17T01:36:45.422783Z", - "iopub.status.idle": "2026-03-17T01:36:45.438474Z", - "shell.execute_reply": "2026-03-17T01:36:45.438096Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Pattern: data/{mag:d}a{aspin}_w{win:d}.h5\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
pathmagaspinwin
01a0.5_w10.h51.00.5010.0
11a0.5_w20.h51.00.5020.0
21a0_w10.h51.00.0010.0
31a0_w20.h51.00.0020.0
41am0.5_w10.h51.0-0.5010.0
51am0.5_w20.h51.0-0.5020.0
61am0.94_w10.h51.0-0.9410.0
71am0.94_w20.h51.0-0.9420.0
82a0.5_w10.h52.00.5010.0
92a0.5_w20.h52.00.5020.0
102a0_w10.h52.00.0010.0
112a0_w20.h52.00.0020.0
122am0.5_w10.h52.0-0.5010.0
132am0.5_w20.h52.0-0.5020.0
142am0.94_w10.h52.0-0.9410.0
152am0.94_w20.h52.0-0.9420.0
\n", - "
" - ], - "text/plain": [ - " path mag aspin win\n", - "0 1a0.5_w10.h5 1.0 0.50 10.0\n", - "1 1a0.5_w20.h5 1.0 0.50 20.0\n", - "2 1a0_w10.h5 1.0 0.00 10.0\n", - "3 1a0_w20.h5 1.0 0.00 20.0\n", - "4 1am0.5_w10.h5 1.0 -0.50 10.0\n", - "5 1am0.5_w20.h5 1.0 -0.50 20.0\n", - "6 1am0.94_w10.h5 1.0 -0.94 10.0\n", - "7 1am0.94_w20.h5 1.0 -0.94 20.0\n", - "8 2a0.5_w10.h5 2.0 0.50 10.0\n", - "9 2a0.5_w20.h5 2.0 0.50 20.0\n", - "10 2a0_w10.h5 2.0 0.00 10.0\n", - "11 2a0_w20.h5 2.0 0.00 20.0\n", - "12 2am0.5_w10.h5 2.0 -0.50 10.0\n", - "13 2am0.5_w20.h5 2.0 -0.50 20.0\n", - "14 2am0.94_w10.h5 2.0 -0.94 10.0\n", - "15 2am0.94_w20.h5 2.0 -0.94 20.0" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# encoding=True applies the regex from .hallmark.yaml before parsing\n", - "# so m0.94 in the filename becomes -0.94 in the dataframe\n", - "pf_spin = ParaFrame.parse(\"{mag:d}a{aspin}_w{win:d}.h5\", repo_path=\"data\", encoding=True)\n", - "pf_spin" - ] - }, - { - "cell_type": "markdown", - "id": "25", - "metadata": {}, - "source": [ - "The `aspin` column contains decoded values (e.g. `-0.94`) even though the filenames use `m0.94`. Filtering uses the decoded values:" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "id": "26", - "metadata": { - "execution": { - "iopub.execute_input": "2026-03-17T01:36:45.439842Z", - "iopub.status.busy": "2026-03-17T01:36:45.439764Z", - "iopub.status.idle": "2026-03-17T01:36:45.451332Z", - "shell.execute_reply": "2026-03-17T01:36:45.451016Z" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
pathmagaspinwin
41am0.5_w10.h51.0-0.5010.0
51am0.5_w20.h51.0-0.5020.0
61am0.94_w10.h51.0-0.9410.0
71am0.94_w20.h51.0-0.9420.0
122am0.5_w10.h52.0-0.5010.0
132am0.5_w20.h52.0-0.5020.0
142am0.94_w10.h52.0-0.9410.0
152am0.94_w20.h52.0-0.9420.0
\n", - "
" - ], - "text/plain": [ - " path mag aspin win\n", - "4 1am0.5_w10.h5 1.0 -0.50 10.0\n", - "5 1am0.5_w20.h5 1.0 -0.50 20.0\n", - "6 1am0.94_w10.h5 1.0 -0.94 10.0\n", - "7 1am0.94_w20.h5 1.0 -0.94 20.0\n", - "12 2am0.5_w10.h5 2.0 -0.50 10.0\n", - "13 2am0.5_w20.h5 2.0 -0.50 20.0\n", - "14 2am0.94_w10.h5 2.0 -0.94 10.0\n", - "15 2am0.94_w20.h5 2.0 -0.94 20.0" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# filter using the decoded float values not the raw filename strings\n", - "pf_spin(aspin=[-0.5, -0.94])" - ] - }, - { - "cell_type": "markdown", - "id": "27", - "metadata": {}, - "source": [ - "### Encoding validation\n", - "\n", - "`ParaFrame` enforces consistency between your call and the YAML spec:\n", - "- If the YAML has a regex for a parameter, you **must** pass `encoding=True`\n", - "- If the YAML has no regex, you **must** use `encoding=False` (the default)\n", - "\n", - "This prevents silent mis-parses." - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "id": "28", - "metadata": { - "execution": { - "iopub.execute_input": "2026-03-17T01:36:45.452772Z", - "iopub.status.busy": "2026-03-17T01:36:45.452695Z", - "iopub.status.idle": "2026-03-17T01:36:45.461001Z", - "shell.execute_reply": "2026-03-17T01:36:45.460544Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Error: '{mag:d}a{aspin}_w{win:d}.h5' has a regex spec, \n", - " so you must use encoding=True\n" - ] - } - ], - "source": [ - "# hallmark raises an error if you forget encoding=True when the yaml has a regex\n", - "try:\n", - " ParaFrame.parse(\"{mag:d}a{aspin}_w{win:d}.h5\", repo_path=\"data\", encoding=False)\n", - "except ValueError as e:\n", - " print(e)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "hallmark", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.12" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/demo/demo_python.ipynb b/demo/demo_python.ipynb index 05fb0fe..cbc7e24 100644 --- a/demo/demo_python.ipynb +++ b/demo/demo_python.ipynb @@ -2,218 +2,240 @@ "cells": [ { "cell_type": "markdown", - "id": "0", + "id": "md-title", "metadata": {}, "source": [ - "# `Hallmark` Demo\n", + "# Hallmark Python Demo\n", "\n", - "This Jupyter notebook demonstrates how `hallmark` is used." + "Hallmark helps you keep track of data files in scientific workflows.\n", + "It stores a file index inside a small git repository (`.hm`) that lives\n", + "next to your data, so you always know which files belong to which run.\n", + "\n", + "This notebook walks through the full workflow:\n", + "1. Create a hallmark repository\n", + "2. Add files and commit\n", + "3. Discover and filter files with `ParaFrame`\n", + "4. Handle custom filename conventions with encoding" ] }, { "cell_type": "code", - "execution_count": 75, - "id": "1", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The autoreload extension is already loaded. To reload it, use:\n", - " %reload_ext autoreload\n" - ] + "execution_count": 1, + "id": "code-setup", + "metadata": { + "execution": { + "iopub.execute_input": "2026-04-02T23:09:18.373603Z", + "iopub.status.busy": "2026-04-02T23:09:18.373314Z", + "iopub.status.idle": "2026-04-02T23:09:19.238495Z", + "shell.execute_reply": "2026-04-02T23:09:19.237979Z" } - ], + }, + "outputs": [], "source": [ + "# auto-reload modules when they change\n", "%load_ext autoreload\n", - "%autoreload 2" - ] - }, - { - "cell_type": "markdown", - "id": "2", - "metadata": {}, - "source": [ - "## Remove Old Demo Repositories\n", + "%autoreload 2\n", "\n", - "First, let's remove all old demo repositories." + "from hallmark import Repo, ParaFrame" ] }, { "cell_type": "code", - "execution_count": 76, - "id": "3", - "metadata": {}, + "execution_count": 2, + "id": "code-cleanup", + "metadata": { + "execution": { + "iopub.execute_input": "2026-04-02T23:09:19.240387Z", + "iopub.status.busy": "2026-04-02T23:09:19.240145Z", + "iopub.status.idle": "2026-04-02T23:09:19.542023Z", + "shell.execute_reply": "2026-04-02T23:09:19.541351Z" + } + }, "outputs": [], "source": [ + "# remove any repositories left over from a previous run\n", "! rm -rf repo*" ] }, { "cell_type": "markdown", - "id": "4", + "id": "md-s1", "metadata": {}, "source": [ - "## Initialize a Standard (Non-Bare) `Hallmark` Repository\n", + "## 1. Create a Repository\n", "\n", - "`Hallmark` is similar to `git`.\n", - "To create an empty (standard) `hallmark` repository, we call `hallmark.Repo.init(REPO)`." + "`Repo.init(path)` creates a new hallmark repository at `path`.\n", + "It works like `git init`: the directory is created with a `.hm` folder\n", + "inside, which is itself a git repository used to store the file index." ] }, { "cell_type": "code", - "execution_count": 77, - "id": "5", - "metadata": {}, + "execution_count": 3, + "id": "code-init", + "metadata": { + "execution": { + "iopub.execute_input": "2026-04-02T23:09:19.543922Z", + "iopub.status.busy": "2026-04-02T23:09:19.543803Z", + "iopub.status.idle": "2026-04-02T23:09:19.643693Z", + "shell.execute_reply": "2026-04-02T23:09:19.643162Z" + } + }, "outputs": [], "source": [ - "from hallmark import Repo\n", - "\n", "repo1 = Repo.init(\"repo1\")" ] }, - { - "cell_type": "markdown", - "id": "6", - "metadata": {}, - "source": [ - "Check the content of the hallmark repository.\n", - "It contains a \".hm\" directory, which is itself a git repository." - ] - }, { "cell_type": "code", - "execution_count": 78, - "id": "7", - "metadata": {}, + "execution_count": 4, + "id": "code-ls-repo", + "metadata": { + "execution": { + "iopub.execute_input": "2026-04-02T23:09:19.645744Z", + "iopub.status.busy": "2026-04-02T23:09:19.645621Z", + "iopub.status.idle": "2026-04-02T23:09:19.911367Z", + "shell.execute_reply": "2026-04-02T23:09:19.910878Z" + } + }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "repo1:\n", - ". .. .hm\n", - "\n", - "repo1/.hm:\n", - ". .. .git README.md\tconfig.yml data.tsv meta.yml\n" + "repo1:\r\n", + "\u001b[1m\u001b[36m.\u001b[m\u001b[m \u001b[1m\u001b[36m..\u001b[m\u001b[m \u001b[1m\u001b[36m.hm\u001b[m\u001b[m\r\n", + "\r\n", + "repo1/.hm:\r\n", + "\u001b[1m\u001b[36m.\u001b[m\u001b[m \u001b[1m\u001b[36m..\u001b[m\u001b[m \u001b[1m\u001b[36m.git\u001b[m\u001b[m config.yml data.tsv meta.yml README.md\r\n" ] } ], "source": [ + "# repo1/ holds your data files; repo1/.hm/ holds the hallmark index\n", "! ls -a repo1 repo1/.hm" ] }, - { - "cell_type": "markdown", - "id": "8", - "metadata": {}, - "source": [ - "We can `cd` into the \".hm\" directory and then run `git`.\n", - "However, because `cd` is not a very good practice in Jupyter notebook,\n", - "we will use `pushd` and `popd` instead." - ] - }, { "cell_type": "code", - "execution_count": 79, - "id": "9", - "metadata": {}, + "execution_count": 5, + "id": "code-git-status", + "metadata": { + "execution": { + "iopub.execute_input": "2026-04-02T23:09:19.913428Z", + "iopub.status.busy": "2026-04-02T23:09:19.913306Z", + "iopub.status.idle": "2026-04-02T23:09:20.192212Z", + "shell.execute_reply": "2026-04-02T23:09:20.191537Z" + } + }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "~/EHT/hallmark/demo/repo1/.hm ~/EHT/hallmark/demo\n", - "On branch master\n", - "Changes to be committed:\n", - " (use \"git restore --staged ...\" to unstage)\n", - "\t\u001b[32mnew file: config.yml\u001b[m\n", - "\t\u001b[32mnew file: data.tsv\u001b[m\n", - "\t\u001b[32mnew file: meta.yml\u001b[m\n", - "\n", - "~/EHT/hallmark/demo\n" + "On branch master\r\n", + "Changes to be committed:\r\n", + " (use \"git restore --staged ...\" to unstage)\r\n", + "\t\u001b[32mnew file: config.yml\u001b[m\r\n", + "\t\u001b[32mnew file: data.tsv\u001b[m\r\n", + "\t\u001b[32mnew file: meta.yml\u001b[m\r\n", + "\r\n" ] } ], "source": [ + "# the .hm directory is a git repo — you can run git commands inside it\n", "! pushd repo1/.hm && git status && popd" ] }, { "cell_type": "markdown", - "id": "10", + "id": "md-s1-note", "metadata": {}, "source": [ - "Note, similar to `git init`, `hallmark.Repo.init()` does not commit your files." + "The index files (`config.yml`, `data.tsv`, `meta.yml`) are staged but not\n", + "yet committed — just like `git init`, `Repo.init()` does not make the first\n", + "commit for you." ] }, { "cell_type": "markdown", - "id": "11", + "id": "md-bare", "metadata": {}, "source": [ - "## Initialize a Bare Hallmark Repository\n", + "### Bare repository\n", "\n", - "Next, we will initialize a bare repository.\n", - "Unlikely `git`, all we need is to explicitly postfix \".hm\" to the repository name." + "A bare repository has no data directory — it is just the index.\n", + "Name the path with a `.hm` suffix to create one." ] }, { "cell_type": "code", - "execution_count": 80, - "id": "12", - "metadata": {}, + "execution_count": 6, + "id": "code-bare-init", + "metadata": { + "execution": { + "iopub.execute_input": "2026-04-02T23:09:20.194165Z", + "iopub.status.busy": "2026-04-02T23:09:20.194035Z", + "iopub.status.idle": "2026-04-02T23:09:20.263443Z", + "shell.execute_reply": "2026-04-02T23:09:20.262965Z" + } + }, "outputs": [], "source": [ "repo2 = Repo.init(\"repo2.hm\")" ] }, - { - "cell_type": "markdown", - "id": "13", - "metadata": {}, - "source": [ - "`repo2.hm` is itself a `git` repository." - ] - }, { "cell_type": "code", - "execution_count": 81, - "id": "14", - "metadata": {}, + "execution_count": 7, + "id": "code-bare-ls", + "metadata": { + "execution": { + "iopub.execute_input": "2026-04-02T23:09:20.265247Z", + "iopub.status.busy": "2026-04-02T23:09:20.265126Z", + "iopub.status.idle": "2026-04-02T23:09:20.530462Z", + "shell.execute_reply": "2026-04-02T23:09:20.529938Z" + } + }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - ". .. .git README.md\tconfig.yml data.tsv meta.yml\n" + "\u001b[1m\u001b[36m.\u001b[m\u001b[m \u001b[1m\u001b[36m..\u001b[m\u001b[m \u001b[1m\u001b[36m.git\u001b[m\u001b[m config.yml data.tsv meta.yml README.md\r\n" ] } ], "source": [ + "# repo2.hm/ is the index itself — there is no separate data folder\n", "! ls -a repo2.hm" ] }, { "cell_type": "code", - "execution_count": 82, - "id": "15", - "metadata": {}, + "execution_count": 8, + "id": "code-bare-git-status", + "metadata": { + "execution": { + "iopub.execute_input": "2026-04-02T23:09:20.532801Z", + "iopub.status.busy": "2026-04-02T23:09:20.532654Z", + "iopub.status.idle": "2026-04-02T23:09:20.819552Z", + "shell.execute_reply": "2026-04-02T23:09:20.818822Z" + } + }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "~/EHT/hallmark/demo/repo2.hm ~/EHT/hallmark/demo\n", - "On branch master\n", - "Changes to be committed:\n", - " (use \"git restore --staged ...\" to unstage)\n", - "\t\u001b[32mnew file: config.yml\u001b[m\n", - "\t\u001b[32mnew file: data.tsv\u001b[m\n", - "\t\u001b[32mnew file: meta.yml\u001b[m\n", - "\n", - "~/EHT/hallmark/demo\n" + "On branch master\r\n", + "Changes to be committed:\r\n", + " (use \"git restore --staged ...\" to unstage)\r\n", + "\t\u001b[32mnew file: config.yml\u001b[m\r\n", + "\t\u001b[32mnew file: data.tsv\u001b[m\r\n", + "\t\u001b[32mnew file: meta.yml\u001b[m\r\n", + "\r\n" ] } ], @@ -223,41 +245,130 @@ }, { "cell_type": "markdown", - "id": "16", + "id": "md-s2", "metadata": {}, "source": [ - "## Add Files According to Python Format String" + "## 2. Add and Commit Files\n", + "\n", + "`repo.add(fmt)` scans `repo1/` for files whose names match the format\n", + "string, computes a checksum for each one, and stages them in the index.\n", + "It returns a table showing the matched files and their parameters.\n", + "\n", + "`repo.commit(msg)` saves the current state of the index as a git commit\n", + "inside `.hm`, giving you a permanent snapshot you can return to later." ] }, { "cell_type": "code", - "execution_count": 83, - "id": "17", - "metadata": {}, + "execution_count": 9, + "id": "code-touch-files", + "metadata": { + "execution": { + "iopub.execute_input": "2026-04-02T23:09:20.821599Z", + "iopub.status.busy": "2026-04-02T23:09:20.821475Z", + "iopub.status.idle": "2026-04-02T23:09:20.851435Z", + "shell.execute_reply": "2026-04-02T23:09:20.850987Z" + } + }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "~/EHT/hallmark/demo/repo1 ~/EHT/hallmark/demo\n", - "~/EHT/hallmark/demo\n", - "a0.75_i0.h5 a0.75_i30.h5 a0.94_i120.h5 a0.94_i60.h5 a0_i150.h5 a0_i90.h5\n", - "a0.75_i120.h5 a0.75_i60.h5 a0.94_i150.h5 a0.94_i90.h5 a0_i180.h5\n", - "a0.75_i150.h5 a0.75_i90.h5 a0.94_i180.h5 a0_i0.h5\t a0_i30.h5\n", - "a0.75_i180.h5 a0.94_i0.h5 a0.94_i30.h5 a0_i120.h5\t a0_i60.h5\n" + "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallma" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "rk/demo/repo1 ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/github/hallmark/demo\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallma" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "rk/demo\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "a0_i0.h5\n", + "a0_i120.h5\n", + "a0_i150.h5\n", + "a0_i180.h5\n", + "a0_i30.h5\n", + "a0_i60.h5\n", + "a0_i90.h5\n", + "a0.75_i0.h5\n", + "a0.75_i120.h5\n", + "a0" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + ".75_i150.h5\n", + "a0.75_i180.h5\n", + "a0.75_i30.h5\n", + "a0.75_i60.h5\n", + "a0.75_i90.h5\n", + "a0.94_i0.h5\n", + "a0.94_i120.h5\n", + "a0.94_i15" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0.h5\n", + "a0.94_i180.h5\n", + "a0.94_i30.h5\n", + "a0.94_i60.h5\n", + "a0.94_i90.h5\n" ] } ], "source": [ - "! pushd repo1 && touch a{0,0.75,0.94}_i{0..180..30}.h5 && popd\n", - "! ls repo1" + "%%bash\n", + "# create sample data files: spin values 0, 0.75, 0.94 at inclinations 0–180\n", + "pushd repo1\n", + "touch a{0,0.75,0.94}_i{0,30,60,90,120,150,180}.h5\n", + "popd\n", + "ls repo1" ] }, { "cell_type": "code", - "execution_count": 84, - "id": "18", - "metadata": {}, + "execution_count": 10, + "id": "code-add", + "metadata": { + "execution": { + "iopub.execute_input": "2026-04-02T23:09:20.853005Z", + "iopub.status.busy": "2026-04-02T23:09:20.852903Z", + "iopub.status.idle": "2026-04-02T23:09:20.888112Z", + "shell.execute_reply": "2026-04-02T23:09:20.887629Z" + } + }, "outputs": [ { "data": { @@ -463,20 +574,29 @@ "20 a0_i90.h5 0.00 90.0 da39a3ee5e6b4b0d3255bfef95601890afd80709" ] }, - "execution_count": 84, + "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "repo1.add(\"a{a}_i{i}.h5\")" + "# scan repo1/ for files matching the pattern and record them in the index\n", + "pf = repo1.add(\"a{a}_i{i}.h5\")\n", + "pf" ] }, { "cell_type": "code", - "execution_count": 85, - "id": "19", - "metadata": {}, + "execution_count": 11, + "id": "code-commit", + "metadata": { + "execution": { + "iopub.execute_input": "2026-04-02T23:09:20.889534Z", + "iopub.status.busy": "2026-04-02T23:09:20.889431Z", + "iopub.status.idle": "2026-04-02T23:09:20.946351Z", + "shell.execute_reply": "2026-04-02T23:09:20.945794Z" + } + }, "outputs": [ { "data": { @@ -484,84 +604,43 @@ "True" ] }, - "execution_count": 85, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "repo1.commit(\"Commit test\")" + "# save this snapshot — returns True if a commit was made\n", + "repo1.commit(\"add spin and inclination files\")" ] }, { "cell_type": "markdown", - "id": "c7ca72f0", - "metadata": {}, - "source": [ - "## Use Case with Encoding" - ] - }, - { - "cell_type": "code", - "execution_count": 86, - "id": "b9326142", + "id": "md-s3", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "~/EHT/hallmark/demo/repo1 ~/EHT/hallmark/demo\n", - "~/EHT/hallmark/demo\n" - ] - } - ], "source": [ - "%%bash\n", - "pushd repo1\n", + "## 3. Discover and Filter Files with ParaFrame\n", "\n", - "touch am0.75_i0.h5 \\\n", - " am0.75_i30.h5 \\\n", - " am0.75_i60.h5 \\\n", - " am0.94_i0.h5 \\\n", - " am0.94_i30.h5 \\\n", - " am0.94_i+30.h5 \\\n", - " am0.94_i60.h5\n", + "`repo.add()` uses `ParaFrame` under the hood to find files.\n", + "You can also call `ParaFrame.parse()` directly — useful when you want to\n", + "look up files without adding them to the index.\n", "\n", - "popd" - ] - }, - { - "cell_type": "code", - "execution_count": 87, - "id": "8571f48d", - "metadata": {}, - "outputs": [], - "source": [ - "%%bash\n", - "cat > repo1/.hm/config.yml <<'EOF'\n", - "encodings:\n", - " - fmt: \"a{aspin}_i{i}.h5\"\n", - " encoding:\n", - " aspin: \"m([0-9]+(\\\\.[0-9]+)?)\"\n", - "EOF" - ] - }, - { - "cell_type": "code", - "execution_count": 88, - "id": "36f58ff9", - "metadata": {}, - "outputs": [], - "source": [ - "repo1.state.config = repo1.dothm.load_yml(\"config\")" + "`ParaFrame` is a table (a pandas DataFrame) where each row is a file and\n", + "each column is a parameter read from the file name." ] }, { "cell_type": "code", - "execution_count": 89, - "id": "3c10efb2", - "metadata": {}, + "execution_count": 12, + "id": "code-pf-parse", + "metadata": { + "execution": { + "iopub.execute_input": "2026-04-02T23:09:20.948276Z", + "iopub.status.busy": "2026-04-02T23:09:20.948157Z", + "iopub.status.idle": "2026-04-02T23:09:20.963498Z", + "shell.execute_reply": "2026-04-02T23:09:20.963012Z" + } + }, "outputs": [ { "data": { @@ -585,9 +664,8 @@ " \n", " \n", " path\n", - " aspin\n", + " a\n", " i\n", - " sha1\n", " \n", " \n", " \n", @@ -596,246 +674,1719 @@ " a0.75_i0.h5\n", " 0.75\n", " 0.0\n", - " da39a3ee5e6b4b0d3255bfef95601890afd80709\n", " \n", " \n", " 1\n", " a0.75_i120.h5\n", " 0.75\n", " 120.0\n", - " da39a3ee5e6b4b0d3255bfef95601890afd80709\n", " \n", " \n", " 2\n", " a0.75_i150.h5\n", " 0.75\n", " 150.0\n", - " da39a3ee5e6b4b0d3255bfef95601890afd80709\n", " \n", " \n", " 3\n", " a0.75_i180.h5\n", " 0.75\n", " 180.0\n", - " da39a3ee5e6b4b0d3255bfef95601890afd80709\n", " \n", " \n", " 4\n", " a0.75_i30.h5\n", " 0.75\n", " 30.0\n", - " da39a3ee5e6b4b0d3255bfef95601890afd80709\n", " \n", " \n", " 5\n", " a0.75_i60.h5\n", " 0.75\n", " 60.0\n", - " da39a3ee5e6b4b0d3255bfef95601890afd80709\n", " \n", " \n", " 6\n", " a0.75_i90.h5\n", " 0.75\n", " 90.0\n", - " da39a3ee5e6b4b0d3255bfef95601890afd80709\n", " \n", " \n", " 7\n", " a0.94_i0.h5\n", " 0.94\n", " 0.0\n", - " da39a3ee5e6b4b0d3255bfef95601890afd80709\n", " \n", " \n", " 8\n", " a0.94_i120.h5\n", " 0.94\n", " 120.0\n", - " da39a3ee5e6b4b0d3255bfef95601890afd80709\n", " \n", " \n", " 9\n", " a0.94_i150.h5\n", " 0.94\n", " 150.0\n", - " da39a3ee5e6b4b0d3255bfef95601890afd80709\n", " \n", " \n", " 10\n", " a0.94_i180.h5\n", " 0.94\n", " 180.0\n", - " da39a3ee5e6b4b0d3255bfef95601890afd80709\n", " \n", " \n", " 11\n", " a0.94_i30.h5\n", " 0.94\n", " 30.0\n", - " da39a3ee5e6b4b0d3255bfef95601890afd80709\n", " \n", " \n", " 12\n", " a0.94_i60.h5\n", " 0.94\n", " 60.0\n", - " da39a3ee5e6b4b0d3255bfef95601890afd80709\n", " \n", " \n", " 13\n", " a0.94_i90.h5\n", " 0.94\n", " 90.0\n", - " da39a3ee5e6b4b0d3255bfef95601890afd80709\n", " \n", " \n", " 14\n", " a0_i0.h5\n", " 0.00\n", " 0.0\n", - " da39a3ee5e6b4b0d3255bfef95601890afd80709\n", " \n", " \n", " 15\n", " a0_i120.h5\n", " 0.00\n", " 120.0\n", - " da39a3ee5e6b4b0d3255bfef95601890afd80709\n", " \n", " \n", " 16\n", " a0_i150.h5\n", " 0.00\n", " 150.0\n", - " da39a3ee5e6b4b0d3255bfef95601890afd80709\n", " \n", " \n", " 17\n", " a0_i180.h5\n", " 0.00\n", " 180.0\n", - " da39a3ee5e6b4b0d3255bfef95601890afd80709\n", " \n", " \n", " 18\n", " a0_i30.h5\n", " 0.00\n", " 30.0\n", - " da39a3ee5e6b4b0d3255bfef95601890afd80709\n", " \n", " \n", " 19\n", " a0_i60.h5\n", " 0.00\n", " 60.0\n", - " da39a3ee5e6b4b0d3255bfef95601890afd80709\n", " \n", " \n", " 20\n", " a0_i90.h5\n", " 0.00\n", " 90.0\n", - " da39a3ee5e6b4b0d3255bfef95601890afd80709\n", " \n", + " \n", + "\n", + "" + ], + "text/plain": [ + " path a i\n", + "0 a0.75_i0.h5 0.75 0.0\n", + "1 a0.75_i120.h5 0.75 120.0\n", + "2 a0.75_i150.h5 0.75 150.0\n", + "3 a0.75_i180.h5 0.75 180.0\n", + "4 a0.75_i30.h5 0.75 30.0\n", + "5 a0.75_i60.h5 0.75 60.0\n", + "6 a0.75_i90.h5 0.75 90.0\n", + "7 a0.94_i0.h5 0.94 0.0\n", + "8 a0.94_i120.h5 0.94 120.0\n", + "9 a0.94_i150.h5 0.94 150.0\n", + "10 a0.94_i180.h5 0.94 180.0\n", + "11 a0.94_i30.h5 0.94 30.0\n", + "12 a0.94_i60.h5 0.94 60.0\n", + "13 a0.94_i90.h5 0.94 90.0\n", + "14 a0_i0.h5 0.00 0.0\n", + "15 a0_i120.h5 0.00 120.0\n", + "16 a0_i150.h5 0.00 150.0\n", + "17 a0_i180.h5 0.00 180.0\n", + "18 a0_i30.h5 0.00 30.0\n", + "19 a0_i60.h5 0.00 60.0\n", + "20 a0_i90.h5 0.00 90.0" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# discover files in repo1/ directly — no index update\n", + "pf = ParaFrame.parse(\"a{a}_i{i}.h5\", base_path=\"repo1\")\n", + "pf" + ] + }, + { + "cell_type": "markdown", + "id": "md-s3-debug", + "metadata": {}, + "source": [ + "### Debug mode\n", + "\n", + "`debug=True` prints the glob pattern used to search for files.\n", + "This helps you check the pattern when no files are found." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "code-debug", + "metadata": { + "execution": { + "iopub.execute_input": "2026-04-02T23:09:20.965095Z", + "iopub.status.busy": "2026-04-02T23:09:20.964973Z", + "iopub.status.idle": "2026-04-02T23:09:20.978901Z", + "shell.execute_reply": "2026-04-02T23:09:20.978605Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0 repo1/a{a}_i{i}.h5 () {}\n", + "1 repo1/a{a:s}_i{i}.h5 () {'a': '*'}\n", + "2 repo1/a{a:s}_i{i:s}.h5 () {'a': '*', 'i': '*'}\n", + "Pattern: \"repo1/a*_i*.h5\"\n", + "21 matches, e.g., \"repo1/a0.75_i0.h5\"\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", - " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
pathai
21am0.75_i0.h5-0.750a0.75_i0.h50.750.0da39a3ee5e6b4b0d3255bfef95601890afd80709
22am0.75_i30.h5-0.7530.0da39a3ee5e6b4b0d3255bfef95601890afd807091a0.75_i120.h50.75120.0
23am0.75_i60.h5-0.7560.0da39a3ee5e6b4b0d3255bfef95601890afd807092a0.75_i150.h50.75150.0
24am0.94_i+30.h5-0.9430.03a0.75_i180.h50.75180.0
4a0.75_i30.h50.7530.0
5a0.75_i60.h50.7560.0
6a0.75_i90.h50.7590.0
7a0.94_i0.h50.940.0
8a0.94_i120.h50.94120.0
9a0.94_i150.h50.94150.0
10a0.94_i180.h50.94180.0
11a0.94_i30.h50.9430.0
12a0.94_i60.h50.9460.0
13a0.94_i90.h50.9490.0
14a0_i0.h50.000.0
15a0_i120.h50.00120.0
16a0_i150.h50.00150.0
17a0_i180.h50.00180.0
18a0_i30.h50.0030.0
19a0_i60.h50.0060.0
20a0_i90.h50.0090.0
\n", + "
" + ], + "text/plain": [ + " path a i\n", + "0 a0.75_i0.h5 0.75 0.0\n", + "1 a0.75_i120.h5 0.75 120.0\n", + "2 a0.75_i150.h5 0.75 150.0\n", + "3 a0.75_i180.h5 0.75 180.0\n", + "4 a0.75_i30.h5 0.75 30.0\n", + "5 a0.75_i60.h5 0.75 60.0\n", + "6 a0.75_i90.h5 0.75 90.0\n", + "7 a0.94_i0.h5 0.94 0.0\n", + "8 a0.94_i120.h5 0.94 120.0\n", + "9 a0.94_i150.h5 0.94 150.0\n", + "10 a0.94_i180.h5 0.94 180.0\n", + "11 a0.94_i30.h5 0.94 30.0\n", + "12 a0.94_i60.h5 0.94 60.0\n", + "13 a0.94_i90.h5 0.94 90.0\n", + "14 a0_i0.h5 0.00 0.0\n", + "15 a0_i120.h5 0.00 120.0\n", + "16 a0_i150.h5 0.00 150.0\n", + "17 a0_i180.h5 0.00 180.0\n", + "18 a0_i30.h5 0.00 30.0\n", + "19 a0_i60.h5 0.00 60.0\n", + "20 a0_i90.h5 0.00 90.0" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ParaFrame.parse(\"a{a}_i{i}.h5\", base_path=\"repo1\", debug=True)" + ] + }, + { + "cell_type": "markdown", + "id": "md-s3-filter", + "metadata": {}, + "source": [ + "### Filter by parameter value\n", + "\n", + "Call a `ParaFrame` like a function to filter rows:\n", + "- **scalar** — keep rows where the column equals that value\n", + "- **list** — keep rows where the column matches any value in the list\n", + "- **chaining** — each call narrows the result further (AND logic)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "code-filter-scalar", + "metadata": { + "execution": { + "iopub.execute_input": "2026-04-02T23:09:20.980423Z", + "iopub.status.busy": "2026-04-02T23:09:20.980321Z", + "iopub.status.idle": "2026-04-02T23:09:20.995478Z", + "shell.execute_reply": "2026-04-02T23:09:20.995058Z" + } + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
pathai
14a0_i0.h50.00.0
15a0_i120.h50.0120.0
16a0_i150.h50.0150.0
17a0_i180.h50.0180.0
18a0_i30.h50.030.0
19a0_i60.h50.060.0
20a0_i90.h50.090.0
\n", + "
" + ], + "text/plain": [ + " path a i\n", + "14 a0_i0.h5 0.0 0.0\n", + "15 a0_i120.h5 0.0 120.0\n", + "16 a0_i150.h5 0.0 150.0\n", + "17 a0_i180.h5 0.0 180.0\n", + "18 a0_i30.h5 0.0 30.0\n", + "19 a0_i60.h5 0.0 60.0\n", + "20 a0_i90.h5 0.0 90.0" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# keep only rows where a == 0\n", + "pf(a=0)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "code-filter-list", + "metadata": { + "execution": { + "iopub.execute_input": "2026-04-02T23:09:20.996721Z", + "iopub.status.busy": "2026-04-02T23:09:20.996641Z", + "iopub.status.idle": "2026-04-02T23:09:21.009061Z", + "shell.execute_reply": "2026-04-02T23:09:21.008623Z" + } + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
pathai
0a0.75_i0.h50.750.0
1a0.75_i120.h50.75120.0
2a0.75_i150.h50.75150.0
3a0.75_i180.h50.75180.0
4a0.75_i30.h50.7530.0
5a0.75_i60.h50.7560.0
6a0.75_i90.h50.7590.0
14a0_i0.h50.000.0
15a0_i120.h50.00120.0
16a0_i150.h50.00150.0
17a0_i180.h50.00180.0
18a0_i30.h50.0030.0
19a0_i60.h50.0060.0
20a0_i90.h50.0090.0
\n", + "
" + ], + "text/plain": [ + " path a i\n", + "0 a0.75_i0.h5 0.75 0.0\n", + "1 a0.75_i120.h5 0.75 120.0\n", + "2 a0.75_i150.h5 0.75 150.0\n", + "3 a0.75_i180.h5 0.75 180.0\n", + "4 a0.75_i30.h5 0.75 30.0\n", + "5 a0.75_i60.h5 0.75 60.0\n", + "6 a0.75_i90.h5 0.75 90.0\n", + "14 a0_i0.h5 0.00 0.0\n", + "15 a0_i120.h5 0.00 120.0\n", + "16 a0_i150.h5 0.00 150.0\n", + "17 a0_i180.h5 0.00 180.0\n", + "18 a0_i30.h5 0.00 30.0\n", + "19 a0_i60.h5 0.00 60.0\n", + "20 a0_i90.h5 0.00 90.0" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# keep rows where a == 0 or a == 0.75\n", + "pf(a=[0, 0.75])" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "code-filter-chain", + "metadata": { + "execution": { + "iopub.execute_input": "2026-04-02T23:09:21.010493Z", + "iopub.status.busy": "2026-04-02T23:09:21.010403Z", + "iopub.status.idle": "2026-04-02T23:09:21.022395Z", + "shell.execute_reply": "2026-04-02T23:09:21.021918Z" + } + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
pathai
7a0.94_i0.h50.940.0
\n", + "
" + ], + "text/plain": [ + " path a i\n", + "7 a0.94_i0.h5 0.94 0.0" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# keep rows where a == 0.94 AND i == 0 (chain two calls)\n", + "pf(a=0.94)(i=0)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "code-filter-mask", + "metadata": { + "execution": { + "iopub.execute_input": "2026-04-02T23:09:21.023802Z", + "iopub.status.busy": "2026-04-02T23:09:21.023704Z", + "iopub.status.idle": "2026-04-02T23:09:21.035134Z", + "shell.execute_reply": "2026-04-02T23:09:21.034660Z" + } + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
pathai
0a0.75_i0.h50.750.0
4a0.75_i30.h50.7530.0
5a0.75_i60.h50.7560.0
7a0.94_i0.h50.940.0
11a0.94_i30.h50.9430.0
12a0.94_i60.h50.9460.0
14a0_i0.h50.000.0
18a0_i30.h50.0030.0
19a0_i60.h50.0060.0
\n", + "
" + ], + "text/plain": [ + " path a i\n", + "0 a0.75_i0.h5 0.75 0.0\n", + "4 a0.75_i30.h5 0.75 30.0\n", + "5 a0.75_i60.h5 0.75 60.0\n", + "7 a0.94_i0.h5 0.94 0.0\n", + "11 a0.94_i30.h5 0.94 30.0\n", + "12 a0.94_i60.h5 0.94 60.0\n", + "14 a0_i0.h5 0.00 0.0\n", + "18 a0_i30.h5 0.00 30.0\n", + "19 a0_i60.h5 0.00 60.0" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# use standard pandas boolean indexing for range or custom conditions\n", + "pf[pf.i <= 60]" + ] + }, + { + "cell_type": "markdown", + "id": "md-s3-loop", + "metadata": {}, + "source": [ + "### Use filtered paths in a loop\n", + "\n", + "Note: passing multiple keyword arguments in a single `pf(...)` call gives\n", + "OR logic across all of them. Chain calls to get AND logic." + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "code-loop", + "metadata": { + "execution": { + "iopub.execute_input": "2026-04-02T23:09:21.036445Z", + "iopub.status.busy": "2026-04-02T23:09:21.036350Z", + "iopub.status.idle": "2026-04-02T23:09:21.046246Z", + "shell.execute_reply": "2026-04-02T23:09:21.045705Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Processing \"a0.75_i0.h5\"...\n", + "Processing \"a0.94_i0.h5\"...\n", + "Processing \"a0_i0.h5\"...\n", + "Processing \"a0_i120.h5\"...\n", + "Processing \"a0_i150.h5\"...\n", + "Processing \"a0_i180.h5\"...\n", + "Processing \"a0_i30.h5\"...\n", + "Processing \"a0_i60.h5\"...\n", + "Processing \"a0_i90.h5\"...\n" + ] + } + ], + "source": [ + "# selects rows where a == 0 OR i == 0 (not both at once)\n", + "for path in pf(a=0, i=0).path:\n", + " print(f'Processing \"{path}\"...')" + ] + }, + { + "cell_type": "markdown", + "id": "md-s4", + "metadata": {}, + "source": [ + "## 4. Custom Filename Encoding\n", + "\n", + "Some datasets use a special convention to write certain values in file names.\n", + "For example, a negative spin value like `-0.94` might be stored as `m0.94`\n", + "in the filename, where the `m` prefix means \"minus\".\n", + "\n", + "Hallmark handles this with an encoding config: you write a regular expression\n", + "that describes the pattern, and `hallmark` applies it before reading the\n", + "parameter values. You store this config in `.hm/config.yml`." + ] + }, + { + "cell_type": "markdown", + "id": "md-s4-create", + "metadata": {}, + "source": [ + "### Create files and add the encoding config" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "code-touch-spin", + "metadata": { + "execution": { + "iopub.execute_input": "2026-04-02T23:09:21.047893Z", + "iopub.status.busy": "2026-04-02T23:09:21.047767Z", + "iopub.status.idle": "2026-04-02T23:09:21.072502Z", + "shell.execute_reply": "2026-04-02T23:09:21.072124Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallma" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "rk/demo/repo1 ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/github/hallmark/demo\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallma" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "rk/demo\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "repo1/am0.75_i0.h5\n", + "repo1/am0.75_i30.h5\n", + "repo1/am0.75_i60.h5\n", + "repo1/am0.94_i0.h5\n", + "repo1/am0.94_i30.h5\n", + "re" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "po1/am0.94_i60.h5\n" + ] + } + ], + "source": [ + "%%bash\n", + "# create files with negative spin values using the m-prefix convention\n", + "pushd repo1\n", + "touch am0.75_i0.h5 am0.75_i30.h5 am0.75_i60.h5 \\\n", + " am0.94_i0.h5 am0.94_i30.h5 am0.94_i60.h5\n", + "popd\n", + "ls repo1/am*.h5" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "code-write-config", + "metadata": { + "execution": { + "iopub.execute_input": "2026-04-02T23:09:21.074248Z", + "iopub.status.busy": "2026-04-02T23:09:21.074158Z", + "iopub.status.idle": "2026-04-02T23:09:21.099065Z", + "shell.execute_reply": "2026-04-02T23:09:21.098487Z" + } + }, + "outputs": [], + "source": [ + "%%bash\n", + "# write the encoding config to .hm/config.yml\n", + "# the regex m([0-9]+(\\.[0-9]+)?) matches the m-prefix and captures the number\n", + "cat > repo1/.hm/config.yml <<'EOF'\n", + "encodings:\n", + " - fmt: \"a{aspin}_i{i}.h5\"\n", + " encoding:\n", + " aspin: \"m([0-9]+(\\\\.[0-9]+)?)\"\n", + "EOF" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "code-reload-config", + "metadata": { + "execution": { + "iopub.execute_input": "2026-04-02T23:09:21.100614Z", + "iopub.status.busy": "2026-04-02T23:09:21.100508Z", + "iopub.status.idle": "2026-04-02T23:09:21.111890Z", + "shell.execute_reply": "2026-04-02T23:09:21.111492Z" + } + }, + "outputs": [], + "source": [ + "# reload the config so the repo object knows about the new encoding\n", + "repo1.state.config = repo1.dothm.load_yml(\"config\")" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "code-add-encoding", + "metadata": { + "execution": { + "iopub.execute_input": "2026-04-02T23:09:21.113669Z", + "iopub.status.busy": "2026-04-02T23:09:21.113569Z", + "iopub.status.idle": "2026-04-02T23:09:21.138248Z", + "shell.execute_reply": "2026-04-02T23:09:21.137828Z" + } + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", + " \n", + "
pathaspinisha1
0a0.75_i0.h50.750.0da39a3ee5e6b4b0d3255bfef95601890afd80709
1a0.75_i120.h50.75120.0da39a3ee5e6b4b0d3255bfef95601890afd80709
2a0.75_i150.h50.75150.0da39a3ee5e6b4b0d3255bfef95601890afd80709
3a0.75_i180.h50.75180.0da39a3ee5e6b4b0d3255bfef95601890afd80709
4a0.75_i30.h50.7530.0da39a3ee5e6b4b0d3255bfef95601890afd80709
5a0.75_i60.h50.7560.0da39a3ee5e6b4b0d3255bfef95601890afd80709
6a0.75_i90.h50.7590.0da39a3ee5e6b4b0d3255bfef95601890afd80709
7a0.94_i0.h50.940.0da39a3ee5e6b4b0d3255bfef95601890afd80709
8a0.94_i120.h50.94120.0da39a3ee5e6b4b0d3255bfef95601890afd80709
9a0.94_i150.h50.94150.0da39a3ee5e6b4b0d3255bfef95601890afd80709
10a0.94_i180.h50.94180.0da39a3ee5e6b4b0d3255bfef95601890afd80709
11a0.94_i30.h50.9430.0da39a3ee5e6b4b0d3255bfef95601890afd80709
12a0.94_i60.h50.9460.0da39a3ee5e6b4b0d3255bfef95601890afd80709
13a0.94_i90.h50.9490.0da39a3ee5e6b4b0d3255bfef95601890afd80709
14a0_i0.h50.000.0da39a3ee5e6b4b0d3255bfef95601890afd80709
15a0_i120.h50.00120.0da39a3ee5e6b4b0d3255bfef95601890afd80709
16a0_i150.h50.00150.0da39a3ee5e6b4b0d3255bfef95601890afd80709
17a0_i180.h50.00180.0da39a3ee5e6b4b0d3255bfef95601890afd80709
18a0_i30.h50.0030.0da39a3ee5e6b4b0d3255bfef95601890afd80709
19a0_i60.h50.0060.0da39a3ee5e6b4b0d3255bfef95601890afd80709
20a0_i90.h50.0090.0da39a3ee5e6b4b0d3255bfef95601890afd80709
21am0.75_i0.h5-0.750.0da39a3ee5e6b4b0d3255bfef95601890afd80709
22am0.75_i30.h5-0.7530.0da39a3ee5e6b4b0d3255bfef95601890afd80709
23am0.75_i60.h5-0.7560.0da39a3ee5e6b4b0d3255bfef95601890afd80709
24am0.94_i0.h5-0.940.0da39a3ee5e6b4b0d3255bfef95601890afd80709
25am0.94_i30.h5-0.9430.0da39a3ee5e6b4b0d3255bfef95601890afd80709
26am0.94_i60.h5-0.9460.0da39a3ee5e6b4b0d3255bfef95601890afd80709
\n", + "
" + ], + "text/plain": [ + " path aspin i sha1\n", + "0 a0.75_i0.h5 0.75 0.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", + "1 a0.75_i120.h5 0.75 120.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", + "2 a0.75_i150.h5 0.75 150.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", + "3 a0.75_i180.h5 0.75 180.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", + "4 a0.75_i30.h5 0.75 30.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", + "5 a0.75_i60.h5 0.75 60.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", + "6 a0.75_i90.h5 0.75 90.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", + "7 a0.94_i0.h5 0.94 0.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", + "8 a0.94_i120.h5 0.94 120.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", + "9 a0.94_i150.h5 0.94 150.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", + "10 a0.94_i180.h5 0.94 180.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", + "11 a0.94_i30.h5 0.94 30.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", + "12 a0.94_i60.h5 0.94 60.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", + "13 a0.94_i90.h5 0.94 90.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", + "14 a0_i0.h5 0.00 0.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", + "15 a0_i120.h5 0.00 120.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", + "16 a0_i150.h5 0.00 150.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", + "17 a0_i180.h5 0.00 180.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", + "18 a0_i30.h5 0.00 30.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", + "19 a0_i60.h5 0.00 60.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", + "20 a0_i90.h5 0.00 90.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", + "21 am0.75_i0.h5 -0.75 0.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", + "22 am0.75_i30.h5 -0.75 30.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", + "23 am0.75_i60.h5 -0.75 60.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", + "24 am0.94_i0.h5 -0.94 0.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", + "25 am0.94_i30.h5 -0.94 30.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", + "26 am0.94_i60.h5 -0.94 60.0 da39a3ee5e6b4b0d3255bfef95601890afd80709" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# add all files — m0.94 in the filename is decoded to -0.94 in the table\n", + "pf_spin = repo1.add(\"a{aspin}_i{i}.h5\", encoding=True)\n", + "pf_spin" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "code-commit-spin", + "metadata": { + "execution": { + "iopub.execute_input": "2026-04-02T23:09:21.139430Z", + "iopub.status.busy": "2026-04-02T23:09:21.139347Z", + "iopub.status.idle": "2026-04-02T23:09:21.174031Z", + "shell.execute_reply": "2026-04-02T23:09:21.173468Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "repo1.commit(\"add negative spin files with encoding\")" + ] + }, + { + "cell_type": "markdown", + "id": "md-s4-pf", + "metadata": {}, + "source": [ + "### Discover encoded files with ParaFrame\n", + "\n", + "You can also use `ParaFrame.parse()` directly with encoding.\n", + "Pass the encoding config from `repo1.state.config` so it knows the regex." + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "code-pf-encoding", + "metadata": { + "execution": { + "iopub.execute_input": "2026-04-02T23:09:21.175560Z", + "iopub.status.busy": "2026-04-02T23:09:21.175454Z", + "iopub.status.idle": "2026-04-02T23:09:21.191463Z", + "shell.execute_reply": "2026-04-02T23:09:21.190918Z" + } + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
pathaspini
250a0.75_i0.h50.750.0
1a0.75_i120.h50.75120.0
2a0.75_i150.h50.75150.0
3a0.75_i180.h50.75180.0
4a0.75_i30.h50.7530.0
5a0.75_i60.h50.7560.0
6a0.75_i90.h50.7590.0
7a0.94_i0.h50.940.0
8a0.94_i120.h50.94120.0
9a0.94_i150.h50.94150.0
10a0.94_i180.h50.94180.0
11a0.94_i30.h50.9430.0
12a0.94_i60.h50.9460.0
13a0.94_i90.h50.9490.0
14a0_i0.h50.000.0
15a0_i120.h50.00120.0
16a0_i150.h50.00150.0
17a0_i180.h50.00180.0
18a0_i30.h50.0030.0
19a0_i60.h50.0060.0
20a0_i90.h50.0090.0
21am0.75_i0.h5-0.750.0
22am0.75_i30.h5-0.7530.0
23am0.75_i60.h5-0.7560.0
24am0.94_i0.h5-0.940.0da39a3ee5e6b4b0d3255bfef95601890afd80709
25am0.94_i30.h5-0.9430.0
26am0.94_i60.h5-0.9460.0
\n", + "
" + ], + "text/plain": [ + " path aspin i\n", + "0 a0.75_i0.h5 0.75 0.0\n", + "1 a0.75_i120.h5 0.75 120.0\n", + "2 a0.75_i150.h5 0.75 150.0\n", + "3 a0.75_i180.h5 0.75 180.0\n", + "4 a0.75_i30.h5 0.75 30.0\n", + "5 a0.75_i60.h5 0.75 60.0\n", + "6 a0.75_i90.h5 0.75 90.0\n", + "7 a0.94_i0.h5 0.94 0.0\n", + "8 a0.94_i120.h5 0.94 120.0\n", + "9 a0.94_i150.h5 0.94 150.0\n", + "10 a0.94_i180.h5 0.94 180.0\n", + "11 a0.94_i30.h5 0.94 30.0\n", + "12 a0.94_i60.h5 0.94 60.0\n", + "13 a0.94_i90.h5 0.94 90.0\n", + "14 a0_i0.h5 0.00 0.0\n", + "15 a0_i120.h5 0.00 120.0\n", + "16 a0_i150.h5 0.00 150.0\n", + "17 a0_i180.h5 0.00 180.0\n", + "18 a0_i30.h5 0.00 30.0\n", + "19 a0_i60.h5 0.00 60.0\n", + "20 a0_i90.h5 0.00 90.0\n", + "21 am0.75_i0.h5 -0.75 0.0\n", + "22 am0.75_i30.h5 -0.75 30.0\n", + "23 am0.75_i60.h5 -0.75 60.0\n", + "24 am0.94_i0.h5 -0.94 0.0\n", + "25 am0.94_i30.h5 -0.94 30.0\n", + "26 am0.94_i60.h5 -0.94 60.0" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "encodings = repo1.state.config.get(\"encodings\", [])\n", + "\n", + "pf_spin = ParaFrame.parse(\n", + " \"a{aspin}_i{i}.h5\",\n", + " encodings=encodings,\n", + " base_path=\"repo1\",\n", + " encoding=True,\n", + ")\n", + "pf_spin" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "code-filter-spin", + "metadata": { + "execution": { + "iopub.execute_input": "2026-04-02T23:09:21.192942Z", + "iopub.status.busy": "2026-04-02T23:09:21.192838Z", + "iopub.status.idle": "2026-04-02T23:09:21.204692Z", + "shell.execute_reply": "2026-04-02T23:09:21.204364Z" + } + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", - " \n", " \n", " \n", - " \n", + " \n", " \n", " \n", " \n", - " \n", " \n", " \n", "
pathaspini
21am0.75_i0.h5-0.750.0
22am0.75_i30.h5-0.7530.0
23am0.75_i60.h5-0.7560.0
24am0.94_i0.h5-0.940.0
25am0.94_i30.h5-0.9430.0da39a3ee5e6b4b0d3255bfef95601890afd80709
2726am0.94_i60.h5-0.9460.0da39a3ee5e6b4b0d3255bfef95601890afd80709
\n", "
" ], "text/plain": [ - " path aspin i sha1\n", - "0 a0.75_i0.h5 0.75 0.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "1 a0.75_i120.h5 0.75 120.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "2 a0.75_i150.h5 0.75 150.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "3 a0.75_i180.h5 0.75 180.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "4 a0.75_i30.h5 0.75 30.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "5 a0.75_i60.h5 0.75 60.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "6 a0.75_i90.h5 0.75 90.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "7 a0.94_i0.h5 0.94 0.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "8 a0.94_i120.h5 0.94 120.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "9 a0.94_i150.h5 0.94 150.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "10 a0.94_i180.h5 0.94 180.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "11 a0.94_i30.h5 0.94 30.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "12 a0.94_i60.h5 0.94 60.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "13 a0.94_i90.h5 0.94 90.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "14 a0_i0.h5 0.00 0.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "15 a0_i120.h5 0.00 120.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "16 a0_i150.h5 0.00 150.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "17 a0_i180.h5 0.00 180.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "18 a0_i30.h5 0.00 30.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "19 a0_i60.h5 0.00 60.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "20 a0_i90.h5 0.00 90.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "21 am0.75_i0.h5 -0.75 0.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "22 am0.75_i30.h5 -0.75 30.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "23 am0.75_i60.h5 -0.75 60.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "24 am0.94_i+30.h5 -0.94 30.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "25 am0.94_i0.h5 -0.94 0.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "26 am0.94_i30.h5 -0.94 30.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "27 am0.94_i60.h5 -0.94 60.0 da39a3ee5e6b4b0d3255bfef95601890afd80709" + " path aspin i\n", + "21 am0.75_i0.h5 -0.75 0.0\n", + "22 am0.75_i30.h5 -0.75 30.0\n", + "23 am0.75_i60.h5 -0.75 60.0\n", + "24 am0.94_i0.h5 -0.94 0.0\n", + "25 am0.94_i30.h5 -0.94 30.0\n", + "26 am0.94_i60.h5 -0.94 60.0" ] }, - "execution_count": 89, + "execution_count": 25, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "repo1.add(\"a{aspin}_i{i}.h5\", encoding=True)" + "# filter using the decoded float values — not the raw m-prefix strings\n", + "pf_spin(aspin=[-0.75, -0.94])" ] } ], "metadata": { "kernelspec": { - "display_name": ".venv313 (3.13.7)", + "display_name": "hallmark", "language": "python", "name": "python3" }, @@ -849,7 +2400,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.13.7" + "version": "3.12.12" } }, "nbformat": 4, From 0fe58af950dd0ad708de09b79e164a56c6d4cc31 Mon Sep 17 00:00:00 2001 From: Rohin Sant Date: Fri, 3 Apr 2026 11:10:48 -0700 Subject: [PATCH 02/20] A prototype for the hallmark checkout functionality with the cli feature that includes copying over files from source branch to the target branch using paths in data.tsv. --- mod/hallmark/cli.py | 13 +++++++++++++ mod/hallmark/repo.py | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/mod/hallmark/cli.py b/mod/hallmark/cli.py index 4be5fc6..050953a 100644 --- a/mod/hallmark/cli.py +++ b/mod/hallmark/cli.py @@ -103,3 +103,16 @@ def commit(repo, message): click.echo("Committed staged state changes.") else: click.echo("No changes added to commit.") + +@hallmark.command(short_help="Checkout to a worktree branch.") +@click.argument("target_branch") +@click.pass_obj +def checkout(repo, target_branch): + """Checkout to another worktree branch and restore the state saved in data.tsv. + + This is analogous to `git checkout BRANCH`. + """ + if repo.checkout(target_branch): + click.echo(f'Checked out to "{target_branch}".') + else: + click.echo("No branches to checkout.") \ No newline at end of file diff --git a/mod/hallmark/repo.py b/mod/hallmark/repo.py index 7a36229..e6744e1 100644 --- a/mod/hallmark/repo.py +++ b/mod/hallmark/repo.py @@ -113,3 +113,42 @@ def commit(self, msg: str, allow_empty: bool = False) -> bool: return True else: return False + + def checkout(self, target_branch: str) -> bool: + from shutil import copy2 + from git.exc import GitCommandError + + if not isinstance(target_branch, str) or not target_branch.strip(): + raise ValueError("branch name must be a non-empty string") + + if self.worktree is None: + raise RuntimeError("cannot checkout a bare repository without a worktree") + + source = Path(self.worktree).resolve() + target = source.parent / target_branch + target_dothm = target / ".hm" + + linked_dothm = None + + if target_dothm.exists(): + linked_dothm = Dothm(target_dothm) + else: + target.mkdir(parents=True, exist_ok=True) + existing_branches = {head.name for head in self.dothm.heads} + try: + if target_branch in existing_branches: + linked_dothm = self.dothm.link(target_dothm, target_branch) + else: + self.dothm.git.worktree("add", "-b", target_branch, str(target_dothm)) + linked_dothm = Dothm(target_dothm) + except GitCommandError as e: + raise RuntimeError(f'failed to create worktree for branch "{target_branch}": {e}') + + target_state = linked_dothm.load() + for rel in target_state.data["path"]: + rel_path = Path(rel) + src = source / rel_path + dest = target / rel_path + dest.parent.mkdir(parents=True, exist_ok=True) + copy2(src, dest) + return True \ No newline at end of file From f1109337caaa63ab8a851ae692604c1b092ff0df Mon Sep 17 00:00:00 2001 From: HaydenMarchinek Date: Wed, 8 Apr 2026 23:18:50 -0700 Subject: [PATCH 03/20] implemented the cache functionality in hallmark. Added files in any worktree get cached and the cache is shared across all new worktrees. --- mod/hallmark/cache.py | 16 ++++++++++++++++ mod/hallmark/repo.py | 12 +++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 mod/hallmark/cache.py diff --git a/mod/hallmark/cache.py b/mod/hallmark/cache.py new file mode 100644 index 0000000..1fdd65b --- /dev/null +++ b/mod/hallmark/cache.py @@ -0,0 +1,16 @@ +import shutil +from pathlib import Path + +class Cache: + def __init__(self, path): + self.root = Path(path) / "cache" + + def _split_checksum(self, sha1: str) -> Path: + return self.root / sha1[:2] / sha1[2:] + + def store(self, src: Path, sha1: str) -> Path: + stored_checksum = self._split_checksum(sha1) + if not stored_checksum.exists(): + stored_checksum.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, stored_checksum) + return stored_checksum \ No newline at end of file diff --git a/mod/hallmark/repo.py b/mod/hallmark/repo.py index e6744e1..198ba5e 100644 --- a/mod/hallmark/repo.py +++ b/mod/hallmark/repo.py @@ -26,6 +26,7 @@ from .dothm import Dothm from .worktree import Worktree from .paraframe import ParaFrame +from .cache import Cache @contextmanager @@ -63,11 +64,19 @@ def __init__(self, path: Path | str) -> None: self.dothm = Dothm(dothm_path) self.worktree = worktree_path and Worktree(worktree_path) self.state = self.dothm.load() + + common = Path(self.dothm.common_dir).resolve().parent + self.cache = Cache(common) + dothm_cache = Path(dothm_path) / "cache" + main_cache = common / "cache" + if dothm_cache.resolve() != main_cache.resolve() and not dothm_cache.exists(): + dothm_cache.symlink_to(main_cache) @classmethod def init(cls, path: Path | str) -> "Repo": dothm_path, worktree_path = cls.lwpaths(path) Dothm.init(dothm_path).dump(State()) + Cache(dothm_path) # <-- this creates cache at dothm_path directly worktree_path and Worktree.init(worktree_path) return cls(path) @@ -101,7 +110,8 @@ def add(self, fstr: str, encoding: bool = False) -> ParaFrame: self.state.update(pf) self.dothm.dump(self.state) - + for _, row in pf.iterrows(): + self.cache.store(self.worktree / row["path"], row["sha1"]) return pf def commit(self, msg: str, allow_empty: bool = False) -> bool: From a88e37cd905af0a26e78456f4ef78edb04189824 Mon Sep 17 00:00:00 2001 From: HaydenMarchinek Date: Wed, 8 Apr 2026 23:38:45 -0700 Subject: [PATCH 04/20] Added demo cache notebook --- demo/demo_cache_1.ipynb | 381 ++++++++++++++++++++++++++++++++++++++++ mod/hallmark/repo.py | 2 +- 2 files changed, 382 insertions(+), 1 deletion(-) create mode 100644 demo/demo_cache_1.ipynb diff --git a/demo/demo_cache_1.ipynb b/demo/demo_cache_1.ipynb new file mode 100644 index 0000000..e3c2935 --- /dev/null +++ b/demo/demo_cache_1.ipynb @@ -0,0 +1,381 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Hallmark Cache Demo\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "59a52343", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The autoreload extension is already loaded. To reload it, use:\n", + " %reload_ext autoreload\n" + ] + } + ], + "source": [ + "# auto-reload modules when they change\n", + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "%%bash\n", + "rm -rf repo1\n", + "rm -rf experiment\n", + "mkdir repo1" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Initialize the hallmark repo" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "from hallmark import Repo\n", + "\n", + "repo1 = Repo.init(\"repo1\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Create sample data files on `main` with unique content" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "~/EHT/hallmark/demo/repo1 ~/EHT/hallmark/demo\n", + "~/EHT/hallmark/demo\n", + "a0.75_i0.h5\n", + "a0.75_i120.h5\n", + "a0.75_i150.h5\n", + "a0.75_i180.h5\n", + "a0.75_i30.h5\n", + "a0.75_i60.h5\n", + "a0.75_i90.h5\n", + "a0.94_i0.h5\n", + "a0.94_i120.h5\n", + "a0.94_i150.h5\n", + "a0.94_i180.h5\n", + "a0.94_i30.h5\n", + "a0.94_i60.h5\n", + "a0.94_i90.h5\n", + "a0_i0.h5\n", + "a0_i120.h5\n", + "a0_i150.h5\n", + "a0_i180.h5\n", + "a0_i30.h5\n", + "a0_i60.h5\n", + "a0_i90.h5\n" + ] + } + ], + "source": [ + "%%bash\n", + "pushd repo1\n", + "for f in a{0,0.75,0.94}_i{0,30,60,90,120,150,180}.h5; do\n", + " echo \"$f\" > \"$f\"\n", + "done\n", + "popd\n", + "ls repo1" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. Add files on `main` and commit" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " path spin inc sha1\n", + "0 a0.75_i0.h5 0.75 0.0 f656d419104e7af783c119186a4d81b15563310f\n", + "1 a0.75_i120.h5 0.75 120.0 ec4dae9f0b8f69ac42f0c290fc84e0d6f1bec6cd\n", + "2 a0.75_i150.h5 0.75 150.0 c15ca8e6bdad8707bfdf66d1b4f93b5e56a3c094\n", + "3 a0.75_i180.h5 0.75 180.0 dbc88b1f6975ee423b864dfe729cf4643da93459\n", + "4 a0.75_i30.h5 0.75 30.0 890b2ea509575be9bc74ad515731a2562cd7406e\n", + "5 a0.75_i60.h5 0.75 60.0 d9cfcdd0b44d81c10f88e126d5b1eea4914259a0\n", + "6 a0.75_i90.h5 0.75 90.0 ed2cfc6bb157a42729d3c6f45b228b432edec4be\n", + "7 a0.94_i0.h5 0.94 0.0 b5c93e804c6f8f471163afa8d5eb68f363342ad0\n", + "8 a0.94_i120.h5 0.94 120.0 b83845a38e80bbcaeb4ee9817ef2ff1221cf68f4\n", + "9 a0.94_i150.h5 0.94 150.0 5b6eb09381b58bd223da614886f6d2c5a51b9433\n", + "10 a0.94_i180.h5 0.94 180.0 01f3714023142e16a4724ef326068d086fd7c3d8\n", + "11 a0.94_i30.h5 0.94 30.0 00f6caf5b1a313bde78a907a5f773daefcb715c4\n", + "12 a0.94_i60.h5 0.94 60.0 1101bf0507f2e227abb023a2b541e0f260f3042c\n", + "13 a0.94_i90.h5 0.94 90.0 07fbb0434afd32bf404eb6b3d0c3c87243b40bef\n", + "14 a0_i0.h5 0.00 0.0 18682593a012503b71935017dbc044665254a23c\n", + "15 a0_i120.h5 0.00 120.0 1c057a4885fd04379ee1217375720665713c902f\n", + "16 a0_i150.h5 0.00 150.0 1ed0f877b0802b7d8e91bffa3292255c982e4e85\n", + "17 a0_i180.h5 0.00 180.0 0295ffa54defdc8b51cfc998f5e35aebcf01768d\n", + "18 a0_i30.h5 0.00 30.0 829b87d845b367a9cc87df6ff510d1bdb7444aca\n", + "19 a0_i60.h5 0.00 60.0 075a30e0a9c4dddd92447f463219d4c2a842883d\n", + "20 a0_i90.h5 0.00 90.0 de40f4983d86342b03809bfcc09958f43f359b89\n" + ] + } + ], + "source": [ + "repo = Repo.init(\"repo1\")\n", + "pf = repo.add(\"a{spin}_i{inc}.h5\")\n", + "print(pf)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4. Inspect the cache after `main` add" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "repo1/.hm/cache/00/f6caf5b1a313bde78a907a5f773daefcb715c4\n", + "repo1/.hm/cache/01/f3714023142e16a4724ef326068d086fd7c3d8\n", + "repo1/.hm/cache/02/95ffa54defdc8b51cfc998f5e35aebcf01768d\n", + "repo1/.hm/cache/07/5a30e0a9c4dddd92447f463219d4c2a842883d\n", + "repo1/.hm/cache/07/fbb0434afd32bf404eb6b3d0c3c87243b40bef\n", + "repo1/.hm/cache/11/01bf0507f2e227abb023a2b541e0f260f3042c\n", + "repo1/.hm/cache/18/682593a012503b71935017dbc044665254a23c\n", + "repo1/.hm/cache/1c/057a4885fd04379ee1217375720665713c902f\n", + "repo1/.hm/cache/1e/d0f877b0802b7d8e91bffa3292255c982e4e85\n", + "repo1/.hm/cache/5b/6eb09381b58bd223da614886f6d2c5a51b9433\n", + "repo1/.hm/cache/82/9b87d845b367a9cc87df6ff510d1bdb7444aca\n", + "repo1/.hm/cache/89/0b2ea509575be9bc74ad515731a2562cd7406e\n", + "repo1/.hm/cache/b5/c93e804c6f8f471163afa8d5eb68f363342ad0\n", + "repo1/.hm/cache/b8/3845a38e80bbcaeb4ee9817ef2ff1221cf68f4\n", + "repo1/.hm/cache/c1/5ca8e6bdad8707bfdf66d1b4f93b5e56a3c094\n", + "repo1/.hm/cache/d9/cfcdd0b44d81c10f88e126d5b1eea4914259a0\n", + "repo1/.hm/cache/db/c88b1f6975ee423b864dfe729cf4643da93459\n", + "repo1/.hm/cache/de/40f4983d86342b03809bfcc09958f43f359b89\n", + "repo1/.hm/cache/ec/4dae9f0b8f69ac42f0c290fc84e0d6f1bec6cd\n", + "repo1/.hm/cache/ed/2cfc6bb157a42729d3c6f45b228b432edec4be\n", + "repo1/.hm/cache/f6/56d419104e7af783c119186a4d81b15563310f\n" + ] + } + ], + "source": [ + "%%bash\n", + "find repo1/.hm/cache -type f | sort" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 5. Checkout a new branch" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "repo.checkout(\"experiment\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 6. Create different data files on `experiment`" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "~/EHT/hallmark/demo/experiment ~/EHT/hallmark/demo\n", + "~/EHT/hallmark/demo\n", + "b0.5_i0.h5\n", + "b0.5_i135.h5\n", + "b0.5_i45.h5\n", + "b0.5_i90.h5\n", + "b1.0_i0.h5\n", + "b1.0_i135.h5\n", + "b1.0_i45.h5\n", + "b1.0_i90.h5\n" + ] + } + ], + "source": [ + "%%bash\n", + "pushd experiment\n", + "for f in b{0.5,1.0}_i{0,45,90,135}.h5; do\n", + " echo \"$f\" > \"$f\"\n", + "done\n", + "popd\n", + "ls experiment" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 7. Add files on `experiment`" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " path spin inc sha1\n", + "0 b0.5_i0.h5 0.5 0.0 0473f32e1e78a36133ffb00d00e5ea0499abae64\n", + "1 b0.5_i135.h5 0.5 135.0 765c7b7756a5be9809e5818a6ce9dd02b5027649\n", + "2 b0.5_i45.h5 0.5 45.0 078949b464c37c1208b0884982cea6165e57f689\n", + "3 b0.5_i90.h5 0.5 90.0 87b4a3a1ef37d9f94b5e58ef78a5e61b0e74a4f8\n", + "4 b1.0_i0.h5 1.0 0.0 5dae25cfd3c075a136e88f1ed309a5d7cfbffbc3\n", + "5 b1.0_i135.h5 1.0 135.0 abca08587d969c4549429c8cb062f7541e4261f2\n", + "6 b1.0_i45.h5 1.0 45.0 cddf02809edd7c3651bc58c516e142339f2a3080\n", + "7 b1.0_i90.h5 1.0 90.0 26d32c645c218e21519d279528ba75a657bdc4f9\n" + ] + } + ], + "source": [ + "repo_exp = Repo(\"experiment\")\n", + "pf_exp = repo_exp.add(\"b{spin}_i{inc}.h5\")\n", + "print(pf_exp)" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "b8f057f3", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "repo1/.hm/cache/00/f6caf5b1a313bde78a907a5f773daefcb715c4\n", + "repo1/.hm/cache/01/f3714023142e16a4724ef326068d086fd7c3d8\n", + "repo1/.hm/cache/02/95ffa54defdc8b51cfc998f5e35aebcf01768d\n", + "repo1/.hm/cache/04/73f32e1e78a36133ffb00d00e5ea0499abae64\n", + "repo1/.hm/cache/07/5a30e0a9c4dddd92447f463219d4c2a842883d\n", + "repo1/.hm/cache/07/8949b464c37c1208b0884982cea6165e57f689\n", + "repo1/.hm/cache/07/fbb0434afd32bf404eb6b3d0c3c87243b40bef\n", + "repo1/.hm/cache/11/01bf0507f2e227abb023a2b541e0f260f3042c\n", + "repo1/.hm/cache/18/682593a012503b71935017dbc044665254a23c\n", + "repo1/.hm/cache/1c/057a4885fd04379ee1217375720665713c902f\n", + "repo1/.hm/cache/1e/d0f877b0802b7d8e91bffa3292255c982e4e85\n", + "repo1/.hm/cache/26/d32c645c218e21519d279528ba75a657bdc4f9\n", + "repo1/.hm/cache/5b/6eb09381b58bd223da614886f6d2c5a51b9433\n", + "repo1/.hm/cache/5d/ae25cfd3c075a136e88f1ed309a5d7cfbffbc3\n", + "repo1/.hm/cache/76/5c7b7756a5be9809e5818a6ce9dd02b5027649\n", + "repo1/.hm/cache/82/9b87d845b367a9cc87df6ff510d1bdb7444aca\n", + "repo1/.hm/cache/87/b4a3a1ef37d9f94b5e58ef78a5e61b0e74a4f8\n", + "repo1/.hm/cache/89/0b2ea509575be9bc74ad515731a2562cd7406e\n", + "repo1/.hm/cache/ab/ca08587d969c4549429c8cb062f7541e4261f2\n", + "repo1/.hm/cache/b5/c93e804c6f8f471163afa8d5eb68f363342ad0\n", + "repo1/.hm/cache/b8/3845a38e80bbcaeb4ee9817ef2ff1221cf68f4\n", + "repo1/.hm/cache/c1/5ca8e6bdad8707bfdf66d1b4f93b5e56a3c094\n", + "repo1/.hm/cache/cd/df02809edd7c3651bc58c516e142339f2a3080\n", + "repo1/.hm/cache/d9/cfcdd0b44d81c10f88e126d5b1eea4914259a0\n", + "repo1/.hm/cache/db/c88b1f6975ee423b864dfe729cf4643da93459\n", + "repo1/.hm/cache/de/40f4983d86342b03809bfcc09958f43f359b89\n", + "repo1/.hm/cache/ec/4dae9f0b8f69ac42f0c290fc84e0d6f1bec6cd\n", + "repo1/.hm/cache/ed/2cfc6bb157a42729d3c6f45b228b432edec4be\n", + "repo1/.hm/cache/f6/56d419104e7af783c119186a4d81b15563310f\n" + ] + } + ], + "source": [ + "%%bash\n", + "find repo1/.hm/cache -type f | sort" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".hallmark_env (3.13.7)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.7" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/mod/hallmark/repo.py b/mod/hallmark/repo.py index 198ba5e..778f602 100644 --- a/mod/hallmark/repo.py +++ b/mod/hallmark/repo.py @@ -76,7 +76,7 @@ def __init__(self, path: Path | str) -> None: def init(cls, path: Path | str) -> "Repo": dothm_path, worktree_path = cls.lwpaths(path) Dothm.init(dothm_path).dump(State()) - Cache(dothm_path) # <-- this creates cache at dothm_path directly + Cache(dothm_path) worktree_path and Worktree.init(worktree_path) return cls(path) From 3c0f59765b33ca624c3bf07c89cbe09caa2af60c Mon Sep 17 00:00:00 2001 From: Nayera Abdessalam Date: Fri, 10 Apr 2026 09:53:31 -0500 Subject: [PATCH 05/20] Implementing basic clone functionality for remote hallmark repo --- mod/hallmark/cli.py | 17 ++++++++++++++++- mod/hallmark/dothm.py | 21 ++++++++++++++++++++- mod/hallmark/repo.py | 14 +++++++++++++- 3 files changed, 49 insertions(+), 3 deletions(-) diff --git a/mod/hallmark/cli.py b/mod/hallmark/cli.py index 050953a..03b9890 100644 --- a/mod/hallmark/cli.py +++ b/mod/hallmark/cli.py @@ -20,6 +20,7 @@ from click import ClickException from git.exc import GitError +from .error import DothmError from . import Repo # from "__init__.py" @@ -115,4 +116,18 @@ def checkout(repo, target_branch): if repo.checkout(target_branch): click.echo(f'Checked out to "{target_branch}".') else: - click.echo("No branches to checkout.") \ No newline at end of file + click.echo("No branches to checkout.") + +@hallmark.command(short_help="Clone a hallmark repository from a remote URL.") +@click.argument("url") +@click.argument("path") +def clone(url, path): + """Clone a hallmark repository from URL to PATH. + + This is analogous to `git clone URL PATH`. + """ + try: + Repo.clone(url, path) + click.echo(f'Successfully cloned to "{path}"') + except (DothmError, GitError) as e: + raise ClickException(f'Clone failed: {e}') \ No newline at end of file diff --git a/mod/hallmark/dothm.py b/mod/hallmark/dothm.py index 996aa1e..131a2e9 100644 --- a/mod/hallmark/dothm.py +++ b/mod/hallmark/dothm.py @@ -18,7 +18,7 @@ from pathlib import Path from functools import cached_property -from git import Repo +from git import GitCommandError, Repo import pandas as pd import yaml @@ -61,6 +61,25 @@ def init(cls, *args, **kwargs) -> "Dothm": dothm.index.commit("Initial commit: local `.hm` repository") return dothm + @classmethod + def clone(cls, url: str, to_path: Path | str) -> "Dothm": + to_path = Path(to_path) + + try: + git_repo = super().clone_from(url, str(to_path)) + dothm = cls(str(to_path)) + + # Validate required files + required_files = ["config.yml", "meta.yml", "data.tsv"] + for file in required_files: + if not (dothm.path / file).exists(): + raise DothmError(f'Cloned repository missing required file: {file}') + return dothm + except GitCommandError as e: + raise DothmError(f'Failed to clone from "{url}": {e}') + except DothmError: + raise + def link(self, path: Path | str, branch: str | None = None): from git.exc import GitCommandError diff --git a/mod/hallmark/repo.py b/mod/hallmark/repo.py index 778f602..6c05b59 100644 --- a/mod/hallmark/repo.py +++ b/mod/hallmark/repo.py @@ -79,7 +79,19 @@ def init(cls, path: Path | str) -> "Repo": Cache(dothm_path) worktree_path and Worktree.init(worktree_path) return cls(path) - + + @classmethod + def clone(cls, url: str, path: Path | str) -> "Repo": + dothm_path, worktree_path = cls.lwpaths(path) + + Dothm.clone(url, dothm_path) + + # Initialize worktree if non-bare + if worktree_path: + Worktree.init(worktree_path) + + return cls(path) + @staticmethod def checksum(path: Path, chunk_size: int = 1024 * 1024) -> str: digest = sha1() From 22e8c477adc9f9cc995fd7f9aa669ad8e03aa739 Mon Sep 17 00:00:00 2001 From: Nayera Abdessalam Date: Fri, 10 Apr 2026 10:11:49 -0500 Subject: [PATCH 06/20] do not search for existing repo when cloning --- mod/hallmark/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mod/hallmark/cli.py b/mod/hallmark/cli.py index 03b9890..bd80e8a 100644 --- a/mod/hallmark/cli.py +++ b/mod/hallmark/cli.py @@ -34,7 +34,7 @@ def hallmark(ctx): Hallmark is a lightweight package designed to version control and manage data products in a complex workflow. """ - if ctx.invoked_subcommand in [None, "init"]: + if ctx.invoked_subcommand in [None, "init", "clone"]: return # do nothing try: From 1bfbfaef62baa8e83c8956013bb7dc5c8dff2726 Mon Sep 17 00:00:00 2001 From: Ram Adithya Date: Tue, 14 Apr 2026 16:09:17 -0700 Subject: [PATCH 07/20] Updated cache to be named as index --- demo/demo_cache_1.ipynb | 381 ---------------------------- demo/demo_index.ipynb | 315 +++++++++++++++++++++++ mod/hallmark/{cache.py => index.py} | 4 +- mod/hallmark/repo.py | 16 +- 4 files changed, 325 insertions(+), 391 deletions(-) delete mode 100644 demo/demo_cache_1.ipynb create mode 100644 demo/demo_index.ipynb rename mod/hallmark/{cache.py => index.py} (89%) diff --git a/demo/demo_cache_1.ipynb b/demo/demo_cache_1.ipynb deleted file mode 100644 index e3c2935..0000000 --- a/demo/demo_cache_1.ipynb +++ /dev/null @@ -1,381 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Hallmark Cache Demo\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Setup" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "59a52343", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The autoreload extension is already loaded. To reload it, use:\n", - " %reload_ext autoreload\n" - ] - } - ], - "source": [ - "# auto-reload modules when they change\n", - "%load_ext autoreload\n", - "%autoreload 2" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [], - "source": [ - "%%bash\n", - "rm -rf repo1\n", - "rm -rf experiment\n", - "mkdir repo1" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 1. Initialize the hallmark repo" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [], - "source": [ - "from hallmark import Repo\n", - "\n", - "repo1 = Repo.init(\"repo1\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 2. Create sample data files on `main` with unique content" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "~/EHT/hallmark/demo/repo1 ~/EHT/hallmark/demo\n", - "~/EHT/hallmark/demo\n", - "a0.75_i0.h5\n", - "a0.75_i120.h5\n", - "a0.75_i150.h5\n", - "a0.75_i180.h5\n", - "a0.75_i30.h5\n", - "a0.75_i60.h5\n", - "a0.75_i90.h5\n", - "a0.94_i0.h5\n", - "a0.94_i120.h5\n", - "a0.94_i150.h5\n", - "a0.94_i180.h5\n", - "a0.94_i30.h5\n", - "a0.94_i60.h5\n", - "a0.94_i90.h5\n", - "a0_i0.h5\n", - "a0_i120.h5\n", - "a0_i150.h5\n", - "a0_i180.h5\n", - "a0_i30.h5\n", - "a0_i60.h5\n", - "a0_i90.h5\n" - ] - } - ], - "source": [ - "%%bash\n", - "pushd repo1\n", - "for f in a{0,0.75,0.94}_i{0,30,60,90,120,150,180}.h5; do\n", - " echo \"$f\" > \"$f\"\n", - "done\n", - "popd\n", - "ls repo1" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 3. Add files on `main` and commit" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " path spin inc sha1\n", - "0 a0.75_i0.h5 0.75 0.0 f656d419104e7af783c119186a4d81b15563310f\n", - "1 a0.75_i120.h5 0.75 120.0 ec4dae9f0b8f69ac42f0c290fc84e0d6f1bec6cd\n", - "2 a0.75_i150.h5 0.75 150.0 c15ca8e6bdad8707bfdf66d1b4f93b5e56a3c094\n", - "3 a0.75_i180.h5 0.75 180.0 dbc88b1f6975ee423b864dfe729cf4643da93459\n", - "4 a0.75_i30.h5 0.75 30.0 890b2ea509575be9bc74ad515731a2562cd7406e\n", - "5 a0.75_i60.h5 0.75 60.0 d9cfcdd0b44d81c10f88e126d5b1eea4914259a0\n", - "6 a0.75_i90.h5 0.75 90.0 ed2cfc6bb157a42729d3c6f45b228b432edec4be\n", - "7 a0.94_i0.h5 0.94 0.0 b5c93e804c6f8f471163afa8d5eb68f363342ad0\n", - "8 a0.94_i120.h5 0.94 120.0 b83845a38e80bbcaeb4ee9817ef2ff1221cf68f4\n", - "9 a0.94_i150.h5 0.94 150.0 5b6eb09381b58bd223da614886f6d2c5a51b9433\n", - "10 a0.94_i180.h5 0.94 180.0 01f3714023142e16a4724ef326068d086fd7c3d8\n", - "11 a0.94_i30.h5 0.94 30.0 00f6caf5b1a313bde78a907a5f773daefcb715c4\n", - "12 a0.94_i60.h5 0.94 60.0 1101bf0507f2e227abb023a2b541e0f260f3042c\n", - "13 a0.94_i90.h5 0.94 90.0 07fbb0434afd32bf404eb6b3d0c3c87243b40bef\n", - "14 a0_i0.h5 0.00 0.0 18682593a012503b71935017dbc044665254a23c\n", - "15 a0_i120.h5 0.00 120.0 1c057a4885fd04379ee1217375720665713c902f\n", - "16 a0_i150.h5 0.00 150.0 1ed0f877b0802b7d8e91bffa3292255c982e4e85\n", - "17 a0_i180.h5 0.00 180.0 0295ffa54defdc8b51cfc998f5e35aebcf01768d\n", - "18 a0_i30.h5 0.00 30.0 829b87d845b367a9cc87df6ff510d1bdb7444aca\n", - "19 a0_i60.h5 0.00 60.0 075a30e0a9c4dddd92447f463219d4c2a842883d\n", - "20 a0_i90.h5 0.00 90.0 de40f4983d86342b03809bfcc09958f43f359b89\n" - ] - } - ], - "source": [ - "repo = Repo.init(\"repo1\")\n", - "pf = repo.add(\"a{spin}_i{inc}.h5\")\n", - "print(pf)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 4. Inspect the cache after `main` add" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "repo1/.hm/cache/00/f6caf5b1a313bde78a907a5f773daefcb715c4\n", - "repo1/.hm/cache/01/f3714023142e16a4724ef326068d086fd7c3d8\n", - "repo1/.hm/cache/02/95ffa54defdc8b51cfc998f5e35aebcf01768d\n", - "repo1/.hm/cache/07/5a30e0a9c4dddd92447f463219d4c2a842883d\n", - "repo1/.hm/cache/07/fbb0434afd32bf404eb6b3d0c3c87243b40bef\n", - "repo1/.hm/cache/11/01bf0507f2e227abb023a2b541e0f260f3042c\n", - "repo1/.hm/cache/18/682593a012503b71935017dbc044665254a23c\n", - "repo1/.hm/cache/1c/057a4885fd04379ee1217375720665713c902f\n", - "repo1/.hm/cache/1e/d0f877b0802b7d8e91bffa3292255c982e4e85\n", - "repo1/.hm/cache/5b/6eb09381b58bd223da614886f6d2c5a51b9433\n", - "repo1/.hm/cache/82/9b87d845b367a9cc87df6ff510d1bdb7444aca\n", - "repo1/.hm/cache/89/0b2ea509575be9bc74ad515731a2562cd7406e\n", - "repo1/.hm/cache/b5/c93e804c6f8f471163afa8d5eb68f363342ad0\n", - "repo1/.hm/cache/b8/3845a38e80bbcaeb4ee9817ef2ff1221cf68f4\n", - "repo1/.hm/cache/c1/5ca8e6bdad8707bfdf66d1b4f93b5e56a3c094\n", - "repo1/.hm/cache/d9/cfcdd0b44d81c10f88e126d5b1eea4914259a0\n", - "repo1/.hm/cache/db/c88b1f6975ee423b864dfe729cf4643da93459\n", - "repo1/.hm/cache/de/40f4983d86342b03809bfcc09958f43f359b89\n", - "repo1/.hm/cache/ec/4dae9f0b8f69ac42f0c290fc84e0d6f1bec6cd\n", - "repo1/.hm/cache/ed/2cfc6bb157a42729d3c6f45b228b432edec4be\n", - "repo1/.hm/cache/f6/56d419104e7af783c119186a4d81b15563310f\n" - ] - } - ], - "source": [ - "%%bash\n", - "find repo1/.hm/cache -type f | sort" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 5. Checkout a new branch" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "repo.checkout(\"experiment\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 6. Create different data files on `experiment`" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "~/EHT/hallmark/demo/experiment ~/EHT/hallmark/demo\n", - "~/EHT/hallmark/demo\n", - "b0.5_i0.h5\n", - "b0.5_i135.h5\n", - "b0.5_i45.h5\n", - "b0.5_i90.h5\n", - "b1.0_i0.h5\n", - "b1.0_i135.h5\n", - "b1.0_i45.h5\n", - "b1.0_i90.h5\n" - ] - } - ], - "source": [ - "%%bash\n", - "pushd experiment\n", - "for f in b{0.5,1.0}_i{0,45,90,135}.h5; do\n", - " echo \"$f\" > \"$f\"\n", - "done\n", - "popd\n", - "ls experiment" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 7. Add files on `experiment`" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " path spin inc sha1\n", - "0 b0.5_i0.h5 0.5 0.0 0473f32e1e78a36133ffb00d00e5ea0499abae64\n", - "1 b0.5_i135.h5 0.5 135.0 765c7b7756a5be9809e5818a6ce9dd02b5027649\n", - "2 b0.5_i45.h5 0.5 45.0 078949b464c37c1208b0884982cea6165e57f689\n", - "3 b0.5_i90.h5 0.5 90.0 87b4a3a1ef37d9f94b5e58ef78a5e61b0e74a4f8\n", - "4 b1.0_i0.h5 1.0 0.0 5dae25cfd3c075a136e88f1ed309a5d7cfbffbc3\n", - "5 b1.0_i135.h5 1.0 135.0 abca08587d969c4549429c8cb062f7541e4261f2\n", - "6 b1.0_i45.h5 1.0 45.0 cddf02809edd7c3651bc58c516e142339f2a3080\n", - "7 b1.0_i90.h5 1.0 90.0 26d32c645c218e21519d279528ba75a657bdc4f9\n" - ] - } - ], - "source": [ - "repo_exp = Repo(\"experiment\")\n", - "pf_exp = repo_exp.add(\"b{spin}_i{inc}.h5\")\n", - "print(pf_exp)" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "id": "b8f057f3", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "repo1/.hm/cache/00/f6caf5b1a313bde78a907a5f773daefcb715c4\n", - "repo1/.hm/cache/01/f3714023142e16a4724ef326068d086fd7c3d8\n", - "repo1/.hm/cache/02/95ffa54defdc8b51cfc998f5e35aebcf01768d\n", - "repo1/.hm/cache/04/73f32e1e78a36133ffb00d00e5ea0499abae64\n", - "repo1/.hm/cache/07/5a30e0a9c4dddd92447f463219d4c2a842883d\n", - "repo1/.hm/cache/07/8949b464c37c1208b0884982cea6165e57f689\n", - "repo1/.hm/cache/07/fbb0434afd32bf404eb6b3d0c3c87243b40bef\n", - "repo1/.hm/cache/11/01bf0507f2e227abb023a2b541e0f260f3042c\n", - "repo1/.hm/cache/18/682593a012503b71935017dbc044665254a23c\n", - "repo1/.hm/cache/1c/057a4885fd04379ee1217375720665713c902f\n", - "repo1/.hm/cache/1e/d0f877b0802b7d8e91bffa3292255c982e4e85\n", - "repo1/.hm/cache/26/d32c645c218e21519d279528ba75a657bdc4f9\n", - "repo1/.hm/cache/5b/6eb09381b58bd223da614886f6d2c5a51b9433\n", - "repo1/.hm/cache/5d/ae25cfd3c075a136e88f1ed309a5d7cfbffbc3\n", - "repo1/.hm/cache/76/5c7b7756a5be9809e5818a6ce9dd02b5027649\n", - "repo1/.hm/cache/82/9b87d845b367a9cc87df6ff510d1bdb7444aca\n", - "repo1/.hm/cache/87/b4a3a1ef37d9f94b5e58ef78a5e61b0e74a4f8\n", - "repo1/.hm/cache/89/0b2ea509575be9bc74ad515731a2562cd7406e\n", - "repo1/.hm/cache/ab/ca08587d969c4549429c8cb062f7541e4261f2\n", - "repo1/.hm/cache/b5/c93e804c6f8f471163afa8d5eb68f363342ad0\n", - "repo1/.hm/cache/b8/3845a38e80bbcaeb4ee9817ef2ff1221cf68f4\n", - "repo1/.hm/cache/c1/5ca8e6bdad8707bfdf66d1b4f93b5e56a3c094\n", - "repo1/.hm/cache/cd/df02809edd7c3651bc58c516e142339f2a3080\n", - "repo1/.hm/cache/d9/cfcdd0b44d81c10f88e126d5b1eea4914259a0\n", - "repo1/.hm/cache/db/c88b1f6975ee423b864dfe729cf4643da93459\n", - "repo1/.hm/cache/de/40f4983d86342b03809bfcc09958f43f359b89\n", - "repo1/.hm/cache/ec/4dae9f0b8f69ac42f0c290fc84e0d6f1bec6cd\n", - "repo1/.hm/cache/ed/2cfc6bb157a42729d3c6f45b228b432edec4be\n", - "repo1/.hm/cache/f6/56d419104e7af783c119186a4d81b15563310f\n" - ] - } - ], - "source": [ - "%%bash\n", - "find repo1/.hm/cache -type f | sort" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": ".hallmark_env (3.13.7)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.13.7" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/demo/demo_index.ipynb b/demo/demo_index.ipynb new file mode 100644 index 0000000..3baba6e --- /dev/null +++ b/demo/demo_index.ipynb @@ -0,0 +1,315 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Hallmark Index Demo\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "59a52343", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The autoreload extension is already loaded. To reload it, use:\n", + " %reload_ext autoreload\n" + ] + } + ], + "source": [ + "# auto-reload modules when they change\n", + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [], + "source": [ + "%%bash\n", + "rm -rf repo1\n", + "rm -rf experiment\n", + "mkdir repo1" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Initialize the hallmark repo" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [], + "source": [ + "from hallmark import Repo\n", + "\n", + "repo1 = Repo.init(\"repo1\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Create sample data files on `main` with unique content" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1 ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", + "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", + "a0_i0.h5\n", + "a0_i120.h5\n", + "a0_i30.h5\n", + "a0_i60.h5\n", + "a0_i90.h5\n", + "a0.75_i0.h5\n", + "a0.75_i120.h5\n", + "a0.75_i30.h5\n", + "a0.75_i60.h5\n", + "a0.75_i90.h5\n" + ] + } + ], + "source": [ + "%%bash\n", + "pushd repo1\n", + "for f in a{0,0.75}_i{0,30,60,90,120}.h5; do\n", + " echo \"$f\" > \"$f\"\n", + "done\n", + "popd\n", + "ls repo1" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. Add files on `main` and commit" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " path spin inc sha1\n", + "0 a0.75_i0.h5 0.75 0.0 f656d419104e7af783c119186a4d81b15563310f\n", + "1 a0.75_i120.h5 0.75 120.0 ec4dae9f0b8f69ac42f0c290fc84e0d6f1bec6cd\n", + "2 a0.75_i30.h5 0.75 30.0 890b2ea509575be9bc74ad515731a2562cd7406e\n", + "3 a0.75_i60.h5 0.75 60.0 d9cfcdd0b44d81c10f88e126d5b1eea4914259a0\n", + "4 a0.75_i90.h5 0.75 90.0 ed2cfc6bb157a42729d3c6f45b228b432edec4be\n", + "5 a0_i0.h5 0.00 0.0 18682593a012503b71935017dbc044665254a23c\n", + "6 a0_i120.h5 0.00 120.0 1c057a4885fd04379ee1217375720665713c902f\n", + "7 a0_i30.h5 0.00 30.0 829b87d845b367a9cc87df6ff510d1bdb7444aca\n", + "8 a0_i60.h5 0.00 60.0 075a30e0a9c4dddd92447f463219d4c2a842883d\n", + "9 a0_i90.h5 0.00 90.0 de40f4983d86342b03809bfcc09958f43f359b89\n" + ] + } + ], + "source": [ + "repo = Repo.init(\"repo1\")\n", + "pf = repo.add(\"a{spin}_i{inc}.h5\")\n", + "print(pf)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4. Inspect the cache after `main` add" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " 10\n" + ] + } + ], + "source": [ + "%%bash\n", + "find repo1/.hm/index -type f | sort | wc -l" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 5. Checkout a new branch" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "repo.checkout(\"experiment\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 6. Create different data files on `experiment`" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/experiment ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", + "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", + "b0.5_i0.h5\n", + "b0.5_i135.h5\n", + "b0.5_i155.h5\n", + "b0.5_i45.h5\n", + "b0.5_i90.h5\n", + "b1.0_i0.h5\n", + "b1.0_i135.h5\n", + "b1.0_i155.h5\n", + "b1.0_i45.h5\n", + "b1.0_i90.h5\n" + ] + } + ], + "source": [ + "%%bash\n", + "pushd experiment\n", + "for f in b{0.5,1.0}_i{0,45,90,135,155}.h5; do\n", + " echo \"$f\" > \"$f\"\n", + "done\n", + "popd\n", + "ls experiment" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 7. Add files on `experiment`" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " path spin inc sha1\n", + "0 b0.5_i0.h5 0.5 0.0 0473f32e1e78a36133ffb00d00e5ea0499abae64\n", + "1 b0.5_i135.h5 0.5 135.0 765c7b7756a5be9809e5818a6ce9dd02b5027649\n", + "2 b0.5_i155.h5 0.5 155.0 0e1efd47bfd82d1ce17fc97e50db10f317a844c7\n", + "3 b0.5_i45.h5 0.5 45.0 078949b464c37c1208b0884982cea6165e57f689\n", + "4 b0.5_i90.h5 0.5 90.0 87b4a3a1ef37d9f94b5e58ef78a5e61b0e74a4f8\n", + "5 b1.0_i0.h5 1.0 0.0 5dae25cfd3c075a136e88f1ed309a5d7cfbffbc3\n", + "6 b1.0_i135.h5 1.0 135.0 abca08587d969c4549429c8cb062f7541e4261f2\n", + "7 b1.0_i155.h5 1.0 155.0 0854b5b23de3daebf8704a2568ce6eb92d7338a1\n", + "8 b1.0_i45.h5 1.0 45.0 cddf02809edd7c3651bc58c516e142339f2a3080\n", + "9 b1.0_i90.h5 1.0 90.0 26d32c645c218e21519d279528ba75a657bdc4f9\n" + ] + } + ], + "source": [ + "repo_exp = Repo(\"experiment\")\n", + "pf_exp = repo_exp.add(\"b{spin}_i{inc}.h5\")\n", + "print(pf_exp)" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "b8f057f3", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " 20\n" + ] + } + ], + "source": [ + "%%bash\n", + "find repo1/.hm/index -type f | sort | wc -l" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "hallmark", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/mod/hallmark/cache.py b/mod/hallmark/index.py similarity index 89% rename from mod/hallmark/cache.py rename to mod/hallmark/index.py index 1fdd65b..a99a5cb 100644 --- a/mod/hallmark/cache.py +++ b/mod/hallmark/index.py @@ -1,9 +1,9 @@ import shutil from pathlib import Path -class Cache: +class Index: def __init__(self, path): - self.root = Path(path) / "cache" + self.root = Path(path) / "index" def _split_checksum(self, sha1: str) -> Path: return self.root / sha1[:2] / sha1[2:] diff --git a/mod/hallmark/repo.py b/mod/hallmark/repo.py index 778f602..9b5a345 100644 --- a/mod/hallmark/repo.py +++ b/mod/hallmark/repo.py @@ -26,7 +26,7 @@ from .dothm import Dothm from .worktree import Worktree from .paraframe import ParaFrame -from .cache import Cache +from .index import Index @contextmanager @@ -66,17 +66,17 @@ def __init__(self, path: Path | str) -> None: self.state = self.dothm.load() common = Path(self.dothm.common_dir).resolve().parent - self.cache = Cache(common) - dothm_cache = Path(dothm_path) / "cache" - main_cache = common / "cache" - if dothm_cache.resolve() != main_cache.resolve() and not dothm_cache.exists(): - dothm_cache.symlink_to(main_cache) + self.index = Index(common) + dothm_index = Path(dothm_path) / "index" + main_index = common / "index" + if dothm_index.resolve() != main_index.resolve() and not dothm_index.exists(): + dothm_index.symlink_to(main_index) @classmethod def init(cls, path: Path | str) -> "Repo": dothm_path, worktree_path = cls.lwpaths(path) Dothm.init(dothm_path).dump(State()) - Cache(dothm_path) + Index(dothm_path) worktree_path and Worktree.init(worktree_path) return cls(path) @@ -111,7 +111,7 @@ def add(self, fstr: str, encoding: bool = False) -> ParaFrame: self.state.update(pf) self.dothm.dump(self.state) for _, row in pf.iterrows(): - self.cache.store(self.worktree / row["path"], row["sha1"]) + self.index.store(self.worktree / row["path"], row["sha1"]) return pf def commit(self, msg: str, allow_empty: bool = False) -> bool: From 9c65662dc76a963e53355d860a2732690490b8af Mon Sep 17 00:00:00 2001 From: HaydenMarchinek Date: Thu, 16 Apr 2026 16:57:34 -0700 Subject: [PATCH 08/20] Fixed double initialization bug --- demo/demo_index.ipynb | 109 +++++++++++++++++++++++++++++++----------- mod/hallmark/dothm.py | 5 +- mod/hallmark/repo.py | 7 +-- 3 files changed, 88 insertions(+), 33 deletions(-) diff --git a/demo/demo_index.ipynb b/demo/demo_index.ipynb index 3baba6e..823c7ed 100644 --- a/demo/demo_index.ipynb +++ b/demo/demo_index.ipynb @@ -16,7 +16,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 53, "id": "59a52343", "metadata": {}, "outputs": [ @@ -37,18 +37,21 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 54, + "id": "6c48b96a", "metadata": {}, "outputs": [], "source": [ "%%bash\n", "rm -rf repo1\n", "rm -rf experiment\n", + "rm -rf repo\n", "mkdir repo1" ] }, { "cell_type": "markdown", + "id": "f5fea115", "metadata": {}, "source": [ "## 1. Initialize the hallmark repo" @@ -56,7 +59,8 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 55, + "id": "fecfebda", "metadata": {}, "outputs": [], "source": [ @@ -67,6 +71,7 @@ }, { "cell_type": "markdown", + "id": "8600b347", "metadata": {}, "source": [ "## 2. Create sample data files on `main` with unique content" @@ -74,25 +79,32 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 56, + "id": "98aa5cf5", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1 ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", - "a0_i0.h5\n", - "a0_i120.h5\n", - "a0_i30.h5\n", - "a0_i60.h5\n", - "a0_i90.h5\n", + "~/EHT/hallmark/demo/repo1 ~/EHT/hallmark/demo\n", + "~/EHT/hallmark/demo\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ "a0.75_i0.h5\n", "a0.75_i120.h5\n", "a0.75_i30.h5\n", "a0.75_i60.h5\n", - "a0.75_i90.h5\n" + "a0.75_i90.h5\n", + "a0_i0.h5\n", + "a0_i120.h5\n", + "a0_i30.h5\n", + "a0_i60.h5\n", + "a0_i90.h5\n" ] } ], @@ -108,6 +120,7 @@ }, { "cell_type": "markdown", + "id": "8e6b8af6", "metadata": {}, "source": [ "## 3. Add files on `main` and commit" @@ -115,7 +128,8 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 57, + "id": "10a64a1c", "metadata": {}, "outputs": [ { @@ -134,16 +148,27 @@ "8 a0_i60.h5 0.00 60.0 075a30e0a9c4dddd92447f463219d4c2a842883d\n", "9 a0_i90.h5 0.00 90.0 de40f4983d86342b03809bfcc09958f43f359b89\n" ] + }, + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 57, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "repo = Repo.init(\"repo1\")\n", - "pf = repo.add(\"a{spin}_i{inc}.h5\")\n", - "print(pf)" + "pf = repo1.add(\"a{spin}_i{inc}.h5\")\n", + "print(pf)\n", + "repo1.commit(\"add spin and inclination files\")" ] }, { "cell_type": "markdown", + "id": "ded2ad92", "metadata": {}, "source": [ "## 4. Inspect the cache after `main` add" @@ -151,14 +176,15 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 59, + "id": "8e041284", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - " 10\n" + "10\n" ] } ], @@ -169,6 +195,7 @@ }, { "cell_type": "markdown", + "id": "4d8b8c86", "metadata": {}, "source": [ "## 5. Checkout a new branch" @@ -176,7 +203,8 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 60, + "id": "6069f86a", "metadata": {}, "outputs": [ { @@ -185,17 +213,18 @@ "True" ] }, - "execution_count": 27, + "execution_count": 60, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "repo.checkout(\"experiment\")" + "repo1.checkout(\"experiment\")" ] }, { "cell_type": "markdown", + "id": "dc96f623", "metadata": {}, "source": [ "## 6. Create different data files on `experiment`" @@ -203,15 +232,26 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 61, + "id": "082434ad", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/experiment ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", + "~/EHT/hallmark/demo/experiment ~/EHT/hallmark/demo\n", + "~/EHT/hallmark/demo\n", + "a0.75_i0.h5\n", + "a0.75_i120.h5\n", + "a0.75_i30.h5\n", + "a0.75_i60.h5\n", + "a0.75_i90.h5\n", + "a0_i0.h5\n", + "a0_i120.h5\n", + "a0_i30.h5\n", + "a0_i60.h5\n", + "a0_i90.h5\n", "b0.5_i0.h5\n", "b0.5_i135.h5\n", "b0.5_i155.h5\n", @@ -237,6 +277,7 @@ }, { "cell_type": "markdown", + "id": "fc95b2c2", "metadata": {}, "source": [ "## 7. Add files on `experiment`" @@ -244,7 +285,8 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 62, + "id": "83692fe6", "metadata": {}, "outputs": [ { @@ -263,17 +305,28 @@ "8 b1.0_i45.h5 1.0 45.0 cddf02809edd7c3651bc58c516e142339f2a3080\n", "9 b1.0_i90.h5 1.0 90.0 26d32c645c218e21519d279528ba75a657bdc4f9\n" ] + }, + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 62, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ "repo_exp = Repo(\"experiment\")\n", "pf_exp = repo_exp.add(\"b{spin}_i{inc}.h5\")\n", - "print(pf_exp)" + "print(pf_exp)\n", + "repo_exp.commit(\"added more spin and inclination files\")" ] }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 63, "id": "b8f057f3", "metadata": {}, "outputs": [ @@ -281,7 +334,7 @@ "name": "stdout", "output_type": "stream", "text": [ - " 20\n" + "20\n" ] } ], diff --git a/mod/hallmark/dothm.py b/mod/hallmark/dothm.py index 996aa1e..655b4d9 100644 --- a/mod/hallmark/dothm.py +++ b/mod/hallmark/dothm.py @@ -57,8 +57,9 @@ def init(cls, *args, **kwargs) -> "Dothm": It is a git-version-controlled dataset index used by `hallmark`. See https://l6a.github.io/hallmark/ for `hallmark` usage. """) - dothm.index.add([readme_path]) - dothm.index.commit("Initial commit: local `.hm` repository") + if not dothm.heads: + dothm.index.add([readme_path]) + dothm.index.commit("Initial commit: local `.hm` repository") return dothm def link(self, path: Path | str, branch: str | None = None): diff --git a/mod/hallmark/repo.py b/mod/hallmark/repo.py index 9b5a345..c5f4870 100644 --- a/mod/hallmark/repo.py +++ b/mod/hallmark/repo.py @@ -110,15 +110,16 @@ def add(self, fstr: str, encoding: bool = False) -> ParaFrame: self.state.update(pf) self.dothm.dump(self.state) - for _, row in pf.iterrows(): - self.index.store(self.worktree / row["path"], row["sha1"]) return pf def commit(self, msg: str, allow_empty: bool = False) -> bool: if not isinstance(msg, str) or not msg.strip(): raise ValueError("commit message must be a non-empty string") - + if allow_empty or self.dothm.index.diff("HEAD"): + + for _, row in pf.iterrows(): + self.index.store(self.worktree / row["path"], row["sha1"]) self.dothm.index.commit(msg) return True else: From a2071a85dd2dac8ebede8da24b4fa286f8f25cd3 Mon Sep 17 00:00:00 2001 From: HaydenMarchinek Date: Thu, 16 Apr 2026 17:01:32 -0700 Subject: [PATCH 09/20] Restored proper repo.py version --- mod/hallmark/repo.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/mod/hallmark/repo.py b/mod/hallmark/repo.py index c5f4870..9b5a345 100644 --- a/mod/hallmark/repo.py +++ b/mod/hallmark/repo.py @@ -110,16 +110,15 @@ def add(self, fstr: str, encoding: bool = False) -> ParaFrame: self.state.update(pf) self.dothm.dump(self.state) + for _, row in pf.iterrows(): + self.index.store(self.worktree / row["path"], row["sha1"]) return pf def commit(self, msg: str, allow_empty: bool = False) -> bool: if not isinstance(msg, str) or not msg.strip(): raise ValueError("commit message must be a non-empty string") - + if allow_empty or self.dothm.index.diff("HEAD"): - - for _, row in pf.iterrows(): - self.index.store(self.worktree / row["path"], row["sha1"]) self.dothm.index.commit(msg) return True else: From 11cb4eea1726b53bda6787090b64e5463478d3b6 Mon Sep 17 00:00:00 2001 From: Ram Adithya Date: Thu, 16 Apr 2026 17:10:56 -0700 Subject: [PATCH 10/20] Updated the index/ to be made in hallmark commit and removed the sha1 from being displayed for the Paraframe --- mod/hallmark/repo.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mod/hallmark/repo.py b/mod/hallmark/repo.py index 9b5a345..0e0a4fa 100644 --- a/mod/hallmark/repo.py +++ b/mod/hallmark/repo.py @@ -110,15 +110,15 @@ def add(self, fstr: str, encoding: bool = False) -> ParaFrame: self.state.update(pf) self.dothm.dump(self.state) - for _, row in pf.iterrows(): - self.index.store(self.worktree / row["path"], row["sha1"]) - return pf + return pf.drop(columns=["sha1"], errors="ignore") def commit(self, msg: str, allow_empty: bool = False) -> bool: if not isinstance(msg, str) or not msg.strip(): raise ValueError("commit message must be a non-empty string") if allow_empty or self.dothm.index.diff("HEAD"): + for _, row in self.state.data.iterrows(): + self.index.store(self.worktree / row["path"], row["sha1"]) self.dothm.index.commit(msg) return True else: From 25a49d6fecc8fa5736a8c7605309b6d0004cee6b Mon Sep 17 00:00:00 2001 From: Ram Adithya Date: Thu, 16 Apr 2026 19:17:33 -0700 Subject: [PATCH 11/20] Updated the checkout() feature that covers all scenarios; Updated demo notebooks; Updated few test suites --- demo/demo_bash.ipynb | 656 +++++++++++++++++++++++++++++++++++----- demo/demo_index.ipynb | 633 ++++++++++++++++++++++++++++++-------- demo/demo_python.ipynb | 50 +-- mod/hallmark/cli.py | 16 +- mod/hallmark/dothm.py | 1 + mod/hallmark/index.py | 16 - mod/hallmark/objects.py | 28 ++ mod/hallmark/repo.py | 107 ++++++- mod/hallmark/state.py | 14 +- test/test_cli.py | 19 ++ test/test_hallmark.py | 100 +++++- 11 files changed, 1363 insertions(+), 277 deletions(-) delete mode 100644 mod/hallmark/index.py create mode 100644 mod/hallmark/objects.py diff --git a/demo/demo_bash.ipynb b/demo/demo_bash.ipynb index 3364fc4..c1334b0 100644 --- a/demo/demo_bash.ipynb +++ b/demo/demo_bash.ipynb @@ -2,15 +2,14 @@ "cells": [ { "cell_type": "markdown", - "id": "0", "metadata": {}, "source": [ - "# `Hallmark` Command Line Demo\n", + "# Hallmark Command Line Demo\n", "\n", - "This Jupyter notebook demonstrates how `hallmark` is used in a command line.\n", + "This notebook shows how to use `hallmark` from the command line.\n", "\n", - "Because all cells are shell command, it is much more natural to use a `bash` kernel.\n", - "To install a `bash` kernel to your Jupyter/python (virtual) environment, run\n", + "Because all cells are shell commands, it is much more natural to use a `bash` kernel.\n", + "To install a `bash` kernel to your Jupyter/python environment, run\n", "```bash\n", "pip install bash_kernel\n", "python -m bash_kernel.install\n", @@ -19,18 +18,14 @@ }, { "cell_type": "markdown", - "id": "1", "metadata": {}, "source": [ - "## Remove Old Demo Repositories\n", - "\n", - "First, let's remove all old demo repositories." + "## Remove old demo repos" ] }, { "cell_type": "code", - "execution_count": 1, - "id": "2", + "execution_count": 41, "metadata": { "vscode": { "languageId": "shellscript" @@ -43,19 +38,32 @@ }, { "cell_type": "markdown", - "id": "3", "metadata": {}, "source": [ - "## Initialize a Standard (Non-Bare) `Hallmark` Repository\n", + "## 1. Initialize a hallmark repository\n", "\n", - "`Hallmark`'s commandline syntax is similar to `git`.\n", - "To create an empty (standard) `hallmark` repository, we run `hallmark init [REPO]`." + "`hallmark init` works like `git init`. It creates a `.hm/` directory\n", + "inside the repo path. The `.hm/` is a git repo that tracks `data.tsv`,\n", + "`config.yml`, and `meta.yml`. The shared `objects/` store is created\n", + "lazily when the first `hallmark commit` stores file bytes.\n" ] }, { "cell_type": "code", - "execution_count": 2, - "id": "4", + "execution_count": 42, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [], + "source": [ + "hallmark init repo1" + ] + }, + { + "cell_type": "code", + "execution_count": 43, "metadata": { "vscode": { "languageId": "shellscript" @@ -66,33 +74,76 @@ "name": "stdout", "output_type": "stream", "text": [ - "bash: hallmark: command not found\n" + "repo1:\n", + "\u001b[36m \u001b[1m.hm\u001b[0m\n", + "\n", + "repo1/.hm:\n", + "\u001b[36m \u001b[1m.git\u001b[0m\n", + " config.yml\n", + " data.tsv\n", + " meta.yml\n", + "\u001b[33m󰂺 \u001b[1;4mREADME.md\u001b[0m\n" ] - }, + } + ], + "source": [ + "ls -a repo1 repo1/.hm" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [ { - "ename": "", - "evalue": "127", - "output_type": "error", - "traceback": [] + "name": "stdout", + "output_type": "stream", + "text": [ + "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1/.hm ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", + "On branch main\n", + "Changes to be committed:\n", + " (use \"git restore --staged ...\" to unstage)\n", + "\t\u001b[32mnew file: config.yml\u001b[m\n", + "\t\u001b[32mnew file: data.tsv\u001b[m\n", + "\t\u001b[32mnew file: meta.yml\u001b[m\n", + "\n", + "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n" + ] } ], "source": [ - "hallmark init repo1" + "pushd repo1/.hm && git status && popd" ] }, { "cell_type": "markdown", - "id": "5", "metadata": {}, "source": [ - "Check the content of the hallmark repository.\n", - "It contains a \".hm\" directory, which is itself a git repository." + "### Bare repository\n", + "\n", + "A bare repository has no data directory. Name the path with `.hm` to create one." ] }, { "cell_type": "code", - "execution_count": 3, - "id": "6", + "execution_count": 45, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [], + "source": [ + "hallmark init repo2.hm" + ] + }, + { + "cell_type": "code", + "execution_count": 46, "metadata": { "vscode": { "languageId": "shellscript" @@ -103,163 +154,604 @@ "name": "stdout", "output_type": "stream", "text": [ - "\"repo1\": No such file or directory (os error 2)\n", - "\"repo1/.hm\": No such file or directory (os error 2)\n" + "\u001b[36m \u001b[1m.git\u001b[0m\n", + " config.yml\n", + " data.tsv\n", + " meta.yml\n", + "\u001b[33m󰂺 \u001b[1;4mREADME.md\u001b[0m\n" ] - }, - { - "ename": "", - "evalue": "2", - "output_type": "error", - "traceback": [] } ], "source": [ - "ls -a repo1 repo1/.hm" + "ls -a repo2.hm" ] }, { "cell_type": "markdown", - "id": "7", "metadata": {}, "source": [ - "We can `cd` into the \".hm\" directory and then run `git`.\n", - "However, because `cd` is not a very good practice in Jupyter notebook,\n", - "we will use `pushd` and `popd` instead." + "## 2. Create data files and add them\n", + "\n", + "`hallmark add` scans the working directory for files matching a format string\n", + "and stages a manifest in `data.tsv`. The persisted manifest contains only\n", + "`sha1` and `path`, even though the add result can still show parsed parameters.\n" ] }, { "cell_type": "code", - "execution_count": null, - "id": "8", + "execution_count": 47, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1 ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", + "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", + " a0.75_i0.h5\n", + " a0.75_i30.h5\n", + " a0.75_i60.h5\n", + " a0.75_i90.h5\n", + " a0.75_i120.h5\n", + " a0_i0.h5\n", + " a0_i30.h5\n", + " a0_i60.h5\n", + " a0_i90.h5\n", + " a0_i120.h5\n" + ] + } + ], "source": [ - "pushd repo1/.hm && git status && popd" + "pushd repo1\n", + "for f in a{0,0.75}_i{0,30,60,90,120}.h5; do echo \"$f\" > \"$f\"; done\n", + "popd\n", + "ls repo1" + ] + }, + { + "cell_type": "code", + "execution_count": 48, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1 ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", + "Changes to be committed\n", + " a0.75_i0.h5\n", + "a0.75_i120.h5\n", + " a0.75_i30.h5\n", + " a0.75_i60.h5\n", + " a0.75_i90.h5\n", + " a0_i0.h5\n", + " a0_i120.h5\n", + " a0_i30.h5\n", + " a0_i60.h5\n", + " a0_i90.h5\n", + "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n" + ] + } + ], + "source": [ + "pushd repo1 && hallmark add \"a{a}_i{i}.h5\" && popd" ] }, { "cell_type": "markdown", - "id": "9", "metadata": {}, "source": [ - "Note, similar to `git init`, `hallmark init` does not commit your files." + "## 3. Commit\n", + "\n", + "`hallmark commit` saves the current state. It copies file bytes into the\n", + "shared `objects/` store (keyed by sha1) and commits `data.tsv` in `.hm/`.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 49, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1 ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", + "Committed staged state changes.\n", + "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n" + ] + } + ], + "source": [ + "pushd repo1 && hallmark commit -m \"add spin and inclination files\" && popd" + ] + }, + { + "cell_type": "code", + "execution_count": 50, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Objects stored:\n", + " 10\n" + ] + } + ], + "source": [ + "echo \"Objects stored:\"\n", + "find repo1/.hm/objects -type f | wc -l" ] }, { "cell_type": "markdown", - "id": "10", "metadata": {}, "source": [ - "## Initialize a Bare Hallmark Repository\n", + "## 4. Checkout a new branch\n", "\n", - "Next, we will initialize a bare repository.\n", - "Unlikely `git`, all we need is to explicitly postfix \".hm\" to the repository name." + "`hallmark checkout` works like `git checkout`. If the branch does not exist,\n", + "it creates it (like `git checkout -b`). The working directory is rewritten\n", + "in place using the target branch's committed `data.tsv`. Tracked files from\n", + "the old branch are removed, tracked files from the new branch are restored\n", + "from the shared `objects/` store, and unrelated untracked files are left alone\n", + "unless they block restoration.\n" ] }, { "cell_type": "code", - "execution_count": null, - "id": "11", + "execution_count": 51, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1 ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", + "Switched to branch \"experiment\".\n", + "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n" + ] + } + ], "source": [ - "hallmark init repo2.hm" + "pushd repo1 && hallmark checkout experiment && popd" + ] + }, + { + "cell_type": "code", + "execution_count": 52, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Files after checkout:\n", + " \u001b[36mrepo1/\u001b[0ma0.75_i0.h5\n", + " \u001b[36mrepo1/\u001b[0ma0.75_i30.h5\n", + " \u001b[36mrepo1/\u001b[0ma0.75_i60.h5\n", + " \u001b[36mrepo1/\u001b[0ma0.75_i90.h5\n", + " \u001b[36mrepo1/\u001b[0ma0.75_i120.h5\n", + " \u001b[36mrepo1/\u001b[0ma0_i0.h5\n", + " \u001b[36mrepo1/\u001b[0ma0_i30.h5\n", + " \u001b[36mrepo1/\u001b[0ma0_i60.h5\n", + " \u001b[36mrepo1/\u001b[0ma0_i90.h5\n", + " \u001b[36mrepo1/\u001b[0ma0_i120.h5\n", + "\n", + "Current .hm branch:\n", + "experiment\n" + ] + } + ], + "source": [ + "echo \"Files after checkout:\"\n", + "ls repo1/*.h5\n", + "echo \"\"\n", + "echo \"Current .hm branch:\"\n", + "git -C repo1/.hm branch --show-current" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Same files as before — `experiment` just branched off `main`." ] }, { "cell_type": "markdown", - "id": "12", "metadata": {}, "source": [ - "`repo2.hm` is itself a `git` repository." + "## 5. Add new files on `experiment` and commit" ] }, { "cell_type": "code", - "execution_count": null, - "id": "13", + "execution_count": 53, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1 ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", + "Files after creating experiment files:\n", + " a0.75_i0.h5\n", + " a0.75_i30.h5\n", + " a0.75_i60.h5\n", + " a0.75_i90.h5\n", + " a0.75_i120.h5\n", + " a0_i0.h5\n", + " a0_i30.h5\n", + " a0_i60.h5\n", + " a0_i90.h5\n", + " a0_i120.h5\n", + " b0.5_i0.h5\n", + " b0.5_i45.h5\n", + " b0.5_i90.h5\n", + " b0.5_i135.h5\n", + " b0.5_i155.h5\n", + " b1.0_i0.h5\n", + " b1.0_i45.h5\n", + " b1.0_i90.h5\n", + " b1.0_i135.h5\n", + " b1.0_i155.h5\n", + "Changes to be committed\n", + " b0.5_i0.h5\n", + "b0.5_i135.h5\n", + "b0.5_i155.h5\n", + " b0.5_i45.h5\n", + " b0.5_i90.h5\n", + " b1.0_i0.h5\n", + "b1.0_i135.h5\n", + "b1.0_i155.h5\n", + " b1.0_i45.h5\n", + " b1.0_i90.h5\n", + "Committed staged state changes.\n", + "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n" + ] + } + ], "source": [ - "ls -a repo2.hm" + "pushd repo1\n", + "for f in b{0.5,1.0}_i{0,45,90,135,155}.h5; do echo \"$f\" > \"$f\"; done\n", + "echo \"Files after creating experiment files:\"\n", + "ls *.h5\n", + "hallmark add \"b{spin}_i{inc}.h5\"\n", + "hallmark commit -m \"add experiment files\"\n", + "popd" ] }, { "cell_type": "code", - "execution_count": null, - "id": "14", + "execution_count": 54, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Objects stored after experiment commit:\n", + " 20\n" + ] + } + ], + "source": [ + "echo \"Objects stored after experiment commit:\"\n", + "find repo1/.hm/objects -type f | wc -l" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, "source": [ - "pushd repo2.hm && git status && popd" + "The objects store grew. It holds files from both `main` and `experiment`,\n", + "while each branch keeps its own committed `data.tsv` manifest.\n" ] }, { "cell_type": "markdown", - "id": "15", "metadata": {}, "source": [ - "## Add Files According to Python Format String" + "## 6. Checkout back to `main`\n", + "\n", + "The experiment branch's tracked files are removed from the working directory and\n", + "main's tracked files are restored from `objects/`. No data is lost.\n" ] }, { "cell_type": "code", - "execution_count": null, - "id": "16", + "execution_count": 55, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1 ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", + "Switched to branch \"main\".\n", + "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n" + ] + } + ], "source": [ - "pushd repo1 && touch a{0,0.75,0.94}_i{0..180..30}.h5 && popd\n", - "ls repo1" + "pushd repo1 && hallmark checkout main && popd" ] }, { "cell_type": "code", - "execution_count": null, - "id": "17", + "execution_count": 56, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Files after checkout main:\n", + " \u001b[36mrepo1/\u001b[0ma0.75_i0.h5\n", + " \u001b[36mrepo1/\u001b[0ma0.75_i30.h5\n", + " \u001b[36mrepo1/\u001b[0ma0.75_i60.h5\n", + " \u001b[36mrepo1/\u001b[0ma0.75_i90.h5\n", + " \u001b[36mrepo1/\u001b[0ma0.75_i120.h5\n", + " \u001b[36mrepo1/\u001b[0ma0_i0.h5\n", + " \u001b[36mrepo1/\u001b[0ma0_i30.h5\n", + " \u001b[36mrepo1/\u001b[0ma0_i60.h5\n", + " \u001b[36mrepo1/\u001b[0ma0_i90.h5\n", + " \u001b[36mrepo1/\u001b[0ma0_i120.h5\n", + "\n", + "Current .hm branch:\n", + "main\n", + "\n", + "Objects still in store (nothing lost):\n", + " 20\n" + ] + } + ], "source": [ - "pushd repo1 && hallmark add \"a{a}_i{i}.h5\" && popd" + "echo \"Files after checkout main:\"\n", + "ls repo1/*.h5\n", + "echo \"\"\n", + "echo \"Current .hm branch:\"\n", + "git -C repo1/.hm branch --show-current\n", + "echo \"\"\n", + "echo \"Objects still in store (nothing lost):\"\n", + "find repo1/.hm/objects -type f | wc -l" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Only main's files are in the working directory. The experiment files are safe\n", + "in `objects/` and will come back when you checkout `experiment` again." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 7. Checkout back to `experiment` to verify\n", + "\n", + "Switch back once more to confirm the branch-specific manifest is enough to\n", + "rebuild the worktree from the shared `objects/` store.\n" ] }, { "cell_type": "code", - "execution_count": null, - "id": "18", + "execution_count": 57, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1 ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", + "Switched to branch \"experiment\".\n", + "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n" + ] + } + ], + "source": [ + "pushd repo1 && hallmark checkout experiment && popd" + ] + }, + { + "cell_type": "code", + "execution_count": 58, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Files after checkout experiment:\n", + " \u001b[36mrepo1/\u001b[0ma0.75_i0.h5\n", + " \u001b[36mrepo1/\u001b[0ma0.75_i30.h5\n", + " \u001b[36mrepo1/\u001b[0ma0.75_i60.h5\n", + " \u001b[36mrepo1/\u001b[0ma0.75_i90.h5\n", + " \u001b[36mrepo1/\u001b[0ma0.75_i120.h5\n", + " \u001b[36mrepo1/\u001b[0ma0_i0.h5\n", + " \u001b[36mrepo1/\u001b[0ma0_i30.h5\n", + " \u001b[36mrepo1/\u001b[0ma0_i60.h5\n", + " \u001b[36mrepo1/\u001b[0ma0_i90.h5\n", + " \u001b[36mrepo1/\u001b[0ma0_i120.h5\n", + " \u001b[36mrepo1/\u001b[0mb0.5_i0.h5\n", + " \u001b[36mrepo1/\u001b[0mb0.5_i45.h5\n", + " \u001b[36mrepo1/\u001b[0mb0.5_i90.h5\n", + " \u001b[36mrepo1/\u001b[0mb0.5_i135.h5\n", + " \u001b[36mrepo1/\u001b[0mb0.5_i155.h5\n", + " \u001b[36mrepo1/\u001b[0mb1.0_i0.h5\n", + " \u001b[36mrepo1/\u001b[0mb1.0_i45.h5\n", + " \u001b[36mrepo1/\u001b[0mb1.0_i90.h5\n", + " \u001b[36mrepo1/\u001b[0mb1.0_i135.h5\n", + " \u001b[36mrepo1/\u001b[0mb1.0_i155.h5\n", + "\n", + "Current .hm branch:\n", + "experiment\n" + ] + } + ], + "source": [ + "echo \"Files after checkout experiment:\"\n", + "ls repo1/*.h5\n", + "echo \"\"\n", + "echo \"Current .hm branch:\"\n", + "git -C repo1/.hm branch --show-current" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "All tracked files for `experiment` are back — the files inherited from `main`\n", + "plus the `b*` files committed on `experiment`.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 8. Uncommitted changes block checkout\n", + "\n", + "Just like git, hallmark refuses to checkout if hallmark state has been staged\n", + "but not committed.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 59, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1 ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", + "Files after creating c0_i5.h5:\n", + " a0.75_i0.h5\n", + " a0.75_i30.h5\n", + " a0.75_i60.h5\n", + " a0.75_i90.h5\n", + " a0.75_i120.h5\n", + " a0_i0.h5\n", + " a0_i30.h5\n", + " a0_i60.h5\n", + " a0_i90.h5\n", + " a0_i120.h5\n", + " b0.5_i0.h5\n", + " b0.5_i45.h5\n", + " b0.5_i90.h5\n", + " b0.5_i135.h5\n", + " b0.5_i155.h5\n", + " b1.0_i0.h5\n", + " b1.0_i45.h5\n", + " b1.0_i90.h5\n", + " b1.0_i135.h5\n", + " b1.0_i155.h5\n", + " c0_i5.h5\n", + "Changes to be committed\n", + "c0_i5.h5\n", + "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n" + ] + } + ], + "source": [ + "pushd repo1\n", + "echo \"c0_i5.h5\" > c0_i5.h5\n", + "echo \"Files after creating c0_i5.h5:\"\n", + "ls *.h5\n", + "hallmark add \"c{c}_i{i}.h5\"\n", + "popd" + ] + }, + { + "cell_type": "code", + "execution_count": 60, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1 ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", + "Error: you have uncommitted hallmark state changes — commit them before checkout\n", + "checkout failed as expected\n", + "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n" + ] + } + ], + "source": [ + "pushd repo1\n", + "if hallmark checkout main; then\n", + " echo \"checkout unexpectedly succeeded\"\n", + "else\n", + " echo \"checkout failed as expected\"\n", + "fi\n", + "popd" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, "source": [ - "pushd repo1 && hallmark commit -m \"Commit test\" && popd" + "To proceed, either commit the staged hallmark state first or discard it.\n" ] } ], @@ -277,5 +769,5 @@ } }, "nbformat": 4, - "nbformat_minor": 5 + "nbformat_minor": 4 } diff --git a/demo/demo_index.ipynb b/demo/demo_index.ipynb index 823c7ed..8a89f03 100644 --- a/demo/demo_index.ipynb +++ b/demo/demo_index.ipynb @@ -4,7 +4,16 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Hallmark Index Demo\n" + "# Hallmark Demo: add, commit, and checkout\n", + "\n", + "This notebook walks through the basic hallmark workflow:\n", + "\n", + "1. Initialize a hallmark repo\n", + "2. Create data files, `add` them, and `commit`\n", + "3. `checkout` a new branch\n", + "4. Add different files on the new branch and commit\n", + "5. `checkout` back to `main` \u2014 files swap in place\n", + "6. Try to checkout with uncommitted changes \u2014 hallmark refuses" ] }, { @@ -16,19 +25,9 @@ }, { "cell_type": "code", - "execution_count": 53, - "id": "59a52343", + "execution_count": 1, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The autoreload extension is already loaded. To reload it, use:\n", - " %reload_ext autoreload\n" - ] - } - ], + "outputs": [], "source": [ "# auto-reload modules when they change\n", "%load_ext autoreload\n", @@ -37,174 +36,228 @@ }, { "cell_type": "code", - "execution_count": 54, - "id": "6c48b96a", + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "%%bash\n", "rm -rf repo1\n", - "rm -rf experiment\n", - "rm -rf repo\n", "mkdir repo1" ] }, { "cell_type": "markdown", - "id": "f5fea115", "metadata": {}, "source": [ - "## 1. Initialize the hallmark repo" + "## 1. Initialize the hallmark repo\n", + "\n", + "This creates a `repo1/.hm/` directory. The `.hm/` is a git repo that tracks\n", + "`data.tsv` (the file manifest), `config.yml`, and `meta.yml`.\n", + "The shared `objects/` store is used for file bytes, but it is created lazily on\n", + "the first `commit()` that actually stores blobs.\n" ] }, { "cell_type": "code", - "execution_count": 55, - "id": "fecfebda", + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "from hallmark import Repo\n", "\n", - "repo1 = Repo.init(\"repo1\")" + "repo = Repo.init(\"repo1\")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "=== .hm directory ===\n", + "config.yml\n", + "data.tsv\n", + "meta.yml\n", + "README.md\n", + "\n", + "=== objects stored so far (0 before first commit) ===\n", + " 0\n" + ] + } + ], + "source": [ + "%%bash\n", + "echo \"=== .hm directory ===\"\n", + "ls repo1/.hm/\n", + "echo \"\"\n", + "echo \"=== objects stored so far (0 before first commit) ===\"\n", + "find repo1/.hm/objects -type f 2>/dev/null | wc -l\n" ] }, { "cell_type": "markdown", - "id": "8600b347", "metadata": {}, "source": [ - "## 2. Create sample data files on `main` with unique content" + "## 2. Create data files on `main`\n", + "\n", + "We create 10 `.h5` files with different spin and inclination values.\n", + "These are just dummy files for the demo \u2014 in a real project, these would be\n", + "simulation outputs or observational data." ] }, { "cell_type": "code", - "execution_count": 56, - "id": "98aa5cf5", + "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "~/EHT/hallmark/demo/repo1 ~/EHT/hallmark/demo\n", - "~/EHT/hallmark/demo\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "a0.75_i0.h5\n", - "a0.75_i120.h5\n", - "a0.75_i30.h5\n", - "a0.75_i60.h5\n", - "a0.75_i90.h5\n", + "Files in repo1/:\n", "a0_i0.h5\n", "a0_i120.h5\n", "a0_i30.h5\n", "a0_i60.h5\n", - "a0_i90.h5\n" + "a0_i90.h5\n", + "a0.75_i0.h5\n", + "a0.75_i120.h5\n", + "a0.75_i30.h5\n", + "a0.75_i60.h5\n", + "a0.75_i90.h5\n" ] } ], "source": [ "%%bash\n", - "pushd repo1\n", + "cd repo1\n", "for f in a{0,0.75}_i{0,30,60,90,120}.h5; do\n", " echo \"$f\" > \"$f\"\n", "done\n", - "popd\n", - "ls repo1" + "echo \"Files in repo1/:\"\n", + "ls *.h5" ] }, { "cell_type": "markdown", - "id": "8e6b8af6", "metadata": {}, "source": [ - "## 3. Add files on `main` and commit" + "## 3. Add files and commit on `main`\n", + "\n", + "`repo.add()` does two things:\n", + "- Parses the format string to find matching files and extract parameters (spin, inc)\n", + "- Computes a sha1 checksum for each file and stages a manifest containing only `sha1` and `path` in `data.tsv`\n", + "\n", + "`repo.commit()` does two things:\n", + "- Copies each file's bytes into the shared `objects/` store, keyed by sha1\n", + "- Commits `data.tsv` in the `.hm/` git repo\n", + "\n", + "After commit, the file bytes are safe in `objects/`. Even if the files get\n", + "deleted from the working directory, they can be restored.\n" ] }, { "cell_type": "code", - "execution_count": 57, - "id": "10a64a1c", + "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - " path spin inc sha1\n", - "0 a0.75_i0.h5 0.75 0.0 f656d419104e7af783c119186a4d81b15563310f\n", - "1 a0.75_i120.h5 0.75 120.0 ec4dae9f0b8f69ac42f0c290fc84e0d6f1bec6cd\n", - "2 a0.75_i30.h5 0.75 30.0 890b2ea509575be9bc74ad515731a2562cd7406e\n", - "3 a0.75_i60.h5 0.75 60.0 d9cfcdd0b44d81c10f88e126d5b1eea4914259a0\n", - "4 a0.75_i90.h5 0.75 90.0 ed2cfc6bb157a42729d3c6f45b228b432edec4be\n", - "5 a0_i0.h5 0.00 0.0 18682593a012503b71935017dbc044665254a23c\n", - "6 a0_i120.h5 0.00 120.0 1c057a4885fd04379ee1217375720665713c902f\n", - "7 a0_i30.h5 0.00 30.0 829b87d845b367a9cc87df6ff510d1bdb7444aca\n", - "8 a0_i60.h5 0.00 60.0 075a30e0a9c4dddd92447f463219d4c2a842883d\n", - "9 a0_i90.h5 0.00 90.0 de40f4983d86342b03809bfcc09958f43f359b89\n" + " path spin inc\n", + "0 a0.75_i0.h5 0.75 0.0\n", + "1 a0.75_i120.h5 0.75 120.0\n", + "2 a0.75_i30.h5 0.75 30.0\n", + "3 a0.75_i60.h5 0.75 60.0\n", + "4 a0.75_i90.h5 0.75 90.0\n", + "5 a0_i0.h5 0.00 0.0\n", + "6 a0_i120.h5 0.00 120.0\n", + "7 a0_i30.h5 0.00 30.0\n", + "8 a0_i60.h5 0.00 60.0\n", + "9 a0_i90.h5 0.00 90.0\n" ] - }, + } + ], + "source": [ + "pf = repo.add(\"a{spin}_i{inc}.h5\")\n", + "print(pf)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ { "data": { "text/plain": [ "True" ] }, - "execution_count": 57, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "pf = repo1.add(\"a{spin}_i{inc}.h5\")\n", - "print(pf)\n", - "repo1.commit(\"add spin and inclination files\")" - ] - }, - { - "cell_type": "markdown", - "id": "ded2ad92", - "metadata": {}, - "source": [ - "## 4. Inspect the cache after `main` add" + "repo.commit(\"add spin and inclination files\")" ] }, { "cell_type": "code", - "execution_count": 59, - "id": "8e041284", + "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "10\n" + "Objects stored after commit:\n", + " 10\n" ] } ], "source": [ "%%bash\n", - "find repo1/.hm/index -type f | sort | wc -l" + "echo \"Objects stored after commit:\"\n", + "find repo1/.hm/objects -type f | wc -l" ] }, { "cell_type": "markdown", - "id": "4d8b8c86", "metadata": {}, "source": [ - "## 5. Checkout a new branch" + "10 files added, 10 objects stored. Each object is a copy of a file's bytes,\n", + "named by its sha1 hash (split as `sha1[:2]/sha1[2:]`, same as git)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4. Checkout a new branch\n", + "\n", + "This works like `git checkout -b experiment`. The `.hm/` repo switches to a new\n", + "branch called `experiment`. Since it just branched off `main`, the files are\n", + "the same \u2014 `data.tsv` is identical, and the same files are restored from `objects/`.\n", + "\n", + "What happens under the hood:\n", + "1. Validate that tracked files in the working directory still match the current branch's `data.tsv`\n", + "2. Delete current tracked files from the working directory\n", + "3. Run `git checkout -b experiment` inside `.hm/`\n", + "4. Read the new branch's `data.tsv`\n", + "5. Restore each tracked file from `objects/` using hardlinks\n", + "\n", + "Untracked files are left alone unless they block restoration of a tracked path.\n" ] }, { "cell_type": "code", - "execution_count": 60, - "id": "6069f86a", + "execution_count": 9, "metadata": {}, "outputs": [ { @@ -213,45 +266,88 @@ "True" ] }, - "execution_count": 60, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "repo1.checkout(\"experiment\")" + "repo.checkout(\"experiment\")" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Files in repo1/ after checkout:\n", + "repo1/a0_i0.h5\n", + "repo1/a0_i120.h5\n", + "repo1/a0_i30.h5\n", + "repo1/a0_i60.h5\n", + "repo1/a0_i90.h5\n", + "repo1/a0.75_i0.h5\n", + "repo1/a0.75_i120.h5\n", + "repo1/a0.75_i30.h5\n", + "repo1/a0.75_i60.h5\n", + "repo1/a0.75_i90.h5\n", + "\n", + "Current .hm branch:\n", + "experiment\n" + ] + } + ], + "source": [ + "%%bash\n", + "echo \"Files in repo1/ after checkout:\"\n", + "ls repo1/*.h5\n", + "echo \"\"\n", + "echo \"Current .hm branch:\"\n", + "git -C repo1/.hm branch --show-current" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Same files as before \u2014 because `experiment` just branched off `main`." ] }, { "cell_type": "markdown", - "id": "dc96f623", "metadata": {}, "source": [ - "## 6. Create different data files on `experiment`" + "## 5. Add new files on `experiment` and commit\n", + "\n", + "Now we create different files on the `experiment` branch and track them.\n", + "After commit, `objects/` grows \u2014 it now holds files from both branches, while\n", + "`data.tsv` on each branch still lists only that branch's manifest.\n" ] }, { "cell_type": "code", - "execution_count": 61, - "id": "082434ad", + "execution_count": 11, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "~/EHT/hallmark/demo/experiment ~/EHT/hallmark/demo\n", - "~/EHT/hallmark/demo\n", - "a0.75_i0.h5\n", - "a0.75_i120.h5\n", - "a0.75_i30.h5\n", - "a0.75_i60.h5\n", - "a0.75_i90.h5\n", + "All files in repo1/ now:\n", "a0_i0.h5\n", "a0_i120.h5\n", "a0_i30.h5\n", "a0_i60.h5\n", "a0_i90.h5\n", + "a0.75_i0.h5\n", + "a0.75_i120.h5\n", + "a0.75_i30.h5\n", + "a0.75_i60.h5\n", + "a0.75_i90.h5\n", "b0.5_i0.h5\n", "b0.5_i135.h5\n", "b0.5_i155.h5\n", @@ -267,102 +363,377 @@ ], "source": [ "%%bash\n", - "pushd experiment\n", + "cd repo1\n", "for f in b{0.5,1.0}_i{0,45,90,135,155}.h5; do\n", " echo \"$f\" > \"$f\"\n", "done\n", - "popd\n", - "ls experiment" + "echo \"All files in repo1/ now:\"\n", + "ls *.h5" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " path spin inc\n", + "0 b0.5_i0.h5 0.5 0.0\n", + "1 b0.5_i135.h5 0.5 135.0\n", + "2 b0.5_i155.h5 0.5 155.0\n", + "3 b0.5_i45.h5 0.5 45.0\n", + "4 b0.5_i90.h5 0.5 90.0\n", + "5 b1.0_i0.h5 1.0 0.0\n", + "6 b1.0_i135.h5 1.0 135.0\n", + "7 b1.0_i155.h5 1.0 155.0\n", + "8 b1.0_i45.h5 1.0 45.0\n", + "9 b1.0_i90.h5 1.0 90.0\n" + ] + } + ], + "source": [ + "pf_exp = repo.add(\"b{spin}_i{inc}.h5\")\n", + "print(pf_exp)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "repo.commit(\"add experiment files\")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Objects stored after experiment commit:\n", + " 20\n" + ] + } + ], + "source": [ + "%%bash\n", + "echo \"Objects stored after experiment commit:\"\n", + "find repo1/.hm/objects -type f | wc -l" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "20 objects total \u2014 10 from `main` + 10 new from `experiment`.\n", + "The objects store holds everything across all branches, while each branch keeps\n", + "its own committed `data.tsv`.\n" ] }, { "cell_type": "markdown", - "id": "fc95b2c2", "metadata": {}, "source": [ - "## 7. Add files on `experiment`" + "## 6. Checkout back to `main`\n", + "\n", + "This is where the magic happens. The working directory gets rewritten in place:\n", + "1. All of experiment's tracked files get deleted from the working directory\n", + "2. `.hm/` switches to the `main` branch\n", + "3. Main's files get restored from `objects/`\n", + "\n", + "No data is lost \u2014 experiment's file bytes are still in `objects/`, but `main`\n", + "is rebuilt only from `main`'s `data.tsv`.\n" ] }, { "cell_type": "code", - "execution_count": 62, - "id": "83692fe6", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "repo.checkout(\"main\")" + ] + }, + { + "cell_type": "code", + "execution_count": 16, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - " path spin inc sha1\n", - "0 b0.5_i0.h5 0.5 0.0 0473f32e1e78a36133ffb00d00e5ea0499abae64\n", - "1 b0.5_i135.h5 0.5 135.0 765c7b7756a5be9809e5818a6ce9dd02b5027649\n", - "2 b0.5_i155.h5 0.5 155.0 0e1efd47bfd82d1ce17fc97e50db10f317a844c7\n", - "3 b0.5_i45.h5 0.5 45.0 078949b464c37c1208b0884982cea6165e57f689\n", - "4 b0.5_i90.h5 0.5 90.0 87b4a3a1ef37d9f94b5e58ef78a5e61b0e74a4f8\n", - "5 b1.0_i0.h5 1.0 0.0 5dae25cfd3c075a136e88f1ed309a5d7cfbffbc3\n", - "6 b1.0_i135.h5 1.0 135.0 abca08587d969c4549429c8cb062f7541e4261f2\n", - "7 b1.0_i155.h5 1.0 155.0 0854b5b23de3daebf8704a2568ce6eb92d7338a1\n", - "8 b1.0_i45.h5 1.0 45.0 cddf02809edd7c3651bc58c516e142339f2a3080\n", - "9 b1.0_i90.h5 1.0 90.0 26d32c645c218e21519d279528ba75a657bdc4f9\n" + "Files in repo1/ after checkout main:\n", + "repo1/a0_i0.h5\n", + "repo1/a0_i120.h5\n", + "repo1/a0_i30.h5\n", + "repo1/a0_i60.h5\n", + "repo1/a0_i90.h5\n", + "repo1/a0.75_i0.h5\n", + "repo1/a0.75_i120.h5\n", + "repo1/a0.75_i30.h5\n", + "repo1/a0.75_i60.h5\n", + "repo1/a0.75_i90.h5\n", + "\n", + "Current .hm branch:\n", + "main\n", + "\n", + "Objects still in store (nothing lost):\n", + " 20\n" ] - }, + } + ], + "source": [ + "%%bash\n", + "echo \"Files in repo1/ after checkout main:\"\n", + "ls repo1/*.h5\n", + "echo \"\"\n", + "echo \"Current .hm branch:\"\n", + "git -C repo1/.hm branch --show-current\n", + "echo \"\"\n", + "echo \"Objects still in store (nothing lost):\"\n", + "find repo1/.hm/objects -type f | wc -l" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Only main's 10 `a*` files are in the working directory. The `b*` files are gone\n", + "from the directory but their bytes are safe in `objects/`. Checkout back to\n", + "`experiment` and they come right back.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 7. Checkout back to `experiment` to verify\n", + "\n", + "Let's go back to `experiment` to confirm the files are restored correctly." + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ { "data": { "text/plain": [ "True" ] }, - "execution_count": 62, + "execution_count": 17, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "repo_exp = Repo(\"experiment\")\n", - "pf_exp = repo_exp.add(\"b{spin}_i{inc}.h5\")\n", - "print(pf_exp)\n", - "repo_exp.commit(\"added more spin and inclination files\")" + "repo.checkout(\"experiment\")\n" ] }, { "cell_type": "code", - "execution_count": 63, - "id": "b8f057f3", + "execution_count": 18, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "20\n" + "Files in repo1/ after checkout experiment:\n", + "repo1/a0_i0.h5\n", + "repo1/a0_i120.h5\n", + "repo1/a0_i30.h5\n", + "repo1/a0_i60.h5\n", + "repo1/a0_i90.h5\n", + "repo1/a0.75_i0.h5\n", + "repo1/a0.75_i120.h5\n", + "repo1/a0.75_i30.h5\n", + "repo1/a0.75_i60.h5\n", + "repo1/a0.75_i90.h5\n", + "repo1/b0.5_i0.h5\n", + "repo1/b0.5_i135.h5\n", + "repo1/b0.5_i155.h5\n", + "repo1/b0.5_i45.h5\n", + "repo1/b0.5_i90.h5\n", + "repo1/b1.0_i0.h5\n", + "repo1/b1.0_i135.h5\n", + "repo1/b1.0_i155.h5\n", + "repo1/b1.0_i45.h5\n", + "repo1/b1.0_i90.h5\n", + "\n", + "Current .hm branch:\n", + "experiment\n" ] } ], "source": [ "%%bash\n", - "find repo1/.hm/index -type f | sort | wc -l" + "echo \"Files in repo1/ after checkout experiment:\"\n", + "ls repo1/*.h5\n", + "echo \"\"\n", + "echo \"Current .hm branch:\"\n", + "git -C repo1/.hm branch --show-current" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "All 20 files are back \u2014 the 10 from `main` (inherited when we branched) plus\n", + "the 10 we added on `experiment`." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 8. Uncommitted changes block checkout\n", + "\n", + "Just like git, hallmark will not let you checkout if tracked files have local\n", + "changes or if hallmark state has been added but not committed.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [], + "source": [ + "%%bash\n", + "cd repo1\n", + "echo \"c0_i20.h5\" > c0_i20.h5\n", + "echo \"Files in repo1/ after creating c0_i20.h5:\"\n", + "ls *.h5" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
pathci
0c0_i20.h50.020.0
\n", + "
" + ], + "text/plain": [ + " path c i\n", + "0 c0_i20.h5 0.0 20.0" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "repo.add(\"c{c}_i{i}.h5\")" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Error: you have uncommitted hallmark state changes \u2014 commit them before checkout\n" + ] + } + ], + "source": [ + "# this should fail because we have uncommitted changes\n", + "try:\n", + " repo.checkout(\"main\")\n", + "except RuntimeError as e:\n", + " print(f\"Error: {e}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To fix this, either commit the changes first, or discard them." ] } ], "metadata": { "kernelspec": { - "display_name": "hallmark", + "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.11.0" } }, "nbformat": 4, - "nbformat_minor": 5 + "nbformat_minor": 4 } diff --git a/demo/demo_python.ipynb b/demo/demo_python.ipynb index cbc7e24..c961e5e 100644 --- a/demo/demo_python.ipynb +++ b/demo/demo_python.ipynb @@ -66,7 +66,9 @@ "\n", "`Repo.init(path)` creates a new hallmark repository at `path`.\n", "It works like `git init`: the directory is created with a `.hm` folder\n", - "inside, which is itself a git repository used to store the file index." + "inside, which is itself a git repository used to store the file manifest.\n", + "The shared `objects/` store is created lazily on the first commit that stores\n", + "file bytes.\n" ] }, { @@ -144,7 +146,7 @@ } ], "source": [ - "# the .hm directory is a git repo — you can run git commands inside it\n", + "# the .hm directory is a git repo \u2014 you can run git commands inside it\n", "! pushd repo1/.hm && git status && popd" ] }, @@ -153,9 +155,9 @@ "id": "md-s1-note", "metadata": {}, "source": [ - "The index files (`config.yml`, `data.tsv`, `meta.yml`) are staged but not\n", - "yet committed — just like `git init`, `Repo.init()` does not make the first\n", - "commit for you." + "The `.hm` repository already has an initial README commit. The hallmark state\n", + "files (`config.yml`, `data.tsv`, `meta.yml`) are then staged in the working tree\n", + "so the next `Repo.commit(...)` captures the first manifest snapshot.\n" ] }, { @@ -165,7 +167,7 @@ "source": [ "### Bare repository\n", "\n", - "A bare repository has no data directory — it is just the index.\n", + "A bare repository has no data directory \u2014 it is just the index.\n", "Name the path with a `.hm` suffix to create one." ] }, @@ -208,7 +210,7 @@ } ], "source": [ - "# repo2.hm/ is the index itself — there is no separate data folder\n", + "# repo2.hm/ is the index itself \u2014 there is no separate data folder\n", "! ls -a repo2.hm" ] }, @@ -251,11 +253,12 @@ "## 2. Add and Commit Files\n", "\n", "`repo.add(fmt)` scans `repo1/` for files whose names match the format\n", - "string, computes a checksum for each one, and stages them in the index.\n", - "It returns a table showing the matched files and their parameters.\n", + "string, computes a checksum for each one, and stages them in the manifest.\n", + "It returns a table showing the matched files and their parsed parameters.\n", "\n", - "`repo.commit(msg)` saves the current state of the index as a git commit\n", - "inside `.hm`, giving you a permanent snapshot you can return to later." + "`repo.commit(msg)` saves the current state of the manifest as a git commit\n", + "inside `.hm`, and stores the corresponding file bytes in the shared `objects/`\n", + "store.\n" ] }, { @@ -350,9 +353,9 @@ ], "source": [ "%%bash\n", - "# create sample data files: spin values 0, 0.75, 0.94 at inclinations 0–180\n", + "# create sample data files: spin values 0, 0.75, 0.94 at inclinations 0\u2013180\n", "pushd repo1\n", - "touch a{0,0.75,0.94}_i{0,30,60,90,120,150,180}.h5\n", + "for f in a{0,0.75,0.94}_i{0,30,60,90,120,150,180}.h5; do echo \"$f\" > \"$f\"; done\n", "popd\n", "ls repo1" ] @@ -610,7 +613,7 @@ } ], "source": [ - "# save this snapshot — returns True if a commit was made\n", + "# save this snapshot \u2014 returns True if a commit was made\n", "repo1.commit(\"add spin and inclination files\")" ] }, @@ -622,7 +625,7 @@ "## 3. Discover and Filter Files with ParaFrame\n", "\n", "`repo.add()` uses `ParaFrame` under the hood to find files.\n", - "You can also call `ParaFrame.parse()` directly — useful when you want to\n", + "You can also call `ParaFrame.parse()` directly \u2014 useful when you want to\n", "look up files without adding them to the index.\n", "\n", "`ParaFrame` is a table (a pandas DataFrame) where each row is a file and\n", @@ -830,7 +833,7 @@ } ], "source": [ - "# discover files in repo1/ directly — no index update\n", + "# discover files in repo1/ directly \u2014 no index update\n", "pf = ParaFrame.parse(\"a{a}_i{i}.h5\", base_path=\"repo1\")\n", "pf" ] @@ -1069,9 +1072,9 @@ "### Filter by parameter value\n", "\n", "Call a `ParaFrame` like a function to filter rows:\n", - "- **scalar** — keep rows where the column equals that value\n", - "- **list** — keep rows where the column matches any value in the list\n", - "- **chaining** — each call narrows the result further (AND logic)" + "- **scalar** \u2014 keep rows where the column equals that value\n", + "- **list** \u2014 keep rows where the column matches any value in the list\n", + "- **chaining** \u2014 each call narrows the result further (AND logic)" ] }, { @@ -1665,8 +1668,7 @@ "%%bash\n", "# create files with negative spin values using the m-prefix convention\n", "pushd repo1\n", - "touch am0.75_i0.h5 am0.75_i30.h5 am0.75_i60.h5 \\\n", - " am0.94_i0.h5 am0.94_i30.h5 am0.94_i60.h5\n", + "for f in am0.75_i0.h5 am0.75_i30.h5 am0.75_i60.h5 am0.94_i0.h5 am0.94_i30.h5 am0.94_i60.h5; do echo \"$f\" > \"$f\"; done\n", "popd\n", "ls repo1/am*.h5" ] @@ -1985,7 +1987,7 @@ } ], "source": [ - "# add all files — m0.94 in the filename is decoded to -0.94 in the table\n", + "# add all files \u2014 m0.94 in the filename is decoded to -0.94 in the table\n", "pf_spin = repo1.add(\"a{aspin}_i{i}.h5\", encoding=True)\n", "pf_spin" ] @@ -2379,7 +2381,7 @@ } ], "source": [ - "# filter using the decoded float values — not the raw m-prefix strings\n", + "# filter using the decoded float values \u2014 not the raw m-prefix strings\n", "pf_spin(aspin=[-0.75, -0.94])" ] } @@ -2405,4 +2407,4 @@ }, "nbformat": 4, "nbformat_minor": 5 -} +} \ No newline at end of file diff --git a/mod/hallmark/cli.py b/mod/hallmark/cli.py index 050953a..718e9d5 100644 --- a/mod/hallmark/cli.py +++ b/mod/hallmark/cli.py @@ -104,15 +104,19 @@ def commit(repo, message): else: click.echo("No changes added to commit.") -@hallmark.command(short_help="Checkout to a worktree branch.") +@hallmark.command(short_help="Switch to another branch.") @click.argument("target_branch") @click.pass_obj def checkout(repo, target_branch): - """Checkout to another worktree branch and restore the state saved in data.tsv. + """Switch branches and rewrite tracked files from branch state. This is analogous to `git checkout BRANCH`. + If the branch does not exist, it is created from the current branch. + Only hallmark-tracked files are rewritten; unrelated files are left + alone unless they block restoration of a tracked path. """ - if repo.checkout(target_branch): - click.echo(f'Checked out to "{target_branch}".') - else: - click.echo("No branches to checkout.") \ No newline at end of file + try: + if repo.checkout(target_branch): + click.echo(f'Switched to branch "{target_branch}".') + except (GitError, RuntimeError, ValueError, FileNotFoundError) as e: + raise ClickException(str(e)) diff --git a/mod/hallmark/dothm.py b/mod/hallmark/dothm.py index 655b4d9..82450f5 100644 --- a/mod/hallmark/dothm.py +++ b/mod/hallmark/dothm.py @@ -47,6 +47,7 @@ def __init__(self, *args, **kwargs): def init(cls, *args, **kwargs) -> "Dothm": if kwargs.get('bare', False): raise DothmError('A ".hm" directory must not be a bare git repository') + kwargs.setdefault("initial_branch", "main") dothm = super().init(*args, **kwargs) readme_path = dothm.path / "README.md" diff --git a/mod/hallmark/index.py b/mod/hallmark/index.py deleted file mode 100644 index a99a5cb..0000000 --- a/mod/hallmark/index.py +++ /dev/null @@ -1,16 +0,0 @@ -import shutil -from pathlib import Path - -class Index: - def __init__(self, path): - self.root = Path(path) / "index" - - def _split_checksum(self, sha1: str) -> Path: - return self.root / sha1[:2] / sha1[2:] - - def store(self, src: Path, sha1: str) -> Path: - stored_checksum = self._split_checksum(sha1) - if not stored_checksum.exists(): - stored_checksum.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(src, stored_checksum) - return stored_checksum \ No newline at end of file diff --git a/mod/hallmark/objects.py b/mod/hallmark/objects.py new file mode 100644 index 0000000..4c0f0c0 --- /dev/null +++ b/mod/hallmark/objects.py @@ -0,0 +1,28 @@ +import os +import shutil +from pathlib import Path + + +class Objects: + def __init__(self, path): + self.root = Path(path) / "objects" + + def _split_checksum(self, sha1: str) -> Path: + return self.root / sha1[:2] / sha1[2:] + + def store(self, src: Path, sha1: str) -> Path: + stored_checksum = self._split_checksum(sha1) + if not stored_checksum.exists(): + stored_checksum.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, stored_checksum) + return stored_checksum + + def restore(self, sha1: str, dest: Path) -> Path: + stored = self._split_checksum(sha1) + if not stored.exists(): + raise FileNotFoundError(f"object {sha1} not found in objects store") + dest.parent.mkdir(parents=True, exist_ok=True) + if dest.exists(): + dest.unlink() + os.link(stored, dest) + return dest diff --git a/mod/hallmark/repo.py b/mod/hallmark/repo.py index 0e0a4fa..2fa8131 100644 --- a/mod/hallmark/repo.py +++ b/mod/hallmark/repo.py @@ -15,6 +15,7 @@ from __future__ import annotations +from io import StringIO import os from contextlib import contextmanager from dataclasses import dataclass @@ -22,11 +23,13 @@ from pathlib import Path from hashlib import sha1 +import pandas as pd + from .state import State from .dothm import Dothm from .worktree import Worktree from .paraframe import ParaFrame -from .index import Index +from .objects import Objects @contextmanager @@ -66,17 +69,17 @@ def __init__(self, path: Path | str) -> None: self.state = self.dothm.load() common = Path(self.dothm.common_dir).resolve().parent - self.index = Index(common) - dothm_index = Path(dothm_path) / "index" - main_index = common / "index" - if dothm_index.resolve() != main_index.resolve() and not dothm_index.exists(): - dothm_index.symlink_to(main_index) + self.objects = Objects(common) + dothm_objects = Path(dothm_path) / "objects" + main_objects = common / "objects" + if dothm_objects.resolve() != main_objects.resolve() and not dothm_objects.exists(): + dothm_objects.symlink_to(main_objects) @classmethod def init(cls, path: Path | str) -> "Repo": dothm_path, worktree_path = cls.lwpaths(path) Dothm.init(dothm_path).dump(State()) - Index(dothm_path) + Objects(dothm_path) worktree_path and Worktree.init(worktree_path) return cls(path) @@ -88,6 +91,42 @@ def checksum(path: Path, chunk_size: int = 1024 * 1024) -> str: digest.update(block) return digest.hexdigest() + def _tracked_paths(self) -> set[Path]: + return {Path(path) for path in self.state.data["path"].astype(str)} + + def _ensure_clean_tracked_files(self) -> None: + if self.worktree is None: + raise RuntimeError("cannot checkout without a worktree") + + for _, row in self.state.data.iterrows(): + path = self.worktree / row["path"] + if not path.exists(): + raise RuntimeError( + f'tracked file "{row["path"]}" is missing; commit or restore it before checkout') + if self.checksum(path) != row["sha1"]: + raise RuntimeError( + f'tracked file "{row["path"]}" has uncommitted changes; commit them before checkout') + + if self.dothm.index.diff("HEAD"): + raise RuntimeError( + "you have uncommitted hallmark state changes — commit them before checkout") + + def _load_branch_data(self, branch: str) -> State: + if branch in {head.name for head in self.dothm.heads}: + data = self.dothm.git.show(f"{branch}:data.tsv") + frame = State().data if not data.strip() else None + if frame is None: + parsed = pd.read_csv(StringIO(data), sep="\t") + else: + parsed = frame + return State(self.state.config, self.state.meta, parsed) + + return State( + dict(self.state.config), + dict(self.state.meta), + self.state.data.copy(), + ) + def add(self, fstr: str, encoding: bool = False) -> ParaFrame: if self.worktree is None: raise RuntimeError( @@ -118,13 +157,53 @@ def commit(self, msg: str, allow_empty: bool = False) -> bool: if allow_empty or self.dothm.index.diff("HEAD"): for _, row in self.state.data.iterrows(): - self.index.store(self.worktree / row["path"], row["sha1"]) + self.objects.store(self.worktree / row["path"], row["sha1"]) self.dothm.index.commit(msg) return True else: return False def checkout(self, target_branch: str) -> bool: + if not isinstance(target_branch, str) or not target_branch.strip(): + raise ValueError("branch name must be a non-empty string") + + if self.worktree is None: + raise RuntimeError("cannot checkout without a worktree") + self._ensure_clean_tracked_files() + + existing = {head.name for head in self.dothm.heads} + new_branch = target_branch not in existing + current_tracked = self._tracked_paths() + target_state = self._load_branch_data(target_branch) + + for rel in target_state.data["path"].astype(str): + rel_path = Path(rel) + if rel_path not in current_tracked and (self.worktree / rel_path).exists(): + raise RuntimeError( + f'target tracked path "{rel}" already exists as an untracked file') + + # remove current tracked files from worktree + for _, row in self.state.data.iterrows(): + path = self.worktree / row["path"] + if path.exists(): + path.unlink() + + # switch .hm branch + if new_branch: + self.dothm.git.checkout("-b", target_branch) + else: + self.dothm.git.checkout(target_branch) + + # reload state from the new branch + self.state = self.dothm.load() + + # restore files from objects store via hardlinks + for _, row in self.state.data.iterrows(): + self.objects.restore(row["sha1"], self.worktree / row["path"]) + + return True + + def add_worktree(self, target_branch: str) -> bool: from shutil import copy2 from git.exc import GitCommandError @@ -132,14 +211,14 @@ def checkout(self, target_branch: str) -> bool: raise ValueError("branch name must be a non-empty string") if self.worktree is None: - raise RuntimeError("cannot checkout a bare repository without a worktree") - + raise RuntimeError("cannot add a worktree in a bare repository without a worktree") + source = Path(self.worktree).resolve() target = source.parent / target_branch target_dothm = target / ".hm" - + linked_dothm = None - + if target_dothm.exists(): linked_dothm = Dothm(target_dothm) else: @@ -153,7 +232,7 @@ def checkout(self, target_branch: str) -> bool: linked_dothm = Dothm(target_dothm) except GitCommandError as e: raise RuntimeError(f'failed to create worktree for branch "{target_branch}": {e}') - + target_state = linked_dothm.load() for rel in target_state.data["path"]: rel_path = Path(rel) @@ -161,4 +240,4 @@ def checkout(self, target_branch: str) -> bool: dest = target / rel_path dest.parent.mkdir(parents=True, exist_ok=True) copy2(src, dest) - return True \ No newline at end of file + return True diff --git a/mod/hallmark/state.py b/mod/hallmark/state.py index 7204815..fb5e19a 100644 --- a/mod/hallmark/state.py +++ b/mod/hallmark/state.py @@ -18,6 +18,9 @@ import pandas as pd +PERSISTED_COLUMNS = ["sha1", "path"] + + @dataclass class State: """In-memory hallmark state database. @@ -32,15 +35,20 @@ class State: config: dict = field(default_factory=dict) meta: dict = field(default_factory=dict) data: pd.DataFrame = field( - default_factory=lambda: pd.DataFrame(columns=["sha1", "path"]) + default_factory=lambda: pd.DataFrame(columns=PERSISTED_COLUMNS) ) def update(self, pf): - merged = pd.concat([self.data, pf], ignore_index=True, sort=False) + if pf.empty: + incoming = pd.DataFrame(columns=PERSISTED_COLUMNS) + else: + incoming = pf.loc[:, PERSISTED_COLUMNS].copy() + + merged = pd.concat([self.data, incoming], ignore_index=True, sort=False) # If the same path is added again (e.g., file content changed # and a new sha1 is computed), keep only the newest row for # that key. deduped = merged.drop_duplicates(subset=["path"], keep="last") - self.data = deduped + self.data = deduped.loc[:, PERSISTED_COLUMNS] diff --git a/test/test_cli.py b/test/test_cli.py index 904a494..86f3195 100644 --- a/test/test_cli.py +++ b/test/test_cli.py @@ -61,3 +61,22 @@ def test_cli(): result = runner.invoke(hallmark, ["commit", "-m", "Commit test"]) assert result.exit_code == 0 assert "Committed staged state changes." in result.output + + result = runner.invoke(hallmark, ["checkout", "experiment"]) + assert result.exit_code == 0 + assert 'Switched to branch "experiment".' in result.output + + Path("b0_i45.h5").write_text("branch data\n", encoding="utf-8") + result = runner.invoke(hallmark, ["add", "b{spin}_i{inc}.h5"]) + assert result.exit_code == 0 + result = runner.invoke(hallmark, ["commit", "-m", "Commit experiment"]) + assert result.exit_code == 0 + + result = runner.invoke(hallmark, ["checkout", "main"]) + assert result.exit_code == 0 + assert not Path("b0_i45.h5").exists() + + Path("a0_i0.h5").write_text("dirty\n", encoding="utf-8") + result = runner.invoke(hallmark, ["checkout", "experiment"]) + assert result.exit_code != 0 + assert "has uncommitted changes" in result.output diff --git a/test/test_hallmark.py b/test/test_hallmark.py index d8c2eee..c6137e5 100644 --- a/test/test_hallmark.py +++ b/test/test_hallmark.py @@ -1,4 +1,7 @@ +from pathlib import Path + import pandas as pd +import pytest from hallmark import Repo from hallmark import ParaFrame @@ -134,4 +137,99 @@ def test_repo_commit_succeeds(hallmark_test_suite_dictionary): def test_data_tsv_and_worktree_reconstruction(hallmark_test_suite_dictionary): repo = Repo(hallmark_test_suite_dictionary["repo_path"]) assert len(repo.state.data) == 12 - assert repo.worktree.stem == "repo" \ No newline at end of file + assert repo.worktree.stem == "repo" + + +def _write_files(root, names): + for name in names: + (root / name).write_text(f"{name}\n", encoding="utf-8") + + +def test_repo_add_persists_only_sha1_and_path(tmp_path): + repo = Repo.init(tmp_path / "repo") + _write_files(repo.worktree, ["a0_i0.h5", "a0_i30.h5"]) + + result = repo.add("a{a}_i{i}.h5") + + assert list(result.columns) == ["path", "a", "i"] + persisted = repo.dothm.load_tsv("data") + assert list(persisted.columns) == ["sha1", "path"] + assert sorted(persisted["path"]) == ["a0_i0.h5", "a0_i30.h5"] + + +def test_checkout_rewrites_tracked_files_and_shares_objects(tmp_path): + repo = Repo.init(tmp_path / "repo") + _write_files(repo.worktree, ["a0_i0.h5", "a0_i30.h5"]) + repo.add("a{a}_i{i}.h5") + repo.commit("main data") + + main_objects = sorted(p.relative_to(repo.dothm.path) for p in (repo.dothm.path / "objects").rglob("*") if p.is_file()) + assert len(main_objects) == 2 + + repo.checkout("experiment") + _write_files(repo.worktree, ["b0_i45.h5", "b1_i90.h5"]) + repo.add("b{spin}_i{inc}.h5") + repo.commit("experiment data") + + experiment_files = sorted(path.name for path in Path(repo.worktree).glob("*.h5")) + assert experiment_files == ["a0_i0.h5", "a0_i30.h5", "b0_i45.h5", "b1_i90.h5"] + + repo.checkout("main") + main_files = sorted(path.name for path in Path(repo.worktree).glob("*.h5")) + assert main_files == ["a0_i0.h5", "a0_i30.h5"] + + objects_after = [p for p in (repo.dothm.path / "objects").rglob("*") if p.is_file()] + assert len(objects_after) == 4 + + repo.checkout("experiment") + roundtrip_files = sorted(path.name for path in Path(repo.worktree).glob("*.h5")) + assert roundtrip_files == ["a0_i0.h5", "a0_i30.h5", "b0_i45.h5", "b1_i90.h5"] + + +def test_checkout_leaves_untracked_files(tmp_path): + repo = Repo.init(tmp_path / "repo") + _write_files(repo.worktree, ["a0_i0.h5"]) + repo.add("a{a}_i{i}.h5") + repo.commit("main data") + + repo.checkout("experiment") + _write_files(repo.worktree, ["b0_i45.h5"]) + repo.add("b{spin}_i{inc}.h5") + repo.commit("experiment data") + (repo.worktree / "notes.txt").write_text("keep me\n", encoding="utf-8") + + repo.checkout("main") + + assert (repo.worktree / "notes.txt").read_text(encoding="utf-8") == "keep me\n" + assert sorted(path.name for path in Path(repo.worktree).glob("*.h5")) == ["a0_i0.h5"] + + +def test_checkout_aborts_on_dirty_tracked_file(tmp_path): + repo = Repo.init(tmp_path / "repo") + _write_files(repo.worktree, ["a0_i0.h5"]) + repo.add("a{a}_i{i}.h5") + repo.commit("main data") + repo.checkout("experiment") + repo.checkout("main") + (repo.worktree / "a0_i0.h5").write_text("changed\n", encoding="utf-8") + + with pytest.raises(RuntimeError, match='tracked file "a0_i0.h5" has uncommitted changes'): + repo.checkout("experiment") + + +def test_checkout_aborts_on_untracked_path_conflict(tmp_path): + repo = Repo.init(tmp_path / "repo") + _write_files(repo.worktree, ["a0_i0.h5"]) + repo.add("a{a}_i{i}.h5") + repo.commit("main data") + + repo.checkout("experiment") + _write_files(repo.worktree, ["b0_i45.h5"]) + repo.add("b{spin}_i{inc}.h5") + repo.commit("experiment data") + repo.checkout("main") + + (repo.worktree / "b0_i45.h5").write_text("untracked blocker\n", encoding="utf-8") + + with pytest.raises(RuntimeError, match='target tracked path "b0_i45.h5" already exists as an untracked file'): + repo.checkout("experiment") From e82545f8f4a8682a30b2a0ace7df7dfb7eff639e Mon Sep 17 00:00:00 2001 From: Nayera Abdessalam Date: Fri, 17 Apr 2026 10:37:18 -0700 Subject: [PATCH 12/20] implementing download/fetch feature with clone feature (from copilot) --- demo/demo_bash.ipynb | 234 +++++++++++++++++++---- mod/hallmark/cli.py | 77 +++++++- mod/hallmark/downloader.py | 368 +++++++++++++++++++++++++++++++++++++ 3 files changed, 640 insertions(+), 39 deletions(-) create mode 100644 mod/hallmark/downloader.py diff --git a/demo/demo_bash.ipynb b/demo/demo_bash.ipynb index 3364fc4..caf8ccf 100644 --- a/demo/demo_bash.ipynb +++ b/demo/demo_bash.ipynb @@ -61,21 +61,7 @@ "languageId": "shellscript" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "bash: hallmark: command not found\n" - ] - }, - { - "ename": "", - "evalue": "127", - "output_type": "error", - "traceback": [] - } - ], + "outputs": [], "source": [ "hallmark init repo1" ] @@ -103,15 +89,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "\"repo1\": No such file or directory (os error 2)\n", - "\"repo1/.hm\": No such file or directory (os error 2)\n" + "repo1:\n", + ".\t..\t.hm\n", + "\n", + "repo1/.hm:\n", + ".\t\t.git\t\tdata.tsv\tREADME.md\n", + "..\t\tconfig.yml\tmeta.yml\n" ] - }, - { - "ename": "", - "evalue": "2", - "output_type": "error", - "traceback": [] } ], "source": [ @@ -130,14 +114,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "8", "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "~/astro_se_int/hallmark/demo/repo1/.hm ~/astro_se_int/hallmark/demo\n", + "On branch main\n", + "Changes to be committed:\n", + " (use \"git restore --staged ...\" to unstage)\n", + "\t\u001b[32mnew file: config.yml\u001b[m\n", + "\t\u001b[32mnew file: data.tsv\u001b[m\n", + "\t\u001b[32mnew file: meta.yml\u001b[m\n", + "\n", + "~/astro_se_int/hallmark/demo\n" + ] + } + ], "source": [ "pushd repo1/.hm && git status && popd" ] @@ -163,7 +163,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "11", "metadata": { "vscode": { @@ -185,28 +185,53 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "13", "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + ".\t\t.git\t\tdata.tsv\tREADME.md\n", + "..\t\tconfig.yml\tmeta.yml\n" + ] + } + ], "source": [ "ls -a repo2.hm" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "id": "14", "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "~/astro_se_int/hallmark/demo/repo2.hm ~/astro_se_int/hallmark/demo\n", + "On branch main\n", + "Changes to be committed:\n", + " (use \"git restore --staged ...\" to unstage)\n", + "\t\u001b[32mnew file: config.yml\u001b[m\n", + "\t\u001b[32mnew file: data.tsv\u001b[m\n", + "\t\u001b[32mnew file: meta.yml\u001b[m\n", + "\n", + "~/astro_se_int/hallmark/demo\n" + ] + } + ], "source": [ "pushd repo2.hm && git status && popd" ] @@ -221,14 +246,24 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "id": "16", "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "~/astro_se_int/hallmark/demo/repo1 ~/astro_se_int/hallmark/demo\n", + "~/astro_se_int/hallmark/demo\n", + "a0_i{0..180..30}.h5\ta0.75_i{0..180..30}.h5\ta0.94_i{0..180..30}.h5\n" + ] + } + ], "source": [ "pushd repo1 && touch a{0,0.75,0.94}_i{0..180..30}.h5 && popd\n", "ls repo1" @@ -236,31 +271,158 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "id": "17", "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "~/astro_se_int/hallmark/demo/repo1 ~/astro_se_int/hallmark/demo\n", + "Failed to parse \"/Users/nayera/astro_se_int/hallmark/demo/repo1/a0.75_i{0..180..30}.h5\"\n", + "Failed to parse \"/Users/nayera/astro_se_int/hallmark/demo/repo1/a0.94_i{0..180..30}.h5\"\n", + "Failed to parse \"/Users/nayera/astro_se_int/hallmark/demo/repo1/a0_i{0..180..30}.h5\"\n", + "No files matched the format string.\n", + "~/astro_se_int/hallmark/demo\n" + ] + } + ], "source": [ "pushd repo1 && hallmark add \"a{a}_i{i}.h5\" && popd" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "id": "18", "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "~/astro_se_int/hallmark/demo/repo1 ~/astro_se_int/hallmark/demo\n", + "Committed staged state changes.\n", + "~/astro_se_int/hallmark/demo\n" + ] + } + ], "source": [ "pushd repo1 && hallmark commit -m \"Commit test\" && popd" ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "ba243031", + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Traceback (most recent call last):\n", + " File \u001b[35m\"/opt/anaconda3/envs/astro_se_int/bin/hallmark\"\u001b[0m, line \u001b[35m7\u001b[0m, in \u001b[35m\u001b[0m\n", + " sys.exit(\u001b[31mhallmark\u001b[0m\u001b[1;31m()\u001b[0m)\n", + " \u001b[31m~~~~~~~~\u001b[0m\u001b[1;31m^^\u001b[0m\n", + " File \u001b[35m\"/opt/anaconda3/envs/astro_se_int/lib/python3.13/site-packages/click/core.py\"\u001b[0m, line \u001b[35m1485\u001b[0m, in \u001b[35m__call__\u001b[0m\n", + " return \u001b[31mself.main\u001b[0m\u001b[1;31m(*args, **kwargs)\u001b[0m\n", + " \u001b[31m~~~~~~~~~\u001b[0m\u001b[1;31m^^^^^^^^^^^^^^^^^\u001b[0m\n", + " File \u001b[35m\"/opt/anaconda3/envs/astro_se_int/lib/python3.13/site-packages/click/core.py\"\u001b[0m, line \u001b[35m1406\u001b[0m, in \u001b[35mmain\u001b[0m\n", + " rv = self.invoke(ctx)\n", + " File \u001b[35m\"/opt/anaconda3/envs/astro_se_int/lib/python3.13/site-packages/click/core.py\"\u001b[0m, line \u001b[35m1873\u001b[0m, in \u001b[35minvoke\u001b[0m\n", + " return _process_result(\u001b[31msub_ctx.command.invoke\u001b[0m\u001b[1;31m(sub_ctx)\u001b[0m)\n", + " \u001b[31m~~~~~~~~~~~~~~~~~~~~~~\u001b[0m\u001b[1;31m^^^^^^^^^\u001b[0m\n", + " File \u001b[35m\"/opt/anaconda3/envs/astro_se_int/lib/python3.13/site-packages/click/core.py\"\u001b[0m, line \u001b[35m1269\u001b[0m, in \u001b[35minvoke\u001b[0m\n", + " return \u001b[31mctx.invoke\u001b[0m\u001b[1;31m(self.callback, **ctx.params)\u001b[0m\n", + " \u001b[31m~~~~~~~~~~\u001b[0m\u001b[1;31m^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n", + " File \u001b[35m\"/opt/anaconda3/envs/astro_se_int/lib/python3.13/site-packages/click/core.py\"\u001b[0m, line \u001b[35m824\u001b[0m, in \u001b[35minvoke\u001b[0m\n", + " return callback(*args, **kwargs)\n", + " File \u001b[35m\"/Users/nayera/astro_se_int/hallmark/mod/hallmark/cli.py\"\u001b[0m, line \u001b[35m130\u001b[0m, in \u001b[35mclone\u001b[0m\n", + " \u001b[31mRepo.clone\u001b[0m\u001b[1;31m(url, path)\u001b[0m\n", + " \u001b[31m~~~~~~~~~~\u001b[0m\u001b[1;31m^^^^^^^^^^^\u001b[0m\n", + " File \u001b[35m\"/Users/nayera/astro_se_int/hallmark/mod/hallmark/repo.py\"\u001b[0m, line \u001b[35m93\u001b[0m, in \u001b[35mclone\u001b[0m\n", + " return cls(path)\n", + " File \u001b[35m\"/Users/nayera/astro_se_int/hallmark/mod/hallmark/repo.py\"\u001b[0m, line \u001b[35m66\u001b[0m, in \u001b[35m__init__\u001b[0m\n", + " self.state = \u001b[31mself.dothm.load\u001b[0m\u001b[1;31m()\u001b[0m\n", + " \u001b[31m~~~~~~~~~~~~~~~\u001b[0m\u001b[1;31m^^\u001b[0m\n", + " File \u001b[35m\"/Users/nayera/astro_se_int/hallmark/mod/hallmark/dothm.py\"\u001b[0m, line \u001b[35m97\u001b[0m, in \u001b[35mload\u001b[0m\n", + " \u001b[31mself.load_yml\u001b[0m\u001b[1;31m(\"config\")\u001b[0m,\n", + " \u001b[31m~~~~~~~~~~~~~\u001b[0m\u001b[1;31m^^^^^^^^^^\u001b[0m\n", + " File \u001b[35m\"/Users/nayera/astro_se_int/hallmark/mod/hallmark/dothm.py\"\u001b[0m, line \u001b[35m110\u001b[0m, in \u001b[35mload_yml\u001b[0m\n", + " return \u001b[31myaml.safe_load\u001b[0m\u001b[1;31m(f)\u001b[0m\n", + " \u001b[31m~~~~~~~~~~~~~~\u001b[0m\u001b[1;31m^^^\u001b[0m\n", + " File \u001b[35m\"/opt/anaconda3/envs/astro_se_int/lib/python3.13/site-packages/yaml/__init__.py\"\u001b[0m, line \u001b[35m125\u001b[0m, in \u001b[35msafe_load\u001b[0m\n", + " return load(stream, SafeLoader)\n", + " File \u001b[35m\"/opt/anaconda3/envs/astro_se_int/lib/python3.13/site-packages/yaml/__init__.py\"\u001b[0m, line \u001b[35m81\u001b[0m, in \u001b[35mload\u001b[0m\n", + " return \u001b[31mloader.get_single_data\u001b[0m\u001b[1;31m()\u001b[0m\n", + " \u001b[31m~~~~~~~~~~~~~~~~~~~~~~\u001b[0m\u001b[1;31m^^\u001b[0m\n", + " File \u001b[35m\"/opt/anaconda3/envs/astro_se_int/lib/python3.13/site-packages/yaml/constructor.py\"\u001b[0m, line \u001b[35m49\u001b[0m, in \u001b[35mget_single_data\u001b[0m\n", + " node = self.get_single_node()\n", + " File \u001b[35m\"/opt/anaconda3/envs/astro_se_int/lib/python3.13/site-packages/yaml/composer.py\"\u001b[0m, line \u001b[35m36\u001b[0m, in \u001b[35mget_single_node\u001b[0m\n", + " document = self.compose_document()\n", + " File \u001b[35m\"/opt/anaconda3/envs/astro_se_int/lib/python3.13/site-packages/yaml/composer.py\"\u001b[0m, line \u001b[35m55\u001b[0m, in \u001b[35mcompose_document\u001b[0m\n", + " node = self.compose_node(None, None)\n", + " File \u001b[35m\"/opt/anaconda3/envs/astro_se_int/lib/python3.13/site-packages/yaml/composer.py\"\u001b[0m, line \u001b[35m84\u001b[0m, in \u001b[35mcompose_node\u001b[0m\n", + " node = self.compose_mapping_node(anchor)\n", + " File \u001b[35m\"/opt/anaconda3/envs/astro_se_int/lib/python3.13/site-packages/yaml/composer.py\"\u001b[0m, line \u001b[35m133\u001b[0m, in \u001b[35mcompose_mapping_node\u001b[0m\n", + " item_value = self.compose_node(node, item_key)\n", + " File \u001b[35m\"/opt/anaconda3/envs/astro_se_int/lib/python3.13/site-packages/yaml/composer.py\"\u001b[0m, line \u001b[35m64\u001b[0m, in \u001b[35mcompose_node\u001b[0m\n", + " if \u001b[31mself.check_event\u001b[0m\u001b[1;31m(AliasEvent)\u001b[0m:\n", + " \u001b[31m~~~~~~~~~~~~~~~~\u001b[0m\u001b[1;31m^^^^^^^^^^^^\u001b[0m\n", + " File \u001b[35m\"/opt/anaconda3/envs/astro_se_int/lib/python3.13/site-packages/yaml/parser.py\"\u001b[0m, line \u001b[35m98\u001b[0m, in \u001b[35mcheck_event\u001b[0m\n", + " self.current_event = \u001b[31mself.state\u001b[0m\u001b[1;31m()\u001b[0m\n", + " \u001b[31m~~~~~~~~~~\u001b[0m\u001b[1;31m^^\u001b[0m\n", + " File \u001b[35m\"/opt/anaconda3/envs/astro_se_int/lib/python3.13/site-packages/yaml/parser.py\"\u001b[0m, line \u001b[35m449\u001b[0m, in \u001b[35mparse_block_mapping_value\u001b[0m\n", + " if not \u001b[31mself.check_token\u001b[0m\u001b[1;31m(KeyToken, ValueToken, BlockEndToken)\u001b[0m:\n", + " \u001b[31m~~~~~~~~~~~~~~~~\u001b[0m\u001b[1;31m^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n", + " File \u001b[35m\"/opt/anaconda3/envs/astro_se_int/lib/python3.13/site-packages/yaml/scanner.py\"\u001b[0m, line \u001b[35m116\u001b[0m, in \u001b[35mcheck_token\u001b[0m\n", + " \u001b[31mself.fetch_more_tokens\u001b[0m\u001b[1;31m()\u001b[0m\n", + " \u001b[31m~~~~~~~~~~~~~~~~~~~~~~\u001b[0m\u001b[1;31m^^\u001b[0m\n", + " File \u001b[35m\"/opt/anaconda3/envs/astro_se_int/lib/python3.13/site-packages/yaml/scanner.py\"\u001b[0m, line \u001b[35m258\u001b[0m, in \u001b[35mfetch_more_tokens\u001b[0m\n", + " raise ScannerError(\"while scanning for the next token\", None,\n", + " \"found character %r that cannot start any token\" % ch,\n", + " self.get_mark())\n", + "\u001b[1;35myaml.scanner.ScannerError\u001b[0m: \u001b[35mwhile scanning for the next token\n", + "found character '\\t' that cannot start any token\n", + " in \"/Users/nayera/astro_se_int/hallmark/demo/repo3/.hm/config.yml\", line 2, column 1\u001b[0m\n" + ] + }, + { + "ename": "", + "evalue": "1", + "output_type": "error", + "traceback": [] + } + ], + "source": [ + "hallmark clone https://github.com/l6a/hallmark-demo-repo.hm.git repo3" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "145862aa", + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/mod/hallmark/cli.py b/mod/hallmark/cli.py index bd80e8a..e5a6f28 100644 --- a/mod/hallmark/cli.py +++ b/mod/hallmark/cli.py @@ -17,10 +17,12 @@ import click +import sys from click import ClickException from git.exc import GitError from .error import DothmError +from .downloader import DownloadProgress from . import Repo # from "__init__.py" @@ -118,16 +120,85 @@ def checkout(repo, target_branch): else: click.echo("No branches to checkout.") +# Updated mod/hallmark/cli.py section + @hallmark.command(short_help="Clone a hallmark repository from a remote URL.") @click.argument("url") @click.argument("path") -def clone(url, path): +@click.option( + "--no-fetch-data", + is_flag=True, + help="Skip downloading remote data files after clone.") +@click.option( + "--max-workers", + type=int, + default=4, + show_default=True, + help="Number of concurrent downloads.") +def clone(url, path, no_fetch_data, max_workers): """Clone a hallmark repository from URL to PATH. - This is analogous to `git clone URL PATH`. + By default, also downloads data files from the configured remote URL. + Use --no-fetch-data to skip this step. + + Supports concurrent downloads for efficient retrieval of large datasets. """ try: - Repo.clone(url, path) + repo = Repo.clone(url, path) click.echo(f'Successfully cloned to "{path}"') + + if not no_fetch_data: + dothm_path, worktree_path = Repo.lwpaths(path) + if worktree_path is None: + click.echo("Bare repository clone; skipping data download.") + return + + # Setup progress callback for CLI + def progress_callback(progress: DownloadProgress): + """Display download progress.""" + percent = progress.percent + bar_length = 40 + filled = int(bar_length * percent / 100) + bar = "█" * filled + "░" * (bar_length - filled) + mb_downloaded = progress.downloaded_bytes / (1024 * 1024) + mb_total = progress.total_bytes / (1024 * 1024) + sys.stdout.write( + f"\r{progress.filename}: [{bar}] " + f"{percent:.1f}% ({mb_downloaded:.1f}/{mb_total:.1f} MB)" + ) + sys.stdout.flush() + + click.echo("Downloading remote data files...") + from .downloader import download_remote_data, DownloadError + + results = download_remote_data( + repo, + worktree_path, + progress_callback=progress_callback, + max_workers=max_workers, + ) + + click.echo() # New line after progress bar + + if results['failed'] == 0: + mb_total = results['total_bytes'] / (1024 * 1024) + click.echo( + f"✓ Successfully downloaded {results['succeeded']} files " + f"({mb_total:.1f} MB)" + ) + else: + click.echo( + f"⚠ Download completed with errors: " + f"{results['succeeded']} succeeded, " + f"{results['failed']} failed" + ) + for error in results['errors']: + click.echo(f" - {error}", err=True) + + if results['failed'] == len(results['errors']): + raise ClickException( + f"Failed to download {results['failed']} file(s)" + ) + except (DothmError, GitError) as e: raise ClickException(f'Clone failed: {e}') \ No newline at end of file diff --git a/mod/hallmark/downloader.py b/mod/hallmark/downloader.py new file mode 100644 index 0000000..2307f05 --- /dev/null +++ b/mod/hallmark/downloader.py @@ -0,0 +1,368 @@ +# mod/hallmark/downloader.py + +# Copyright 2025 the Hallmark Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Remote data file downloader for hallmark repositories.""" + +import os +import sys +import threading +from pathlib import Path +from typing import Optional, Callable, Tuple +from urllib.parse import urljoin +from concurrent.futures import ThreadPoolExecutor, as_completed +from dataclasses import dataclass +import hashlib + +import pandas as pd +import requests +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry + +from .error import HallmarkError + + +@dataclass +class DownloadProgress: + + """Track download progress for a single file.""" + filename: str + total_bytes: int + downloaded_bytes: int = 0 + + @property + def percent(self) -> float: + if self.total_bytes == 0: + return 0.0 + return (self.downloaded_bytes / self.total_bytes) * 100 + + +class DownloadError(HallmarkError): + """Raised when remote data download fails.""" + + +class CyVerseDownloader: + """Efficient downloader for CyVerse and HTTP-based remote sources. + + Handles: + - Concurrent downloads with connection pooling + - Streaming downloads with minimal memory footprint + - Progress tracking and reporting + - Resume capability for interrupted downloads + - Automatic retry on transient failures + """ + + # Configuration constants + DEFAULT_CHUNK_SIZE = 8192 # 8KB chunks for memory efficiency + DEFAULT_MAX_WORKERS = 4 # Concurrent downloads + CONNECT_TIMEOUT = 10 + READ_TIMEOUT = 30 + MAX_RETRIES = 3 + + def __init__( + self, + max_workers: int = DEFAULT_MAX_WORKERS, + chunk_size: int = DEFAULT_CHUNK_SIZE, + progress_callback: Optional[Callable[[DownloadProgress], None]] = None, + ): + """Initialize downloader. + + Args: + max_workers: Number of concurrent download threads + chunk_size: Size of chunks to download at once + progress_callback: Optional callback for progress updates + """ + self.max_workers = max_workers + self.chunk_size = chunk_size + self.progress_callback = progress_callback + self.session = self._create_session() + self._lock = threading.Lock() # For thread-safe progress updates + + def _create_session(self) -> requests.Session: + """Create requests session with retry strategy and connection pooling.""" + session = requests.Session() + + # Retry strategy for transient failures + retry_strategy = Retry( + total=self.MAX_RETRIES, + backoff_factor=1, # Exponential backoff: 1s, 2s, 4s + status_forcelist=[429, 500, 502, 503, 504], + allowed_methods=["GET", "HEAD"] + ) + + adapter = HTTPAdapter( + max_retries=retry_strategy, + pool_connections=self.max_workers, + pool_maxsize=self.max_workers, + ) + + session.mount("http://", adapter) + session.mount("https://", adapter) + session.mount("dav://", adapter) # WebDAV support for CyVerse + + return session + + def _get_remote_size(self, url: str) -> int: + """Get file size from remote server without downloading. + + Args: + url: Remote file URL + + Returns: + File size in bytes, or -1 if unknown + + Raises: + DownloadError: If HEAD request fails + """ + try: + resp = self.session.head( + url, + timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT), + allow_redirects=True + ) + resp.raise_for_status() + return int(resp.headers.get("content-length", -1)) + except requests.RequestException as e: + raise DownloadError(f"Failed to get size of {url}: {e}") + + def _verify_file_integrity( + self, + filepath: Path, + expected_sha1: Optional[str] = None + ) -> bool: + """Verify downloaded file integrity via checksum. + + Args: + filepath: Path to downloaded file + expected_sha1: Expected SHA1 hash, if available + + Returns: + True if verified or no checksum provided, False otherwise + """ + if not expected_sha1: + return True + + try: + sha1 = hashlib.sha1() + with open(filepath, "rb") as f: + for chunk in iter(lambda: f.read(self.chunk_size), b""): + sha1.update(chunk) + return sha1.hexdigest() == expected_sha1 + except (IOError, OSError) as e: + raise DownloadError(f"Failed to verify {filepath}: {e}") + + def _download_single_file( + self, + url: str, + destination: Path, + expected_sha1: Optional[str] = None, + ) -> Tuple[Path, int]: + """Download a single file with streaming and progress tracking. + + Args: + url: Remote file URL + destination: Local destination path + expected_sha1: Expected SHA1 for verification + + Returns: + Tuple of (destination_path, bytes_downloaded) + + Raises: + DownloadError: If download fails + """ + destination.parent.mkdir(parents=True, exist_ok=True) + + # Check for partial download and get remote size + try: + remote_size = self._get_remote_size(url) + except DownloadError as e: + raise DownloadError(f"Cannot download {url}: {e}") + + # Support resume for partial downloads + resume_header = {} + local_size = destination.stat().st_size if destination.exists() else 0 + + if destination.exists() and local_size < remote_size: + resume_header = {"Range": f"bytes={local_size}-"} + elif destination.exists() and local_size == remote_size: + # File already fully downloaded + return destination, local_size + else: + # Start fresh if file is corrupted + destination.unlink(missing_ok=True) + + # Download with streaming + progress = DownloadProgress( + filename=destination.name, + total_bytes=remote_size if remote_size > 0 else 0, + downloaded_bytes=local_size if resume_header else 0 + ) + + try: + resp = self.session.get( + url, + stream=True, + timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT), + headers=resume_header, + ) + resp.raise_for_status() + + mode = "ab" if resume_header else "wb" + bytes_downloaded = local_size if resume_header else 0 + + with open(destination, mode) as f: + for chunk in resp.iter_content(chunk_size=self.chunk_size): + if chunk: # Filter keep-alive chunks + f.write(chunk) + bytes_downloaded += len(chunk) + progress.downloaded_bytes = bytes_downloaded + + # Thread-safe progress callback + if self.progress_callback: + with self._lock: + self.progress_callback(progress) + + # Verify integrity + if not self._verify_file_integrity(destination, expected_sha1): + raise DownloadError( + f"Checksum mismatch for {destination.name}" + ) + + return destination, bytes_downloaded + + except requests.RequestException as e: + raise DownloadError(f"Failed to download {url}: {e}") + + def download_batch( + self, + files: list[Tuple[str, Path, Optional[str]]], + ) -> dict: + """Download multiple files concurrently. + + Args: + files: List of (url, destination, sha1) tuples + + Returns: + Dictionary with download statistics: + { + 'succeeded': int, + 'failed': int, + 'total_bytes': int, + 'errors': list of error messages + } + """ + results = { + 'succeeded': 0, + 'failed': 0, + 'total_bytes': 0, + 'errors': [] + } + + with ThreadPoolExecutor(max_workers=self.max_workers) as executor: + # Submit all download tasks + future_to_file = { + executor.submit( + self._download_single_file, + url, + dest, + sha1 + ): dest + for url, dest, sha1 in files + } + + # Process completed downloads + for future in as_completed(future_to_file): + dest = future_to_file[future] + try: + _, bytes_downloaded = future.result() + results['succeeded'] += 1 + results['total_bytes'] += bytes_downloaded + except DownloadError as e: + results['failed'] += 1 + results['errors'].append(str(e)) + + return results + + def close(self): + """Close the session and clean up resources.""" + self.session.close() + + def __enter__(self): + """Context manager entry.""" + return self + + def __exit__(self, *args): + """Context manager exit.""" + self.close() + + +def download_remote_data( + repo, + worktree_path: Path, + progress_callback: Optional[Callable[[DownloadProgress], None]] = None, + max_workers: int = CyVerseDownloader.DEFAULT_MAX_WORKERS, +) -> dict: + """Download remote data files for a cloned repository. + + Args: + repo: Hallmark Repo instance + worktree_path: Path to worktree where files should be downloaded + progress_callback: Optional callback for progress updates + max_workers: Number of concurrent downloads + + Returns: + Dictionary with download statistics + + Raises: + DownloadError: If remote configuration is invalid + """ + remote_config = repo.state.config.get("remote", {}) + if not remote_config: + return {'succeeded': 0, 'failed': 0, 'total_bytes': 0, 'errors': []} + + remote_url = remote_config.get("url", "").rstrip("/") + if not remote_url: + raise DownloadError("Remote URL not configured in config.yml") + + # Get list of files to download from data.tsv + files_to_download = [] + data_df = repo.state.data + + if data_df.empty: + return {'succeeded': 0, 'failed': 0, 'total_bytes': 0, 'errors': []} + + for _, row in data_df.iterrows(): + rel_path = Path(row['path']) + destination = worktree_path / rel_path + + # Construct remote URL + file_url = urljoin(remote_url + "/", str(rel_path)) + + # Get SHA1 if available for verification + sha1 = None + if 'sha1' in row.index and pd.notna(row['sha1']): # CORRECTED + sha1 = row['sha1'] + + files_to_download.append((file_url, destination, sha1)) + + if not files_to_download: + return {'succeeded': 0, 'failed': 0, 'total_bytes': 0, 'errors': []} + + # Perform concurrent downloads + with CyVerseDownloader( + max_workers=max_workers, + progress_callback=progress_callback + ) as downloader: + return downloader.download_batch(files_to_download) \ No newline at end of file From c2ef0a4f882d4b3f9fd304cbd6d0efb95ee0aa88 Mon Sep 17 00:00:00 2001 From: Ram Adithya Date: Fri, 17 Apr 2026 17:46:28 -0700 Subject: [PATCH 13/20] Updated the checkout feature to take care of deletion; config.yml is dynamic; Updated demo notebooks; Split repo.py to make it cleaner; Added new CLI for set-config; hallmark now supports 'hallmark add .' and 'hallmark add *' similar to git; Updated test suites --- demo/demo_bash.ipynb | 787 +++++------ demo/demo_index.ipynb | 1022 +++++++++++--- demo/demo_python.ipynb | 2414 ++++----------------------------- mod/hallmark/cli.py | 111 +- mod/hallmark/dothm.py | 16 +- mod/hallmark/repo.py | 165 ++- mod/hallmark/repo_config.py | 119 ++ mod/hallmark/repo_manifest.py | 45 + mod/hallmark/repo_state.py | 66 + mod/hallmark/repo_worktree.py | 54 + mod/hallmark/state.py | 39 +- test/conftest.py | 16 +- test/test_cli.py | 115 +- test/test_hallmark.py | 201 ++- 14 files changed, 2230 insertions(+), 2940 deletions(-) create mode 100644 mod/hallmark/repo_config.py create mode 100644 mod/hallmark/repo_manifest.py create mode 100644 mod/hallmark/repo_state.py create mode 100644 mod/hallmark/repo_worktree.py diff --git a/demo/demo_bash.ipynb b/demo/demo_bash.ipynb index c1334b0..f885abf 100644 --- a/demo/demo_bash.ipynb +++ b/demo/demo_bash.ipynb @@ -4,28 +4,19 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Hallmark Command Line Demo\n", - "\n", - "This notebook shows how to use `hallmark` from the command line.\n", - "\n", - "Because all cells are shell commands, it is much more natural to use a `bash` kernel.\n", - "To install a `bash` kernel to your Jupyter/python environment, run\n", - "```bash\n", - "pip install bash_kernel\n", - "python -m bash_kernel.install\n", - "```" + "# Hallmark Command Line Demo: add, commit, checkout, and status\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Remove old demo repos" + "## Setup\n" ] }, { "cell_type": "code", - "execution_count": 41, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" @@ -33,24 +24,21 @@ }, "outputs": [], "source": [ - "rm -rf repo*" + "rm -rf repo1\n", + "rm -rf repo2.hm\n", + "mkdir repo1\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## 1. Initialize a hallmark repository\n", - "\n", - "`hallmark init` works like `git init`. It creates a `.hm/` directory\n", - "inside the repo path. The `.hm/` is a git repo that tracks `data.tsv`,\n", - "`config.yml`, and `meta.yml`. The shared `objects/` store is created\n", - "lazily when the first `hallmark commit` stores file bytes.\n" + "## 1. Initialize the hallmark repo\n" ] }, { "cell_type": "code", - "execution_count": 42, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" @@ -58,79 +46,63 @@ }, "outputs": [], "source": [ - "hallmark init repo1" + "hallmark init repo1\n" ] }, { "cell_type": "code", - "execution_count": 43, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "repo1:\n", - "\u001b[36m \u001b[1m.hm\u001b[0m\n", - "\n", - "repo1/.hm:\n", - "\u001b[36m \u001b[1m.git\u001b[0m\n", - " config.yml\n", - " data.tsv\n", - " meta.yml\n", - "\u001b[33m󰂺 \u001b[1;4mREADME.md\u001b[0m\n" - ] - } - ], + "outputs": [], + "source": [ + "echo \"=== .hm directory ===\"\n", + "ls repo1/.hm/\n", + "echo \"\"\n", + "echo \"=== initial config.yml ===\"\n", + "cat repo1/.hm/config.yml\n", + "echo \"\"\n", + "echo \"=== objects stored so far ===\"\n", + "find repo1/.hm/objects -type f 2>/dev/null | wc -l\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, "source": [ - "ls -a repo1 repo1/.hm" + "## 2. Create data files on `main`\n" ] }, { "cell_type": "code", - "execution_count": 44, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1/.hm ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", - "On branch main\n", - "Changes to be committed:\n", - " (use \"git restore --staged ...\" to unstage)\n", - "\t\u001b[32mnew file: config.yml\u001b[m\n", - "\t\u001b[32mnew file: data.tsv\u001b[m\n", - "\t\u001b[32mnew file: meta.yml\u001b[m\n", - "\n", - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n" - ] - } - ], + "outputs": [], "source": [ - "pushd repo1/.hm && git status && popd" + "pushd repo1\n", + "for f in a{0,0.75}_i{0,30,60,90,120}.h5; do echo \"$f\" > \"$f\"; done\n", + "echo \"Files in repo1/:\"\n", + "ls *.h5\n", + "popd\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### Bare repository\n", - "\n", - "A bare repository has no data directory. Name the path with `.hm` to create one." + "## 3. Add files and commit on `main`\n" ] }, { "cell_type": "code", - "execution_count": 45, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" @@ -138,605 +110,514 @@ }, "outputs": [], "source": [ - "hallmark init repo2.hm" + "pushd repo1 && hallmark add \"a{a}_i{i}.h5\" && popd\n" ] }, { "cell_type": "code", - "execution_count": 46, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[36m \u001b[1m.git\u001b[0m\n", - " config.yml\n", - " data.tsv\n", - " meta.yml\n", - "\u001b[33m󰂺 \u001b[1;4mREADME.md\u001b[0m\n" - ] + "outputs": [], + "source": [ + "echo \"=== config.yml after add ===\"\n", + "cat repo1/.hm/config.yml\n", + "echo \"\"\n", + "echo \"=== data.tsv after add ===\"\n", + "cat repo1/.hm/data.tsv\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "vscode": { + "languageId": "shellscript" } - ], + }, + "outputs": [], + "source": [ + "pushd repo1 && hallmark commit -m \"add main dataset\" && popd\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [], + "source": [ + "echo \"Objects stored after commit:\"\n", + "find repo1/.hm/objects -type f | wc -l\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Clean status\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [], + "source": [ + "pushd repo1 && hallmark status && popd\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, "source": [ - "ls -a repo2.hm" + "### Current add modes\n", + "\n", + "- `hallmark add \"a{a}_i{i}.h5\"` parses files using the branch fmt and updates `config.yml`.\n", + "- `hallmark add .` rebuilds `data.tsv` from files that currently exist using the fmt already set in `config.yml`.\n", + "- `hallmark set-config --fmt ...` is the explicit way to change branch fmt before `hallmark add .`.\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## 2. Create data files and add them\n", + "## 4. Demo: `hallmark add .` removes deleted files from the manifest\n", "\n", - "`hallmark add` scans the working directory for files matching a format string\n", - "and stages a manifest in `data.tsv`. The persisted manifest contains only\n", - "`sha1` and `path`, even though the add result can still show parsed parameters.\n" + "We use a temporary branch so the main workflow stays intact.\n", + "On this branch we switch to `b*` files first so it is easy to distinguish from `main`.\n", + "Because the filename family changes, we update the branch fmt explicitly with `hallmark set-config` before using `hallmark add .`.\n" ] }, { "cell_type": "code", - "execution_count": 47, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1 ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", - " a0.75_i0.h5\n", - " a0.75_i30.h5\n", - " a0.75_i60.h5\n", - " a0.75_i90.h5\n", - " a0.75_i120.h5\n", - " a0_i0.h5\n", - " a0_i30.h5\n", - " a0_i60.h5\n", - " a0_i90.h5\n", - " a0_i120.h5\n" - ] + "outputs": [], + "source": [ + "pushd repo1 && hallmark checkout sync-demo && popd\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "vscode": { + "languageId": "shellscript" } - ], + }, + "outputs": [], "source": [ "pushd repo1\n", - "for f in a{0,0.75}_i{0,30,60,90,120}.h5; do echo \"$f\" > \"$f\"; done\n", - "popd\n", - "ls repo1" + "rm a*.h5\n", + "for f in b{0,0.75}_i{0,30,60,90,120}.h5; do echo \"$f\" > \"$f\"; done\n", + "echo \"Files after switching sync-demo to b files:\"\n", + "ls *.h5\n", + "popd\n" ] }, { "cell_type": "code", - "execution_count": 48, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1 ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", - "Changes to be committed\n", - " a0.75_i0.h5\n", - "a0.75_i120.h5\n", - " a0.75_i30.h5\n", - " a0.75_i60.h5\n", - " a0.75_i90.h5\n", - " a0_i0.h5\n", - " a0_i120.h5\n", - " a0_i30.h5\n", - " a0_i60.h5\n", - " a0_i90.h5\n", - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n" - ] + "outputs": [], + "source": [ + "pushd repo1\n", + "hallmark set-config --fmt \"b{a}_i{i}.h5\"\n", + "echo \"config.yml after hallmark set-config:\"\n", + "cat .hm/config.yml\n", + "popd\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "vscode": { + "languageId": "shellscript" } - ], + }, + "outputs": [], "source": [ - "pushd repo1 && hallmark add \"a{a}_i{i}.h5\" && popd" + "pushd repo1\n", + "rm b0.75_i{0,30,60,90,120}.h5\n", + "echo \"Files after deleting some tracked b files:\"\n", + "ls *.h5\n", + "popd\n" ] }, { - "cell_type": "markdown", - "metadata": {}, + "cell_type": "code", + "execution_count": null, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [], "source": [ - "## 3. Commit\n", - "\n", - "`hallmark commit` saves the current state. It copies file bytes into the\n", - "shared `objects/` store (keyed by sha1) and commits `data.tsv` in `.hm/`.\n" + "pushd repo1 && hallmark add . && popd\n" ] }, { "cell_type": "code", - "execution_count": 49, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1 ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", - "Committed staged state changes.\n", - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n" - ] + "outputs": [], + "source": [ + "echo \"Manifest after hallmark add dot:\"\n", + "cat repo1/.hm/data.tsv\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "vscode": { + "languageId": "shellscript" } - ], + }, + "outputs": [], "source": [ - "pushd repo1 && hallmark commit -m \"add spin and inclination files\" && popd" + "pushd repo1 && hallmark status && popd\n" ] }, { "cell_type": "code", - "execution_count": 50, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Objects stored:\n", - " 10\n" - ] + "outputs": [], + "source": [ + "pushd repo1 && hallmark commit -m \"sync manifest with hallmark add dot\" && popd\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [], + "source": [ + "pushd repo1 && hallmark checkout main && popd\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "vscode": { + "languageId": "shellscript" } - ], + }, + "outputs": [], "source": [ - "echo \"Objects stored:\"\n", - "find repo1/.hm/objects -type f | wc -l" + "echo \"Files after checkout back to main:\"\n", + "ls repo1/*.h5\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## 4. Checkout a new branch\n", + "## 5. Checkout a new branch\n", "\n", - "`hallmark checkout` works like `git checkout`. If the branch does not exist,\n", - "it creates it (like `git checkout -b`). The working directory is rewritten\n", - "in place using the target branch's committed `data.tsv`. Tracked files from\n", - "the old branch are removed, tracked files from the new branch are restored\n", - "from the shared `objects/` store, and unrelated untracked files are left alone\n", - "unless they block restoration.\n" + "The new branch inherits the same fmt from `main`.\n" ] }, { "cell_type": "code", - "execution_count": 51, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1 ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", - "Switched to branch \"experiment\".\n", - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n" - ] - } - ], + "outputs": [], "source": [ - "pushd repo1 && hallmark checkout experiment && popd" + "pushd repo1 && hallmark checkout experiment && popd\n" ] }, { "cell_type": "code", - "execution_count": 52, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Files after checkout:\n", - " \u001b[36mrepo1/\u001b[0ma0.75_i0.h5\n", - " \u001b[36mrepo1/\u001b[0ma0.75_i30.h5\n", - " \u001b[36mrepo1/\u001b[0ma0.75_i60.h5\n", - " \u001b[36mrepo1/\u001b[0ma0.75_i90.h5\n", - " \u001b[36mrepo1/\u001b[0ma0.75_i120.h5\n", - " \u001b[36mrepo1/\u001b[0ma0_i0.h5\n", - " \u001b[36mrepo1/\u001b[0ma0_i30.h5\n", - " \u001b[36mrepo1/\u001b[0ma0_i60.h5\n", - " \u001b[36mrepo1/\u001b[0ma0_i90.h5\n", - " \u001b[36mrepo1/\u001b[0ma0_i120.h5\n", - "\n", - "Current .hm branch:\n", - "experiment\n" - ] - } - ], + "outputs": [], "source": [ - "echo \"Files after checkout:\"\n", - "ls repo1/*.h5\n", - "echo \"\"\n", "echo \"Current .hm branch:\"\n", - "git -C repo1/.hm branch --show-current" + "git -C repo1/.hm branch --show-current\n", + "echo \"\"\n", + "echo \"config.yml on experiment:\"\n", + "cat repo1/.hm/config.yml\n", + "echo \"\"\n", + "echo \"Files after checkout:\"\n", + "ls repo1/*.h5\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Same files as before — `experiment` just branched off `main`." + "## 6. Replace the experiment branch dataset with new files that still match the same fmt\n" ] }, { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 5. Add new files on `experiment` and commit" - ] - }, - { - "cell_type": "code", - "execution_count": 53, - "metadata": { - "vscode": { - "languageId": "shellscript" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1 ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", - "Files after creating experiment files:\n", - " a0.75_i0.h5\n", - " a0.75_i30.h5\n", - " a0.75_i60.h5\n", - " a0.75_i90.h5\n", - " a0.75_i120.h5\n", - " a0_i0.h5\n", - " a0_i30.h5\n", - " a0_i60.h5\n", - " a0_i90.h5\n", - " a0_i120.h5\n", - " b0.5_i0.h5\n", - " b0.5_i45.h5\n", - " b0.5_i90.h5\n", - " b0.5_i135.h5\n", - " b0.5_i155.h5\n", - " b1.0_i0.h5\n", - " b1.0_i45.h5\n", - " b1.0_i90.h5\n", - " b1.0_i135.h5\n", - " b1.0_i155.h5\n", - "Changes to be committed\n", - " b0.5_i0.h5\n", - "b0.5_i135.h5\n", - "b0.5_i155.h5\n", - " b0.5_i45.h5\n", - " b0.5_i90.h5\n", - " b1.0_i0.h5\n", - "b1.0_i135.h5\n", - "b1.0_i155.h5\n", - " b1.0_i45.h5\n", - " b1.0_i90.h5\n", - "Committed staged state changes.\n", - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n" - ] - } - ], + "cell_type": "code", + "execution_count": null, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [], "source": [ "pushd repo1\n", - "for f in b{0.5,1.0}_i{0,45,90,135,155}.h5; do echo \"$f\" > \"$f\"; done\n", - "echo \"Files after creating experiment files:\"\n", + "rm a*.h5\n", + "for f in a{1,1.5}_i{15,45,75,105,110}.h5; do echo \"$f\" > \"$f\"; done\n", + "echo \"Files after replacing the dataset on experiment:\"\n", "ls *.h5\n", - "hallmark add \"b{spin}_i{inc}.h5\"\n", - "hallmark commit -m \"add experiment files\"\n", - "popd" + "popd\n" ] }, { "cell_type": "code", - "execution_count": 54, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Objects stored after experiment commit:\n", - " 20\n" - ] - } - ], + "outputs": [], "source": [ - "echo \"Objects stored after experiment commit:\"\n", - "find repo1/.hm/objects -type f | wc -l" + "pushd repo1 && hallmark add . && popd\n" ] }, { - "cell_type": "markdown", - "metadata": {}, + "cell_type": "code", + "execution_count": null, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [], "source": [ - "The objects store grew. It holds files from both `main` and `experiment`,\n", - "while each branch keeps its own committed `data.tsv` manifest.\n" + "pushd repo1 && hallmark status && popd\n" ] }, { - "cell_type": "markdown", - "metadata": {}, + "cell_type": "code", + "execution_count": null, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [], "source": [ - "## 6. Checkout back to `main`\n", - "\n", - "The experiment branch's tracked files are removed from the working directory and\n", - "main's tracked files are restored from `objects/`. No data is lost.\n" + "echo \"experiment data.tsv before commit:\"\n", + "cat repo1/.hm/data.tsv\n" ] }, { "cell_type": "code", - "execution_count": 55, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1 ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", - "Switched to branch \"main\".\n", - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n" - ] - } - ], + "outputs": [], "source": [ - "pushd repo1 && hallmark checkout main && popd" + "pushd repo1 && hallmark commit -m \"replace experiment dataset\" && popd\n" ] }, { "cell_type": "code", - "execution_count": 56, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Files after checkout main:\n", - " \u001b[36mrepo1/\u001b[0ma0.75_i0.h5\n", - " \u001b[36mrepo1/\u001b[0ma0.75_i30.h5\n", - " \u001b[36mrepo1/\u001b[0ma0.75_i60.h5\n", - " \u001b[36mrepo1/\u001b[0ma0.75_i90.h5\n", - " \u001b[36mrepo1/\u001b[0ma0.75_i120.h5\n", - " \u001b[36mrepo1/\u001b[0ma0_i0.h5\n", - " \u001b[36mrepo1/\u001b[0ma0_i30.h5\n", - " \u001b[36mrepo1/\u001b[0ma0_i60.h5\n", - " \u001b[36mrepo1/\u001b[0ma0_i90.h5\n", - " \u001b[36mrepo1/\u001b[0ma0_i120.h5\n", - "\n", - "Current .hm branch:\n", - "main\n", - "\n", - "Objects still in store (nothing lost):\n", - " 20\n" - ] - } - ], + "outputs": [], "source": [ - "echo \"Files after checkout main:\"\n", - "ls repo1/*.h5\n", - "echo \"\"\n", - "echo \"Current .hm branch:\"\n", - "git -C repo1/.hm branch --show-current\n", - "echo \"\"\n", - "echo \"Objects still in store (nothing lost):\"\n", - "find repo1/.hm/objects -type f | wc -l" + "echo \"Objects stored after experiment commit:\"\n", + "find repo1/.hm/objects -type f | wc -l\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Only main's files are in the working directory. The experiment files are safe\n", - "in `objects/` and will come back when you checkout `experiment` again." + "## 7. Checkout back to `main`\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [], + "source": [ + "pushd repo1 && hallmark checkout main && popd\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [], + "source": [ + "echo \"Files after checkout main:\"\n", + "ls repo1/*.h5\n", + "echo \"\"\n", + "echo \"main data.tsv:\"\n", + "cat repo1/.hm/data.tsv\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## 7. Checkout back to `experiment` to verify\n", - "\n", - "Switch back once more to confirm the branch-specific manifest is enough to\n", - "rebuild the worktree from the shared `objects/` store.\n" + "## 8. Checkout back to `experiment` to verify\n" ] }, { "cell_type": "code", - "execution_count": 57, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1 ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", - "Switched to branch \"experiment\".\n", - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n" - ] - } - ], + "outputs": [], "source": [ - "pushd repo1 && hallmark checkout experiment && popd" + "pushd repo1 && hallmark checkout experiment && popd\n" ] }, { "cell_type": "code", - "execution_count": 58, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Files after checkout experiment:\n", - " \u001b[36mrepo1/\u001b[0ma0.75_i0.h5\n", - " \u001b[36mrepo1/\u001b[0ma0.75_i30.h5\n", - " \u001b[36mrepo1/\u001b[0ma0.75_i60.h5\n", - " \u001b[36mrepo1/\u001b[0ma0.75_i90.h5\n", - " \u001b[36mrepo1/\u001b[0ma0.75_i120.h5\n", - " \u001b[36mrepo1/\u001b[0ma0_i0.h5\n", - " \u001b[36mrepo1/\u001b[0ma0_i30.h5\n", - " \u001b[36mrepo1/\u001b[0ma0_i60.h5\n", - " \u001b[36mrepo1/\u001b[0ma0_i90.h5\n", - " \u001b[36mrepo1/\u001b[0ma0_i120.h5\n", - " \u001b[36mrepo1/\u001b[0mb0.5_i0.h5\n", - " \u001b[36mrepo1/\u001b[0mb0.5_i45.h5\n", - " \u001b[36mrepo1/\u001b[0mb0.5_i90.h5\n", - " \u001b[36mrepo1/\u001b[0mb0.5_i135.h5\n", - " \u001b[36mrepo1/\u001b[0mb0.5_i155.h5\n", - " \u001b[36mrepo1/\u001b[0mb1.0_i0.h5\n", - " \u001b[36mrepo1/\u001b[0mb1.0_i45.h5\n", - " \u001b[36mrepo1/\u001b[0mb1.0_i90.h5\n", - " \u001b[36mrepo1/\u001b[0mb1.0_i135.h5\n", - " \u001b[36mrepo1/\u001b[0mb1.0_i155.h5\n", - "\n", - "Current .hm branch:\n", - "experiment\n" - ] - } - ], + "outputs": [], "source": [ "echo \"Files after checkout experiment:\"\n", "ls repo1/*.h5\n", "echo \"\"\n", - "echo \"Current .hm branch:\"\n", - "git -C repo1/.hm branch --show-current" + "echo \"experiment data.tsv:\"\n", + "cat repo1/.hm/data.tsv\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "All tracked files for `experiment` are back — the files inherited from `main`\n", - "plus the `b*` files committed on `experiment`.\n" + "## 9. Uncommitted changes block checkout\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [], + "source": [ + "pushd repo1\n", + "echo \"a2_i15.h5\" > a2_i15.h5\n", + "echo \"Files after creating a2_i15.h5:\"\n", + "ls *.h5\n", + "popd\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [], + "source": [ + "pushd repo1 && hallmark add . && popd\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## 8. Uncommitted changes block checkout\n", - "\n", - "Just like git, hallmark refuses to checkout if hallmark state has been staged\n", - "but not committed.\n" + "### Dirty status\n" ] }, { "cell_type": "code", - "execution_count": 59, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1 ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", - "Files after creating c0_i5.h5:\n", - " a0.75_i0.h5\n", - " a0.75_i30.h5\n", - " a0.75_i60.h5\n", - " a0.75_i90.h5\n", - " a0.75_i120.h5\n", - " a0_i0.h5\n", - " a0_i30.h5\n", - " a0_i60.h5\n", - " a0_i90.h5\n", - " a0_i120.h5\n", - " b0.5_i0.h5\n", - " b0.5_i45.h5\n", - " b0.5_i90.h5\n", - " b0.5_i135.h5\n", - " b0.5_i155.h5\n", - " b1.0_i0.h5\n", - " b1.0_i45.h5\n", - " b1.0_i90.h5\n", - " b1.0_i135.h5\n", - " b1.0_i155.h5\n", - " c0_i5.h5\n", - "Changes to be committed\n", - "c0_i5.h5\n", - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n" - ] - } - ], + "outputs": [], "source": [ - "pushd repo1\n", - "echo \"c0_i5.h5\" > c0_i5.h5\n", - "echo \"Files after creating c0_i5.h5:\"\n", - "ls *.h5\n", - "hallmark add \"c{c}_i{i}.h5\"\n", - "popd" + "pushd repo1 && hallmark status && popd\n" ] }, { "cell_type": "code", - "execution_count": 60, + "execution_count": null, "metadata": { "vscode": { "languageId": "shellscript" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo/repo1 ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n", - "Error: you have uncommitted hallmark state changes — commit them before checkout\n", - "checkout failed as expected\n", - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallmark/demo\n" - ] - } - ], + "outputs": [], "source": [ "pushd repo1\n", "if hallmark checkout main; then\n", @@ -744,14 +625,14 @@ "else\n", " echo \"checkout failed as expected\"\n", "fi\n", - "popd" + "popd\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "To proceed, either commit the staged hallmark state first or discard it.\n" + "To fix this, commit the staged changes first or discard them.\n" ] } ], diff --git a/demo/demo_index.ipynb b/demo/demo_index.ipynb index 8a89f03..ea70e90 100644 --- a/demo/demo_index.ipynb +++ b/demo/demo_index.ipynb @@ -4,23 +4,30 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Hallmark Demo: add, commit, and checkout\n", + "# Hallmark Demo: add, commit, checkout, and status\n", "\n", - "This notebook walks through the basic hallmark workflow:\n", + "This notebook shows the current Python and CLI workflow:\n", "\n", "1. Initialize a hallmark repo\n", - "2. Create data files, `add` them, and `commit`\n", - "3. `checkout` a new branch\n", - "4. Add different files on the new branch and commit\n", - "5. `checkout` back to `main` \u2014 files swap in place\n", - "6. Try to checkout with uncommitted changes \u2014 hallmark refuses" + "2. Create files and add them with one branch fmt\n", + "3. Commit and inspect `config.yml` and `data.tsv`\n", + "4. Use `repo.add(\".\")` to rebuild a branch manifest\n", + "5. Checkout between branches and restore files\n", + "6. See how staged changes block checkout\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Setup" + "## Setup\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Auto-reload modules when they change.\n" ] }, { @@ -29,9 +36,8 @@ "metadata": {}, "outputs": [], "source": [ - "# auto-reload modules when they change\n", "%load_ext autoreload\n", - "%autoreload 2" + "%autoreload 2\n" ] }, { @@ -42,7 +48,8 @@ "source": [ "%%bash\n", "rm -rf repo1\n", - "mkdir repo1" + "rm -rf repo2.hm\n", + "mkdir repo1\n" ] }, { @@ -51,10 +58,7 @@ "source": [ "## 1. Initialize the hallmark repo\n", "\n", - "This creates a `repo1/.hm/` directory. The `.hm/` is a git repo that tracks\n", - "`data.tsv` (the file manifest), `config.yml`, and `meta.yml`.\n", - "The shared `objects/` store is used for file bytes, but it is created lazily on\n", - "the first `commit()` that actually stores blobs.\n" + "A fresh repo starts with a commented `config.yml` template. Use it only if your branch fmt needs regex substitutions.\n" ] }, { @@ -65,7 +69,7 @@ "source": [ "from hallmark import Repo\n", "\n", - "repo = Repo.init(\"repo1\")" + "repo = Repo.init(\"repo1\")\n" ] }, { @@ -83,7 +87,19 @@ "meta.yml\n", "README.md\n", "\n", - "=== objects stored so far (0 before first commit) ===\n", + "=== initial config.yml ===\n", + "# Edit this file only if your branch needs regex substitutions or a preset remote.\n", + "# For simple names, you can just run: hallmark add \"a{a}_i{i}.h5\"\n", + "data:\n", + " -\n", + " # fmt: \"{release}_{source}_{year}_{doy:03d}_{band}.uvfits\"\n", + " encoding:\n", + " # aspin: m([0-9]+(\\.[0-9]+)?|\\.[0-9]+)\n", + "remote:\n", + " # name: origin\n", + " # url: https://example.com/path/to/data/\n", + "\n", + "=== objects stored so far ===\n", " 0\n" ] } @@ -93,7 +109,10 @@ "echo \"=== .hm directory ===\"\n", "ls repo1/.hm/\n", "echo \"\"\n", - "echo \"=== objects stored so far (0 before first commit) ===\"\n", + "echo \"=== initial config.yml ===\"\n", + "cat repo1/.hm/config.yml\n", + "echo \"\"\n", + "echo \"=== objects stored so far ===\"\n", "find repo1/.hm/objects -type f 2>/dev/null | wc -l\n" ] }, @@ -101,11 +120,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## 2. Create data files on `main`\n", - "\n", - "We create 10 `.h5` files with different spin and inclination values.\n", - "These are just dummy files for the demo \u2014 in a real project, these would be\n", - "simulation outputs or observational data." + "## 2. Create data files on `main`\n" ] }, { @@ -138,7 +153,7 @@ " echo \"$f\" > \"$f\"\n", "done\n", "echo \"Files in repo1/:\"\n", - "ls *.h5" + "ls *.h5\n" ] }, { @@ -147,16 +162,7 @@ "source": [ "## 3. Add files and commit on `main`\n", "\n", - "`repo.add()` does two things:\n", - "- Parses the format string to find matching files and extract parameters (spin, inc)\n", - "- Computes a sha1 checksum for each file and stages a manifest containing only `sha1` and `path` in `data.tsv`\n", - "\n", - "`repo.commit()` does two things:\n", - "- Copies each file's bytes into the shared `objects/` store, keyed by sha1\n", - "- Commits `data.tsv` in the `.hm/` git repo\n", - "\n", - "After commit, the file bytes are safe in `objects/`. Even if the files get\n", - "deleted from the working directory, they can be restored.\n" + "`repo.add(\"a{a}_i{i}.h5\")` sets the branch fmt and builds a manifest with `sha1`, `a`, and `i`.\n" ] }, { @@ -168,7 +174,7 @@ "name": "stdout", "output_type": "stream", "text": [ - " path spin inc\n", + " path a i\n", "0 a0.75_i0.h5 0.75 0.0\n", "1 a0.75_i120.h5 0.75 120.0\n", "2 a0.75_i30.h5 0.75 30.0\n", @@ -183,14 +189,53 @@ } ], "source": [ - "pf = repo.add(\"a{spin}_i{inc}.h5\")\n", - "print(pf)" + "pf = repo.add(\"a{a}_i{i}.h5\")\n", + "print(pf)\n" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "=== config.yml after add ===\n", + "data:\n", + "- fmt: a{a}_i{i}.h5\n", + " encoding: null\n", + "remote: null\n", + "\n", + "=== data.tsv after add ===\n", + "sha1\ta\ti\n", + "f656d419104e7af783c119186a4d81b15563310f\t0.75\t0\n", + "ec4dae9f0b8f69ac42f0c290fc84e0d6f1bec6cd\t0.75\t120\n", + "890b2ea509575be9bc74ad515731a2562cd7406e\t0.75\t30\n", + "d9cfcdd0b44d81c10f88e126d5b1eea4914259a0\t0.75\t60\n", + "ed2cfc6bb157a42729d3c6f45b228b432edec4be\t0.75\t90\n", + "18682593a012503b71935017dbc044665254a23c\t0\t0\n", + "1c057a4885fd04379ee1217375720665713c902f\t0\t120\n", + "829b87d845b367a9cc87df6ff510d1bdb7444aca\t0\t30\n", + "075a30e0a9c4dddd92447f463219d4c2a842883d\t0\t60\n", + "de40f4983d86342b03809bfcc09958f43f359b89\t0\t90\n" + ] + } + ], + "source": [ + "%%bash\n", + "echo \"=== config.yml after add ===\"\n", + "cat repo1/.hm/config.yml\n", + "echo \"\"\n", + "echo \"=== data.tsv after add ===\"\n", + "cat repo1/.hm/data.tsv\n" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, "outputs": [ { "data": { @@ -198,18 +243,18 @@ "True" ] }, - "execution_count": 7, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "repo.commit(\"add spin and inclination files\")" + "repo.commit(\"add main dataset\")\n" ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 9, "metadata": {}, "outputs": [ { @@ -224,40 +269,67 @@ "source": [ "%%bash\n", "echo \"Objects stored after commit:\"\n", - "find repo1/.hm/objects -type f | wc -l" + "find repo1/.hm/objects -type f | wc -l\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "10 files added, 10 objects stored. Each object is a copy of a file's bytes,\n", - "named by its sha1 hash (split as `sha1[:2]/sha1[2:]`, same as git)." + "### Clean status\n" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "On branch main\n", + "\n", + "Changes to be committed:\n", + "\n", + "Changes not staged for commit:\n", + "\n", + "nothing to commit, working tree clean\n" + ] + } + ], + "source": [ + "%%bash\n", + "cd repo1\n", + "hallmark status\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## 4. Checkout a new branch\n", + "### Current add modes\n", "\n", - "This works like `git checkout -b experiment`. The `.hm/` repo switches to a new\n", - "branch called `experiment`. Since it just branched off `main`, the files are\n", - "the same \u2014 `data.tsv` is identical, and the same files are restored from `objects/`.\n", - "\n", - "What happens under the hood:\n", - "1. Validate that tracked files in the working directory still match the current branch's `data.tsv`\n", - "2. Delete current tracked files from the working directory\n", - "3. Run `git checkout -b experiment` inside `.hm/`\n", - "4. Read the new branch's `data.tsv`\n", - "5. Restore each tracked file from `objects/` using hardlinks\n", + "- `repo.add(\"a{a}_i{i}.h5\")` uses a format string and can still set the branch fmt\n", + "- `repo.set_config(fmt=\"b{a}_i{i}.h5\")` is the explicit way to change branch config\n", + "- `repo.add(\".\")` rebuilds the manifest from current files that match the branch fmt\n", + "- explicit file-list adds such as `hallmark add *` are not supported yet with the parameter-based manifest\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4. Demo: `repo.add(\".\")` removes deleted files from the manifest\n", "\n", - "Untracked files are left alone unless they block restoration of a tracked path.\n" + "We use a temporary branch so the main workflow stays intact.\n", + "On this branch we switch to `b*` files first so it is easy to distinguish from `main`.\n", + "Because the filename family changes, we update the branch fmt explicitly with `repo.set_config(...)` before using `repo.add(\".\")`.\n" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 11, "metadata": {}, "outputs": [ { @@ -266,142 +338,270 @@ "True" ] }, - "execution_count": 9, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "repo.checkout(\"experiment\")" + "repo.checkout(\"sync-demo\")\n" ] }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 12, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Files in repo1/ after checkout:\n", - "repo1/a0_i0.h5\n", - "repo1/a0_i120.h5\n", - "repo1/a0_i30.h5\n", - "repo1/a0_i60.h5\n", - "repo1/a0_i90.h5\n", - "repo1/a0.75_i0.h5\n", - "repo1/a0.75_i120.h5\n", - "repo1/a0.75_i30.h5\n", - "repo1/a0.75_i60.h5\n", - "repo1/a0.75_i90.h5\n", - "\n", - "Current .hm branch:\n", - "experiment\n" + "Files after switching sync-demo to b files:\n", + "b0_i0.h5\n", + "b0_i120.h5\n", + "b0_i30.h5\n", + "b0_i60.h5\n", + "b0_i90.h5\n", + "b0.75_i0.h5\n", + "b0.75_i120.h5\n", + "b0.75_i30.h5\n", + "b0.75_i60.h5\n", + "b0.75_i90.h5\n" ] } ], "source": [ "%%bash\n", - "echo \"Files in repo1/ after checkout:\"\n", - "ls repo1/*.h5\n", - "echo \"\"\n", - "echo \"Current .hm branch:\"\n", - "git -C repo1/.hm branch --show-current" + "cd repo1\n", + "rm a*.h5\n", + "for f in b{0,0.75}_i{0,30,60,90,120}.h5; do\n", + " echo \"$f\" > \"$f\"\n", + "done\n", + "echo \"Files after switching sync-demo to b files:\"\n", + "ls *.h5\n" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": 13, "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'data': [{'fmt': 'b{a}_i{i}.h5', 'encoding': None}], 'remote': None}" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "Same files as before \u2014 because `experiment` just branched off `main`." + "repo.set_config(fmt=\"b{a}_i{i}.h5\")\n" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": 14, "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "config.yml after repo.set_config:\n", + "data:\n", + "- fmt: b{a}_i{i}.h5\n", + " encoding: null\n", + "remote: null\n" + ] + } + ], "source": [ - "## 5. Add new files on `experiment` and commit\n", - "\n", - "Now we create different files on the `experiment` branch and track them.\n", - "After commit, `objects/` grows \u2014 it now holds files from both branches, while\n", - "`data.tsv` on each branch still lists only that branch's manifest.\n" + "%%bash\n", + "cd repo1\n", + "echo \"config.yml after repo.set_config:\"\n", + "cat .hm/config.yml\n" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 15, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "All files in repo1/ now:\n", - "a0_i0.h5\n", - "a0_i120.h5\n", - "a0_i30.h5\n", - "a0_i60.h5\n", - "a0_i90.h5\n", - "a0.75_i0.h5\n", - "a0.75_i120.h5\n", - "a0.75_i30.h5\n", - "a0.75_i60.h5\n", - "a0.75_i90.h5\n", - "b0.5_i0.h5\n", - "b0.5_i135.h5\n", - "b0.5_i155.h5\n", - "b0.5_i45.h5\n", - "b0.5_i90.h5\n", - "b1.0_i0.h5\n", - "b1.0_i135.h5\n", - "b1.0_i155.h5\n", - "b1.0_i45.h5\n", - "b1.0_i90.h5\n" + "Files after deleting some tracked b files:\n", + "b0_i0.h5\n", + "b0_i120.h5\n", + "b0_i30.h5\n", + "b0_i60.h5\n", + "b0_i90.h5\n" ] } ], "source": [ "%%bash\n", "cd repo1\n", - "for f in b{0.5,1.0}_i{0,45,90,135,155}.h5; do\n", - " echo \"$f\" > \"$f\"\n", - "done\n", - "echo \"All files in repo1/ now:\"\n", - "ls *.h5" + "rm b0.75_i{0,30,60,90,120}.h5\n", + "echo \"Files after deleting some tracked b files:\"\n", + "ls *.h5\n" ] }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
pathai
0b0_i0.h50.00.0
1b0_i120.h50.0120.0
2b0_i30.h50.030.0
3b0_i60.h50.060.0
4b0_i90.h50.090.0
\n", + "
" + ], + "text/plain": [ + " path a i\n", + "0 b0_i0.h5 0.0 0.0\n", + "1 b0_i120.h5 0.0 120.0\n", + "2 b0_i30.h5 0.0 30.0\n", + "3 b0_i60.h5 0.0 60.0\n", + "4 b0_i90.h5 0.0 90.0" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "repo.add(\".\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - " path spin inc\n", - "0 b0.5_i0.h5 0.5 0.0\n", - "1 b0.5_i135.h5 0.5 135.0\n", - "2 b0.5_i155.h5 0.5 155.0\n", - "3 b0.5_i45.h5 0.5 45.0\n", - "4 b0.5_i90.h5 0.5 90.0\n", - "5 b1.0_i0.h5 1.0 0.0\n", - "6 b1.0_i135.h5 1.0 135.0\n", - "7 b1.0_i155.h5 1.0 155.0\n", - "8 b1.0_i45.h5 1.0 45.0\n", - "9 b1.0_i90.h5 1.0 90.0\n" + "Manifest after repo.add dot:\n", + "sha1\ta\ti\n", + "56676e3e5629923617f8c9e542bd1c2ccaf8fa3a\t0\t0\n", + "683e82f624b175c075ac3ea0cca56e94cd7422f8\t0\t120\n", + "cfdbae05e3afc29354d948a32263f7ac1ed5850b\t0\t30\n", + "3a947fb1280b5bca62615d6c7543b88080588e7a\t0\t60\n", + "1711d3b47eac6ae479b6f0f240f83f41b393ebb4\t0\t90\n" ] } ], "source": [ - "pf_exp = repo.add(\"b{spin}_i{inc}.h5\")\n", - "print(pf_exp)" + "%%bash\n", + "cd repo1\n", + "echo \"Manifest after repo.add dot:\"\n", + "cat .hm/data.tsv\n" ] }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "On branch sync-demo\n", + "\n", + "Changes to be committed:\n", + " new file: b0_i0.h5\n", + " new file: b0_i120.h5\n", + " new file: b0_i30.h5\n", + " new file: b0_i60.h5\n", + " new file: b0_i90.h5\n", + " deleted: a0.75_i0.h5\n", + " deleted: a0.75_i120.h5\n", + " deleted: a0.75_i30.h5\n", + " deleted: a0.75_i60.h5\n", + " deleted: a0.75_i90.h5\n", + " deleted: a0_i0.h5\n", + " deleted: a0_i120.h5\n", + " deleted: a0_i30.h5\n", + " deleted: a0_i60.h5\n", + " deleted: a0_i90.h5\n", + "\n", + "Changes not staged for commit:\n" + ] + } + ], + "source": [ + "%%bash\n", + "cd repo1\n", + "hallmark status\n" + ] + }, + { + "cell_type": "code", + "execution_count": 19, "metadata": {}, "outputs": [ { @@ -410,62 +610,76 @@ "True" ] }, - "execution_count": 13, + "execution_count": 19, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "repo.commit(\"add experiment files\")" + "repo.commit(\"sync manifest with repo.add dot\")\n" ] }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 20, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "Objects stored after experiment commit:\n", - " 20\n" - ] + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "%%bash\n", - "echo \"Objects stored after experiment commit:\"\n", - "find repo1/.hm/objects -type f | wc -l" + "repo.checkout(\"main\")\n" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": 21, "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Files after checkout back to main:\n", + "repo1/a0_i0.h5\n", + "repo1/a0_i120.h5\n", + "repo1/a0_i30.h5\n", + "repo1/a0_i60.h5\n", + "repo1/a0_i90.h5\n", + "repo1/a0.75_i0.h5\n", + "repo1/a0.75_i120.h5\n", + "repo1/a0.75_i30.h5\n", + "repo1/a0.75_i60.h5\n", + "repo1/a0.75_i90.h5\n" + ] + } + ], "source": [ - "20 objects total \u2014 10 from `main` + 10 new from `experiment`.\n", - "The objects store holds everything across all branches, while each branch keeps\n", - "its own committed `data.tsv`.\n" + "%%bash\n", + "echo \"Files after checkout back to main:\"\n", + "ls repo1/*.h5\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## 6. Checkout back to `main`\n", - "\n", - "This is where the magic happens. The working directory gets rewritten in place:\n", - "1. All of experiment's tracked files get deleted from the working directory\n", - "2. `.hm/` switches to the `main` branch\n", - "3. Main's files get restored from `objects/`\n", + "## 5. Checkout a new branch\n", "\n", - "No data is lost \u2014 experiment's file bytes are still in `objects/`, but `main`\n", - "is rebuilt only from `main`'s `data.tsv`.\n" + "The new branch inherits the same fmt from `main`.\n" ] }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 22, "metadata": {}, "outputs": [ { @@ -474,25 +688,34 @@ "True" ] }, - "execution_count": 15, + "execution_count": 22, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "repo.checkout(\"main\")" + "repo.checkout(\"experiment\")\n" ] }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 23, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Files in repo1/ after checkout main:\n", + "Current .hm branch:\n", + "experiment\n", + "\n", + "config.yml on experiment:\n", + "data:\n", + "- fmt: a{a}_i{i}.h5\n", + " encoding: null\n", + "remote: null\n", + "\n", + "Files after checkout:\n", "repo1/a0_i0.h5\n", "repo1/a0_i120.h5\n", "repo1/a0_i30.h5\n", @@ -502,49 +725,169 @@ "repo1/a0.75_i120.h5\n", "repo1/a0.75_i30.h5\n", "repo1/a0.75_i60.h5\n", - "repo1/a0.75_i90.h5\n", - "\n", - "Current .hm branch:\n", - "main\n", - "\n", - "Objects still in store (nothing lost):\n", - " 20\n" + "repo1/a0.75_i90.h5\n" ] } ], "source": [ "%%bash\n", - "echo \"Files in repo1/ after checkout main:\"\n", - "ls repo1/*.h5\n", - "echo \"\"\n", "echo \"Current .hm branch:\"\n", "git -C repo1/.hm branch --show-current\n", "echo \"\"\n", - "echo \"Objects still in store (nothing lost):\"\n", - "find repo1/.hm/objects -type f | wc -l" + "echo \"config.yml on experiment:\"\n", + "cat repo1/.hm/config.yml\n", + "echo \"\"\n", + "echo \"Files after checkout:\"\n", + "ls repo1/*.h5\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Only main's 10 `a*` files are in the working directory. The `b*` files are gone\n", - "from the directory but their bytes are safe in `objects/`. Checkout back to\n", - "`experiment` and they come right back.\n" + "## 6. Replace the experiment branch dataset with new files that still match the same fmt\n" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": 24, "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Files after replacing the dataset on experiment:\n", + "a1_i105.h5\n", + "a1_i110.h5\n", + "a1_i15.h5\n", + "a1_i45.h5\n", + "a1_i75.h5\n", + "a1.5_i105.h5\n", + "a1.5_i110.h5\n", + "a1.5_i15.h5\n", + "a1.5_i45.h5\n", + "a1.5_i75.h5\n" + ] + } + ], + "source": [ + "%%bash\n", + "cd repo1\n", + "rm a*.h5\n", + "for f in a{1,1.5}_i{15,45,75,105,110}.h5; do\n", + " echo \"$f\" > \"$f\"\n", + "done\n", + "echo \"Files after replacing the dataset on experiment:\"\n", + "ls *.h5\n" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " path a i\n", + "0 a1.5_i105.h5 1.5 105.0\n", + "1 a1.5_i110.h5 1.5 110.0\n", + "2 a1.5_i15.h5 1.5 15.0\n", + "3 a1.5_i45.h5 1.5 45.0\n", + "4 a1.5_i75.h5 1.5 75.0\n", + "5 a1_i105.h5 1.0 105.0\n", + "6 a1_i110.h5 1.0 110.0\n", + "7 a1_i15.h5 1.0 15.0\n", + "8 a1_i45.h5 1.0 45.0\n", + "9 a1_i75.h5 1.0 75.0\n" + ] + } + ], "source": [ - "## 7. Checkout back to `experiment` to verify\n", + "pf_exp = repo.add(\".\")\n", "\n", - "Let's go back to `experiment` to confirm the files are restored correctly." + "print(pf_exp)\n", + "\n" ] }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "On branch experiment\n", + "\n", + "Changes to be committed:\n", + " new file: a1.5_i105.h5\n", + " new file: a1.5_i110.h5\n", + " new file: a1.5_i15.h5\n", + " new file: a1.5_i45.h5\n", + " new file: a1.5_i75.h5\n", + " new file: a1_i105.h5\n", + " new file: a1_i110.h5\n", + " new file: a1_i15.h5\n", + " new file: a1_i45.h5\n", + " new file: a1_i75.h5\n", + " deleted: a0.75_i0.h5\n", + " deleted: a0.75_i120.h5\n", + " deleted: a0.75_i30.h5\n", + " deleted: a0.75_i60.h5\n", + " deleted: a0.75_i90.h5\n", + " deleted: a0_i0.h5\n", + " deleted: a0_i120.h5\n", + " deleted: a0_i30.h5\n", + " deleted: a0_i60.h5\n", + " deleted: a0_i90.h5\n", + "\n", + "Changes not staged for commit:\n" + ] + } + ], + "source": [ + "%%bash\n", + "cd repo1\n", + "hallmark status" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "experiment data.tsv before commit:\n", + "sha1\ta\ti\n", + "f0ca3ed468c8796645672d2ebb48d48ae49b6d3c\t1.5\t105\n", + "9d1801d46e071b98a87afcd1edc59fe9302b9a4b\t1.5\t110\n", + "ed34eda23a05ea4c3cd1bf41988c6e2f76cf41e1\t1.5\t15\n", + "b47e5f555daf30afea5b0f99cadad1e2d8c689ec\t1.5\t45\n", + "b2a40517fd5681088f88903165ae54af3c75cf25\t1.5\t75\n", + "635b4b44a00c0a2bde5a7f0e263bb98dcbdae6e2\t1\t105\n", + "24ef5885eff61f1a8fcd77c19329a2651eca0228\t1\t110\n", + "577954873e813a982f70867b49a374a230df3650\t1\t15\n", + "1580545be2a5d485338e3538c7b16709d5fa6e4a\t1\t45\n", + "e2406e4872637ef09f574731be5b5ba5e8aa5761\t1\t75\n" + ] + } + ], + "source": [ + "%%bash\n", + "echo \"experiment data.tsv before commit:\"\n", + "cat repo1/.hm/data.tsv\n" + ] + }, + { + "cell_type": "code", + "execution_count": 28, "metadata": {}, "outputs": [ { @@ -553,25 +896,72 @@ "True" ] }, - "execution_count": 17, + "execution_count": 28, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "repo.checkout(\"experiment\")\n" + "repo.commit(\"replace experiment dataset\")\n" ] }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Objects stored after experiment commit:\n", + " 25\n" + ] + } + ], + "source": [ + "%%bash\n", + "echo \"Objects stored after experiment commit:\"\n", + "find repo1/.hm/objects -type f | wc -l\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 7. Checkout back to `main`\n" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "repo.checkout(\"main\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 31, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Files in repo1/ after checkout experiment:\n", + "Files after checkout main:\n", "repo1/a0_i0.h5\n", "repo1/a0_i120.h5\n", "repo1/a0_i30.h5\n", @@ -582,65 +972,145 @@ "repo1/a0.75_i30.h5\n", "repo1/a0.75_i60.h5\n", "repo1/a0.75_i90.h5\n", - "repo1/b0.5_i0.h5\n", - "repo1/b0.5_i135.h5\n", - "repo1/b0.5_i155.h5\n", - "repo1/b0.5_i45.h5\n", - "repo1/b0.5_i90.h5\n", - "repo1/b1.0_i0.h5\n", - "repo1/b1.0_i135.h5\n", - "repo1/b1.0_i155.h5\n", - "repo1/b1.0_i45.h5\n", - "repo1/b1.0_i90.h5\n", "\n", - "Current .hm branch:\n", - "experiment\n" + "main data.tsv:\n", + "sha1\ta\ti\n", + "f656d419104e7af783c119186a4d81b15563310f\t0.75\t0\n", + "ec4dae9f0b8f69ac42f0c290fc84e0d6f1bec6cd\t0.75\t120\n", + "890b2ea509575be9bc74ad515731a2562cd7406e\t0.75\t30\n", + "d9cfcdd0b44d81c10f88e126d5b1eea4914259a0\t0.75\t60\n", + "ed2cfc6bb157a42729d3c6f45b228b432edec4be\t0.75\t90\n", + "18682593a012503b71935017dbc044665254a23c\t0\t0\n", + "1c057a4885fd04379ee1217375720665713c902f\t0\t120\n", + "829b87d845b367a9cc87df6ff510d1bdb7444aca\t0\t30\n", + "075a30e0a9c4dddd92447f463219d4c2a842883d\t0\t60\n", + "de40f4983d86342b03809bfcc09958f43f359b89\t0\t90\n" ] } ], "source": [ "%%bash\n", - "echo \"Files in repo1/ after checkout experiment:\"\n", + "echo \"Files after checkout main:\"\n", "ls repo1/*.h5\n", "echo \"\"\n", - "echo \"Current .hm branch:\"\n", - "git -C repo1/.hm branch --show-current" + "echo \"main data.tsv:\"\n", + "cat repo1/.hm/data.tsv\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "All 20 files are back \u2014 the 10 from `main` (inherited when we branched) plus\n", - "the 10 we added on `experiment`." + "## 8. Checkout back to `experiment` to verify\n" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 32, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "repo.checkout(\"experiment\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Files after checkout experiment:\n", + "repo1/a1_i105.h5\n", + "repo1/a1_i110.h5\n", + "repo1/a1_i15.h5\n", + "repo1/a1_i45.h5\n", + "repo1/a1_i75.h5\n", + "repo1/a1.5_i105.h5\n", + "repo1/a1.5_i110.h5\n", + "repo1/a1.5_i15.h5\n", + "repo1/a1.5_i45.h5\n", + "repo1/a1.5_i75.h5\n", + "\n", + "experiment data.tsv:\n", + "sha1\ta\ti\n", + "f0ca3ed468c8796645672d2ebb48d48ae49b6d3c\t1.5\t105\n", + "9d1801d46e071b98a87afcd1edc59fe9302b9a4b\t1.5\t110\n", + "ed34eda23a05ea4c3cd1bf41988c6e2f76cf41e1\t1.5\t15\n", + "b47e5f555daf30afea5b0f99cadad1e2d8c689ec\t1.5\t45\n", + "b2a40517fd5681088f88903165ae54af3c75cf25\t1.5\t75\n", + "635b4b44a00c0a2bde5a7f0e263bb98dcbdae6e2\t1\t105\n", + "24ef5885eff61f1a8fcd77c19329a2651eca0228\t1\t110\n", + "577954873e813a982f70867b49a374a230df3650\t1\t15\n", + "1580545be2a5d485338e3538c7b16709d5fa6e4a\t1\t45\n", + "e2406e4872637ef09f574731be5b5ba5e8aa5761\t1\t75\n" + ] + } + ], + "source": [ + "%%bash\n", + "echo \"Files after checkout experiment:\"\n", + "ls repo1/*.h5\n", + "echo \"\"\n", + "echo \"experiment data.tsv:\"\n", + "cat repo1/.hm/data.tsv\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## 8. Uncommitted changes block checkout\n", - "\n", - "Just like git, hallmark will not let you checkout if tracked files have local\n", - "changes or if hallmark state has been added but not committed.\n" + "## 9. Uncommitted changes block checkout\n" ] }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 34, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Files after creating a2_i15.h5:\n", + "a1_i105.h5\n", + "a1_i110.h5\n", + "a1_i15.h5\n", + "a1_i45.h5\n", + "a1_i75.h5\n", + "a1.5_i105.h5\n", + "a1.5_i110.h5\n", + "a1.5_i15.h5\n", + "a1.5_i45.h5\n", + "a1.5_i75.h5\n", + "a2_i15.h5\n" + ] + } + ], "source": [ "%%bash\n", "cd repo1\n", - "echo \"c0_i20.h5\" > c0_i20.h5\n", - "echo \"Files in repo1/ after creating c0_i20.h5:\"\n", - "ls *.h5" + "echo \"a2_i15.h5\" > a2_i15.h5\n", + "echo \"Files after creating a2_i15.h5:\"\n", + "ls *.h5\n" ] }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 35, "metadata": {}, "outputs": [ { @@ -665,73 +1135,181 @@ " \n", " \n", " path\n", - " c\n", + " a\n", " i\n", " \n", " \n", " \n", " \n", " 0\n", - " c0_i20.h5\n", - " 0.0\n", - " 20.0\n", + " a1.5_i105.h5\n", + " 1.5\n", + " 105.0\n", + " \n", + " \n", + " 1\n", + " a1.5_i110.h5\n", + " 1.5\n", + " 110.0\n", + " \n", + " \n", + " 2\n", + " a1.5_i15.h5\n", + " 1.5\n", + " 15.0\n", + " \n", + " \n", + " 3\n", + " a1.5_i45.h5\n", + " 1.5\n", + " 45.0\n", + " \n", + " \n", + " 4\n", + " a1.5_i75.h5\n", + " 1.5\n", + " 75.0\n", + " \n", + " \n", + " 5\n", + " a1_i105.h5\n", + " 1.0\n", + " 105.0\n", + " \n", + " \n", + " 6\n", + " a1_i110.h5\n", + " 1.0\n", + " 110.0\n", + " \n", + " \n", + " 7\n", + " a1_i15.h5\n", + " 1.0\n", + " 15.0\n", + " \n", + " \n", + " 8\n", + " a1_i45.h5\n", + " 1.0\n", + " 45.0\n", + " \n", + " \n", + " 9\n", + " a1_i75.h5\n", + " 1.0\n", + " 75.0\n", + " \n", + " \n", + " 10\n", + " a2_i15.h5\n", + " 2.0\n", + " 15.0\n", " \n", " \n", "\n", "" ], "text/plain": [ - " path c i\n", - "0 c0_i20.h5 0.0 20.0" + " path a i\n", + "0 a1.5_i105.h5 1.5 105.0\n", + "1 a1.5_i110.h5 1.5 110.0\n", + "2 a1.5_i15.h5 1.5 15.0\n", + "3 a1.5_i45.h5 1.5 45.0\n", + "4 a1.5_i75.h5 1.5 75.0\n", + "5 a1_i105.h5 1.0 105.0\n", + "6 a1_i110.h5 1.0 110.0\n", + "7 a1_i15.h5 1.0 15.0\n", + "8 a1_i45.h5 1.0 45.0\n", + "9 a1_i75.h5 1.0 75.0\n", + "10 a2_i15.h5 2.0 15.0" ] }, - "execution_count": 23, + "execution_count": 35, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "repo.add(\"c{c}_i{i}.h5\")" + "repo.add(\".\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Dirty status\n" ] }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 36, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "On branch experiment\n", + "\n", + "Changes to be committed:\n", + " new file: a2_i15.h5\n", + "\n", + "Changes not staged for commit:\n" + ] + } + ], + "source": [ + "%%bash\n", + "cd repo1\n", + "hallmark status\n" + ] + }, + { + "cell_type": "code", + "execution_count": 37, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Error: you have uncommitted hallmark state changes \u2014 commit them before checkout\n" + "Error: you have uncommitted hallmark state changes — commit them before checkout\n" ] } ], "source": [ - "# this should fail because we have uncommitted changes\n", "try:\n", " repo.checkout(\"main\")\n", "except RuntimeError as e:\n", - " print(f\"Error: {e}\")" + " print(f\"Error: {e}\")\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "To fix this, either commit the changes first, or discard them." + "To fix this, commit the staged changes first or discard them.\n" ] } ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "hallmark", "language": "python", "name": "python3" }, "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", "name": "python", - "version": "3.11.0" + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.12" } }, "nbformat": 4, diff --git a/demo/demo_python.ipynb b/demo/demo_python.ipynb index c961e5e..1033b18 100644 --- a/demo/demo_python.ipynb +++ b/demo/demo_python.ipynb @@ -2,2387 +2,473 @@ "cells": [ { "cell_type": "markdown", - "id": "md-title", "metadata": {}, "source": [ "# Hallmark Python Demo\n", "\n", - "Hallmark helps you keep track of data files in scientific workflows.\n", - "It stores a file index inside a small git repository (`.hm`) that lives\n", - "next to your data, so you always know which files belong to which run.\n", - "\n", - "This notebook walks through the full workflow:\n", - "1. Create a hallmark repository\n", - "2. Add files and commit\n", - "3. Discover and filter files with `ParaFrame`\n", - "4. Handle custom filename conventions with encoding" + "This notebook shows the current Python API with the new config and manifest model.\n", + "Each branch has one dataset fmt in `config.yml`, and `data.tsv` stores `sha1` plus fmt parameters.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "auto-reload modules when they change\n" ] }, { "cell_type": "code", - "execution_count": 1, - "id": "code-setup", - "metadata": { - "execution": { - "iopub.execute_input": "2026-04-02T23:09:18.373603Z", - "iopub.status.busy": "2026-04-02T23:09:18.373314Z", - "iopub.status.idle": "2026-04-02T23:09:19.238495Z", - "shell.execute_reply": "2026-04-02T23:09:19.237979Z" - } - }, + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ - "# auto-reload modules when they change\n", "%load_ext autoreload\n", "%autoreload 2\n", "\n", - "from hallmark import Repo, ParaFrame" + "from hallmark import Repo, ParaFrame\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "remove any repositories left over from a previous run\n" ] }, { "cell_type": "code", - "execution_count": 2, - "id": "code-cleanup", - "metadata": { - "execution": { - "iopub.execute_input": "2026-04-02T23:09:19.240387Z", - "iopub.status.busy": "2026-04-02T23:09:19.240145Z", - "iopub.status.idle": "2026-04-02T23:09:19.542023Z", - "shell.execute_reply": "2026-04-02T23:09:19.541351Z" - } - }, + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ - "# remove any repositories left over from a previous run\n", - "! rm -rf repo*" + "! rm -rf repo*\n" ] }, { "cell_type": "markdown", - "id": "md-s1", "metadata": {}, "source": [ - "## 1. Create a Repository\n", - "\n", - "`Repo.init(path)` creates a new hallmark repository at `path`.\n", - "It works like `git init`: the directory is created with a `.hm` folder\n", - "inside, which is itself a git repository used to store the file manifest.\n", - "The shared `objects/` store is created lazily on the first commit that stores\n", - "file bytes.\n" + "## 1. Create a Repository\n" ] }, { "cell_type": "code", - "execution_count": 3, - "id": "code-init", - "metadata": { - "execution": { - "iopub.execute_input": "2026-04-02T23:09:19.543922Z", - "iopub.status.busy": "2026-04-02T23:09:19.543803Z", - "iopub.status.idle": "2026-04-02T23:09:19.643693Z", - "shell.execute_reply": "2026-04-02T23:09:19.643162Z" - } - }, + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ - "repo1 = Repo.init(\"repo1\")" + "repo1 = Repo.init(\"repo1\")\n" ] }, { - "cell_type": "code", - "execution_count": 4, - "id": "code-ls-repo", - "metadata": { - "execution": { - "iopub.execute_input": "2026-04-02T23:09:19.645744Z", - "iopub.status.busy": "2026-04-02T23:09:19.645621Z", - "iopub.status.idle": "2026-04-02T23:09:19.911367Z", - "shell.execute_reply": "2026-04-02T23:09:19.910878Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "repo1:\r\n", - "\u001b[1m\u001b[36m.\u001b[m\u001b[m \u001b[1m\u001b[36m..\u001b[m\u001b[m \u001b[1m\u001b[36m.hm\u001b[m\u001b[m\r\n", - "\r\n", - "repo1/.hm:\r\n", - "\u001b[1m\u001b[36m.\u001b[m\u001b[m \u001b[1m\u001b[36m..\u001b[m\u001b[m \u001b[1m\u001b[36m.git\u001b[m\u001b[m config.yml data.tsv meta.yml README.md\r\n" - ] - } - ], + "cell_type": "markdown", + "metadata": {}, "source": [ - "# repo1/ holds your data files; repo1/.hm/ holds the hallmark index\n", - "! ls -a repo1 repo1/.hm" + "repo1/ holds your data files; repo1/.hm/ holds the hallmark index\n" ] }, { "cell_type": "code", - "execution_count": 5, - "id": "code-git-status", - "metadata": { - "execution": { - "iopub.execute_input": "2026-04-02T23:09:19.913428Z", - "iopub.status.busy": "2026-04-02T23:09:19.913306Z", - "iopub.status.idle": "2026-04-02T23:09:20.192212Z", - "shell.execute_reply": "2026-04-02T23:09:20.191537Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "On branch master\r\n", - "Changes to be committed:\r\n", - " (use \"git restore --staged ...\" to unstage)\r\n", - "\t\u001b[32mnew file: config.yml\u001b[m\r\n", - "\t\u001b[32mnew file: data.tsv\u001b[m\r\n", - "\t\u001b[32mnew file: meta.yml\u001b[m\r\n", - "\r\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ - "# the .hm directory is a git repo \u2014 you can run git commands inside it\n", - "! pushd repo1/.hm && git status && popd" + "! ls -a repo1 repo1/.hm\n" ] }, { "cell_type": "markdown", - "id": "md-s1-note", "metadata": {}, "source": [ - "The `.hm` repository already has an initial README commit. The hallmark state\n", - "files (`config.yml`, `data.tsv`, `meta.yml`) are then staged in the working tree\n", - "so the next `Repo.commit(...)` captures the first manifest snapshot.\n" + "A fresh repo starts with a commented config template. Use it only if the branch fmt needs regex substitutions.\n" ] }, { - "cell_type": "markdown", - "id": "md-bare", + "cell_type": "code", + "execution_count": null, "metadata": {}, + "outputs": [], "source": [ - "### Bare repository\n", - "\n", - "A bare repository has no data directory \u2014 it is just the index.\n", - "Name the path with a `.hm` suffix to create one." + "! cat repo1/.hm/config.yml\n" ] }, { - "cell_type": "code", - "execution_count": 6, - "id": "code-bare-init", - "metadata": { - "execution": { - "iopub.execute_input": "2026-04-02T23:09:20.194165Z", - "iopub.status.busy": "2026-04-02T23:09:20.194035Z", - "iopub.status.idle": "2026-04-02T23:09:20.263443Z", - "shell.execute_reply": "2026-04-02T23:09:20.262965Z" - } - }, - "outputs": [], + "cell_type": "markdown", + "metadata": {}, "source": [ - "repo2 = Repo.init(\"repo2.hm\")" + "### Bare repository\n" ] }, { "cell_type": "code", - "execution_count": 7, - "id": "code-bare-ls", - "metadata": { - "execution": { - "iopub.execute_input": "2026-04-02T23:09:20.265247Z", - "iopub.status.busy": "2026-04-02T23:09:20.265126Z", - "iopub.status.idle": "2026-04-02T23:09:20.530462Z", - "shell.execute_reply": "2026-04-02T23:09:20.529938Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[1m\u001b[36m.\u001b[m\u001b[m \u001b[1m\u001b[36m..\u001b[m\u001b[m \u001b[1m\u001b[36m.git\u001b[m\u001b[m config.yml data.tsv meta.yml README.md\r\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ - "# repo2.hm/ is the index itself \u2014 there is no separate data folder\n", - "! ls -a repo2.hm" + "repo2 = Repo.init(\"repo2.hm\")\n" ] }, { "cell_type": "code", - "execution_count": 8, - "id": "code-bare-git-status", - "metadata": { - "execution": { - "iopub.execute_input": "2026-04-02T23:09:20.532801Z", - "iopub.status.busy": "2026-04-02T23:09:20.532654Z", - "iopub.status.idle": "2026-04-02T23:09:20.819552Z", - "shell.execute_reply": "2026-04-02T23:09:20.818822Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "On branch master\r\n", - "Changes to be committed:\r\n", - " (use \"git restore --staged ...\" to unstage)\r\n", - "\t\u001b[32mnew file: config.yml\u001b[m\r\n", - "\t\u001b[32mnew file: data.tsv\u001b[m\r\n", - "\t\u001b[32mnew file: meta.yml\u001b[m\r\n", - "\r\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ - "! pushd repo2.hm && git status && popd" + "! ls -a repo2.hm\n" ] }, { "cell_type": "markdown", - "id": "md-s2", "metadata": {}, "source": [ - "## 2. Add and Commit Files\n", - "\n", - "`repo.add(fmt)` scans `repo1/` for files whose names match the format\n", - "string, computes a checksum for each one, and stages them in the manifest.\n", - "It returns a table showing the matched files and their parsed parameters.\n", - "\n", - "`repo.commit(msg)` saves the current state of the manifest as a git commit\n", - "inside `.hm`, and stores the corresponding file bytes in the shared `objects/`\n", - "store.\n" + "## 2. Add and Commit Files\n" ] }, { "cell_type": "code", - "execution_count": 9, - "id": "code-touch-files", - "metadata": { - "execution": { - "iopub.execute_input": "2026-04-02T23:09:20.821599Z", - "iopub.status.busy": "2026-04-02T23:09:20.821475Z", - "iopub.status.idle": "2026-04-02T23:09:20.851435Z", - "shell.execute_reply": "2026-04-02T23:09:20.850987Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallma" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "rk/demo/repo1 ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "/github/hallmark/demo\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallma" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "rk/demo\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "a0_i0.h5\n", - "a0_i120.h5\n", - "a0_i150.h5\n", - "a0_i180.h5\n", - "a0_i30.h5\n", - "a0_i60.h5\n", - "a0_i90.h5\n", - "a0.75_i0.h5\n", - "a0.75_i120.h5\n", - "a0" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - ".75_i150.h5\n", - "a0.75_i180.h5\n", - "a0.75_i30.h5\n", - "a0.75_i60.h5\n", - "a0.75_i90.h5\n", - "a0.94_i0.h5\n", - "a0.94_i120.h5\n", - "a0.94_i15" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0.h5\n", - "a0.94_i180.h5\n", - "a0.94_i30.h5\n", - "a0.94_i60.h5\n", - "a0.94_i90.h5\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "%%bash\n", - "# create sample data files: spin values 0, 0.75, 0.94 at inclinations 0\u2013180\n", "pushd repo1\n", "for f in a{0,0.75,0.94}_i{0,30,60,90,120,150,180}.h5; do echo \"$f\" > \"$f\"; done\n", - "popd\n", - "ls repo1" + "ls *.h5\n", + "popd\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "`repo1.add(\"a{a}_i{i}.h5\")` sets the branch fmt and stages a manifest with `sha1`, `a`, and `i`.\n" ] }, { "cell_type": "code", - "execution_count": 10, - "id": "code-add", - "metadata": { - "execution": { - "iopub.execute_input": "2026-04-02T23:09:20.853005Z", - "iopub.status.busy": "2026-04-02T23:09:20.852903Z", - "iopub.status.idle": "2026-04-02T23:09:20.888112Z", - "shell.execute_reply": "2026-04-02T23:09:20.887629Z" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
pathaisha1
0a0.75_i0.h50.750.0da39a3ee5e6b4b0d3255bfef95601890afd80709
1a0.75_i120.h50.75120.0da39a3ee5e6b4b0d3255bfef95601890afd80709
2a0.75_i150.h50.75150.0da39a3ee5e6b4b0d3255bfef95601890afd80709
3a0.75_i180.h50.75180.0da39a3ee5e6b4b0d3255bfef95601890afd80709
4a0.75_i30.h50.7530.0da39a3ee5e6b4b0d3255bfef95601890afd80709
5a0.75_i60.h50.7560.0da39a3ee5e6b4b0d3255bfef95601890afd80709
6a0.75_i90.h50.7590.0da39a3ee5e6b4b0d3255bfef95601890afd80709
7a0.94_i0.h50.940.0da39a3ee5e6b4b0d3255bfef95601890afd80709
8a0.94_i120.h50.94120.0da39a3ee5e6b4b0d3255bfef95601890afd80709
9a0.94_i150.h50.94150.0da39a3ee5e6b4b0d3255bfef95601890afd80709
10a0.94_i180.h50.94180.0da39a3ee5e6b4b0d3255bfef95601890afd80709
11a0.94_i30.h50.9430.0da39a3ee5e6b4b0d3255bfef95601890afd80709
12a0.94_i60.h50.9460.0da39a3ee5e6b4b0d3255bfef95601890afd80709
13a0.94_i90.h50.9490.0da39a3ee5e6b4b0d3255bfef95601890afd80709
14a0_i0.h50.000.0da39a3ee5e6b4b0d3255bfef95601890afd80709
15a0_i120.h50.00120.0da39a3ee5e6b4b0d3255bfef95601890afd80709
16a0_i150.h50.00150.0da39a3ee5e6b4b0d3255bfef95601890afd80709
17a0_i180.h50.00180.0da39a3ee5e6b4b0d3255bfef95601890afd80709
18a0_i30.h50.0030.0da39a3ee5e6b4b0d3255bfef95601890afd80709
19a0_i60.h50.0060.0da39a3ee5e6b4b0d3255bfef95601890afd80709
20a0_i90.h50.0090.0da39a3ee5e6b4b0d3255bfef95601890afd80709
\n", - "
" - ], - "text/plain": [ - " path a i sha1\n", - "0 a0.75_i0.h5 0.75 0.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "1 a0.75_i120.h5 0.75 120.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "2 a0.75_i150.h5 0.75 150.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "3 a0.75_i180.h5 0.75 180.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "4 a0.75_i30.h5 0.75 30.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "5 a0.75_i60.h5 0.75 60.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "6 a0.75_i90.h5 0.75 90.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "7 a0.94_i0.h5 0.94 0.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "8 a0.94_i120.h5 0.94 120.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "9 a0.94_i150.h5 0.94 150.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "10 a0.94_i180.h5 0.94 180.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "11 a0.94_i30.h5 0.94 30.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "12 a0.94_i60.h5 0.94 60.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "13 a0.94_i90.h5 0.94 90.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "14 a0_i0.h5 0.00 0.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "15 a0_i120.h5 0.00 120.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "16 a0_i150.h5 0.00 150.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "17 a0_i180.h5 0.00 180.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "18 a0_i30.h5 0.00 30.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "19 a0_i60.h5 0.00 60.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "20 a0_i90.h5 0.00 90.0 da39a3ee5e6b4b0d3255bfef95601890afd80709" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ - "# scan repo1/ for files matching the pattern and record them in the index\n", "pf = repo1.add(\"a{a}_i{i}.h5\")\n", - "pf" + "pf\n" ] }, { "cell_type": "code", - "execution_count": 11, - "id": "code-commit", - "metadata": { - "execution": { - "iopub.execute_input": "2026-04-02T23:09:20.889534Z", - "iopub.status.busy": "2026-04-02T23:09:20.889431Z", - "iopub.status.idle": "2026-04-02T23:09:20.946351Z", - "shell.execute_reply": "2026-04-02T23:09:20.945794Z" - } - }, - "outputs": [ - { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ - "# save this snapshot \u2014 returns True if a commit was made\n", - "repo1.commit(\"add spin and inclination files\")" + "! echo \"=== config.yml ===\" && cat repo1/.hm/config.yml && echo \"\" && echo \"=== data.tsv ===\" && cat repo1/.hm/data.tsv\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "repo1.commit(\"add spin and inclination files\")\n" ] }, { "cell_type": "markdown", - "id": "md-s3", "metadata": {}, "source": [ - "## 3. Discover and Filter Files with ParaFrame\n", - "\n", - "`repo.add()` uses `ParaFrame` under the hood to find files.\n", - "You can also call `ParaFrame.parse()` directly \u2014 useful when you want to\n", - "look up files without adding them to the index.\n", - "\n", - "`ParaFrame` is a table (a pandas DataFrame) where each row is a file and\n", - "each column is a parameter read from the file name." + "## 3. Inspect Status\n" ] }, { "cell_type": "code", - "execution_count": 12, - "id": "code-pf-parse", - "metadata": { - "execution": { - "iopub.execute_input": "2026-04-02T23:09:20.948276Z", - "iopub.status.busy": "2026-04-02T23:09:20.948157Z", - "iopub.status.idle": "2026-04-02T23:09:20.963498Z", - "shell.execute_reply": "2026-04-02T23:09:20.963012Z" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
pathai
0a0.75_i0.h50.750.0
1a0.75_i120.h50.75120.0
2a0.75_i150.h50.75150.0
3a0.75_i180.h50.75180.0
4a0.75_i30.h50.7530.0
5a0.75_i60.h50.7560.0
6a0.75_i90.h50.7590.0
7a0.94_i0.h50.940.0
8a0.94_i120.h50.94120.0
9a0.94_i150.h50.94150.0
10a0.94_i180.h50.94180.0
11a0.94_i30.h50.9430.0
12a0.94_i60.h50.9460.0
13a0.94_i90.h50.9490.0
14a0_i0.h50.000.0
15a0_i120.h50.00120.0
16a0_i150.h50.00150.0
17a0_i180.h50.00180.0
18a0_i30.h50.0030.0
19a0_i60.h50.0060.0
20a0_i90.h50.0090.0
\n", - "
" - ], - "text/plain": [ - " path a i\n", - "0 a0.75_i0.h5 0.75 0.0\n", - "1 a0.75_i120.h5 0.75 120.0\n", - "2 a0.75_i150.h5 0.75 150.0\n", - "3 a0.75_i180.h5 0.75 180.0\n", - "4 a0.75_i30.h5 0.75 30.0\n", - "5 a0.75_i60.h5 0.75 60.0\n", - "6 a0.75_i90.h5 0.75 90.0\n", - "7 a0.94_i0.h5 0.94 0.0\n", - "8 a0.94_i120.h5 0.94 120.0\n", - "9 a0.94_i150.h5 0.94 150.0\n", - "10 a0.94_i180.h5 0.94 180.0\n", - "11 a0.94_i30.h5 0.94 30.0\n", - "12 a0.94_i60.h5 0.94 60.0\n", - "13 a0.94_i90.h5 0.94 90.0\n", - "14 a0_i0.h5 0.00 0.0\n", - "15 a0_i120.h5 0.00 120.0\n", - "16 a0_i150.h5 0.00 150.0\n", - "17 a0_i180.h5 0.00 180.0\n", - "18 a0_i30.h5 0.00 30.0\n", - "19 a0_i60.h5 0.00 60.0\n", - "20 a0_i90.h5 0.00 90.0" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ - "# discover files in repo1/ directly \u2014 no index update\n", - "pf = ParaFrame.parse(\"a{a}_i{i}.h5\", base_path=\"repo1\")\n", - "pf" + "repo1.status()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "! cd repo1 && hallmark status\n" ] }, { "cell_type": "markdown", - "id": "md-s3-debug", "metadata": {}, "source": [ - "### Debug mode\n", + "### Config edits\n", "\n", - "`debug=True` prints the glob pattern used to search for files.\n", - "This helps you check the pattern when no files are found." + "Use `repo1.set_config(...)` when you want to change branch config explicitly, such as switching the branch fmt before `repo1.add(\".\")`.\n" ] }, { "cell_type": "code", - "execution_count": 13, - "id": "code-debug", - "metadata": { - "execution": { - "iopub.execute_input": "2026-04-02T23:09:20.965095Z", - "iopub.status.busy": "2026-04-02T23:09:20.964973Z", - "iopub.status.idle": "2026-04-02T23:09:20.978901Z", - "shell.execute_reply": "2026-04-02T23:09:20.978605Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0 repo1/a{a}_i{i}.h5 () {}\n", - "1 repo1/a{a:s}_i{i}.h5 () {'a': '*'}\n", - "2 repo1/a{a:s}_i{i:s}.h5 () {'a': '*', 'i': '*'}\n", - "Pattern: \"repo1/a*_i*.h5\"\n", - "21 matches, e.g., \"repo1/a0.75_i0.h5\"\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
pathai
0a0.75_i0.h50.750.0
1a0.75_i120.h50.75120.0
2a0.75_i150.h50.75150.0
3a0.75_i180.h50.75180.0
4a0.75_i30.h50.7530.0
5a0.75_i60.h50.7560.0
6a0.75_i90.h50.7590.0
7a0.94_i0.h50.940.0
8a0.94_i120.h50.94120.0
9a0.94_i150.h50.94150.0
10a0.94_i180.h50.94180.0
11a0.94_i30.h50.9430.0
12a0.94_i60.h50.9460.0
13a0.94_i90.h50.9490.0
14a0_i0.h50.000.0
15a0_i120.h50.00120.0
16a0_i150.h50.00150.0
17a0_i180.h50.00180.0
18a0_i30.h50.0030.0
19a0_i60.h50.0060.0
20a0_i90.h50.0090.0
\n", - "
" - ], - "text/plain": [ - " path a i\n", - "0 a0.75_i0.h5 0.75 0.0\n", - "1 a0.75_i120.h5 0.75 120.0\n", - "2 a0.75_i150.h5 0.75 150.0\n", - "3 a0.75_i180.h5 0.75 180.0\n", - "4 a0.75_i30.h5 0.75 30.0\n", - "5 a0.75_i60.h5 0.75 60.0\n", - "6 a0.75_i90.h5 0.75 90.0\n", - "7 a0.94_i0.h5 0.94 0.0\n", - "8 a0.94_i120.h5 0.94 120.0\n", - "9 a0.94_i150.h5 0.94 150.0\n", - "10 a0.94_i180.h5 0.94 180.0\n", - "11 a0.94_i30.h5 0.94 30.0\n", - "12 a0.94_i60.h5 0.94 60.0\n", - "13 a0.94_i90.h5 0.94 90.0\n", - "14 a0_i0.h5 0.00 0.0\n", - "15 a0_i120.h5 0.00 120.0\n", - "16 a0_i150.h5 0.00 150.0\n", - "17 a0_i180.h5 0.00 180.0\n", - "18 a0_i30.h5 0.00 30.0\n", - "19 a0_i60.h5 0.00 60.0\n", - "20 a0_i90.h5 0.00 90.0" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ - "ParaFrame.parse(\"a{a}_i{i}.h5\", base_path=\"repo1\", debug=True)" + "repo1.set_config(remote_name=\"origin\", remote_url=\"https://example.com/path\")\nrepo1.state.config\n" ] }, { "cell_type": "markdown", - "id": "md-s3-filter", "metadata": {}, "source": [ - "### Filter by parameter value\n", + "## 4. Rebuild the manifest with `repo.add(\".\")`\n", "\n", - "Call a `ParaFrame` like a function to filter rows:\n", - "- **scalar** \u2014 keep rows where the column equals that value\n", - "- **list** \u2014 keep rows where the column matches any value in the list\n", - "- **chaining** \u2014 each call narrows the result further (AND logic)" + "Delete some tracked files and rebuild the manifest from current files that still match the branch fmt.\n" ] }, { "cell_type": "code", - "execution_count": 14, - "id": "code-filter-scalar", - "metadata": { - "execution": { - "iopub.execute_input": "2026-04-02T23:09:20.980423Z", - "iopub.status.busy": "2026-04-02T23:09:20.980321Z", - "iopub.status.idle": "2026-04-02T23:09:20.995478Z", - "shell.execute_reply": "2026-04-02T23:09:20.995058Z" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
pathai
14a0_i0.h50.00.0
15a0_i120.h50.0120.0
16a0_i150.h50.0150.0
17a0_i180.h50.0180.0
18a0_i30.h50.030.0
19a0_i60.h50.060.0
20a0_i90.h50.090.0
\n", - "
" - ], - "text/plain": [ - " path a i\n", - "14 a0_i0.h5 0.0 0.0\n", - "15 a0_i120.h5 0.0 120.0\n", - "16 a0_i150.h5 0.0 150.0\n", - "17 a0_i180.h5 0.0 180.0\n", - "18 a0_i30.h5 0.0 30.0\n", - "19 a0_i60.h5 0.0 60.0\n", - "20 a0_i90.h5 0.0 90.0" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ - "# keep only rows where a == 0\n", - "pf(a=0)" + "repo1.checkout(\"sync-demo\")\n" ] }, { "cell_type": "code", - "execution_count": 15, - "id": "code-filter-list", - "metadata": { - "execution": { - "iopub.execute_input": "2026-04-02T23:09:20.996721Z", - "iopub.status.busy": "2026-04-02T23:09:20.996641Z", - "iopub.status.idle": "2026-04-02T23:09:21.009061Z", - "shell.execute_reply": "2026-04-02T23:09:21.008623Z" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
pathai
0a0.75_i0.h50.750.0
1a0.75_i120.h50.75120.0
2a0.75_i150.h50.75150.0
3a0.75_i180.h50.75180.0
4a0.75_i30.h50.7530.0
5a0.75_i60.h50.7560.0
6a0.75_i90.h50.7590.0
14a0_i0.h50.000.0
15a0_i120.h50.00120.0
16a0_i150.h50.00150.0
17a0_i180.h50.00180.0
18a0_i30.h50.0030.0
19a0_i60.h50.0060.0
20a0_i90.h50.0090.0
\n", - "
" - ], - "text/plain": [ - " path a i\n", - "0 a0.75_i0.h5 0.75 0.0\n", - "1 a0.75_i120.h5 0.75 120.0\n", - "2 a0.75_i150.h5 0.75 150.0\n", - "3 a0.75_i180.h5 0.75 180.0\n", - "4 a0.75_i30.h5 0.75 30.0\n", - "5 a0.75_i60.h5 0.75 60.0\n", - "6 a0.75_i90.h5 0.75 90.0\n", - "14 a0_i0.h5 0.00 0.0\n", - "15 a0_i120.h5 0.00 120.0\n", - "16 a0_i150.h5 0.00 150.0\n", - "17 a0_i180.h5 0.00 180.0\n", - "18 a0_i30.h5 0.00 30.0\n", - "19 a0_i60.h5 0.00 60.0\n", - "20 a0_i90.h5 0.00 90.0" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ - "# keep rows where a == 0 or a == 0.75\n", - "pf(a=[0, 0.75])" + "%%bash\n", + "cd repo1\n", + "rm a0.75_i{0,30,60,90,120,150,180}.h5\n", + "ls *.h5\n" ] }, { "cell_type": "code", - "execution_count": 16, - "id": "code-filter-chain", - "metadata": { - "execution": { - "iopub.execute_input": "2026-04-02T23:09:21.010493Z", - "iopub.status.busy": "2026-04-02T23:09:21.010403Z", - "iopub.status.idle": "2026-04-02T23:09:21.022395Z", - "shell.execute_reply": "2026-04-02T23:09:21.021918Z" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
pathai
7a0.94_i0.h50.940.0
\n", - "
" - ], - "text/plain": [ - " path a i\n", - "7 a0.94_i0.h5 0.94 0.0" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ - "# keep rows where a == 0.94 AND i == 0 (chain two calls)\n", - "pf(a=0.94)(i=0)" + "repo1.add(\".\")\n" ] }, { "cell_type": "code", - "execution_count": 17, - "id": "code-filter-mask", - "metadata": { - "execution": { - "iopub.execute_input": "2026-04-02T23:09:21.023802Z", - "iopub.status.busy": "2026-04-02T23:09:21.023704Z", - "iopub.status.idle": "2026-04-02T23:09:21.035134Z", - "shell.execute_reply": "2026-04-02T23:09:21.034660Z" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
pathai
0a0.75_i0.h50.750.0
4a0.75_i30.h50.7530.0
5a0.75_i60.h50.7560.0
7a0.94_i0.h50.940.0
11a0.94_i30.h50.9430.0
12a0.94_i60.h50.9460.0
14a0_i0.h50.000.0
18a0_i30.h50.0030.0
19a0_i60.h50.0060.0
\n", - "
" - ], - "text/plain": [ - " path a i\n", - "0 a0.75_i0.h5 0.75 0.0\n", - "4 a0.75_i30.h5 0.75 30.0\n", - "5 a0.75_i60.h5 0.75 60.0\n", - "7 a0.94_i0.h5 0.94 0.0\n", - "11 a0.94_i30.h5 0.94 30.0\n", - "12 a0.94_i60.h5 0.94 60.0\n", - "14 a0_i0.h5 0.00 0.0\n", - "18 a0_i30.h5 0.00 30.0\n", - "19 a0_i60.h5 0.00 60.0" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "! cat repo1/.hm/data.tsv\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "repo1.commit(\"sync manifest with dot add\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ - "# use standard pandas boolean indexing for range or custom conditions\n", - "pf[pf.i <= 60]" + "repo1.checkout(\"main\")\n" ] }, { "cell_type": "markdown", - "id": "md-s3-loop", "metadata": {}, "source": [ - "### Use filtered paths in a loop\n", - "\n", - "Note: passing multiple keyword arguments in a single `pf(...)` call gives\n", - "OR logic across all of them. Chain calls to get AND logic." + "## 5. Discover and Filter Files with ParaFrame\n" ] }, { "cell_type": "code", - "execution_count": 18, - "id": "code-loop", - "metadata": { - "execution": { - "iopub.execute_input": "2026-04-02T23:09:21.036445Z", - "iopub.status.busy": "2026-04-02T23:09:21.036350Z", - "iopub.status.idle": "2026-04-02T23:09:21.046246Z", - "shell.execute_reply": "2026-04-02T23:09:21.045705Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Processing \"a0.75_i0.h5\"...\n", - "Processing \"a0.94_i0.h5\"...\n", - "Processing \"a0_i0.h5\"...\n", - "Processing \"a0_i120.h5\"...\n", - "Processing \"a0_i150.h5\"...\n", - "Processing \"a0_i180.h5\"...\n", - "Processing \"a0_i30.h5\"...\n", - "Processing \"a0_i60.h5\"...\n", - "Processing \"a0_i90.h5\"...\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pf = ParaFrame.parse(\"a{a}_i{i}.h5\", base_path=\"repo1\")\n", + "pf\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ParaFrame.parse(\"a{a}_i{i}.h5\", base_path=\"repo1\", debug=True)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pf(a=0)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pf(a=[0, 0.75])\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pf(a=0.94)(i=0)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pf[pf.i <= 60]\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ - "# selects rows where a == 0 OR i == 0 (not both at once)\n", "for path in pf(a=0, i=0).path:\n", - " print(f'Processing \"{path}\"...')" + " print(f'Processing \"{path}\"...')\n" ] }, { "cell_type": "markdown", - "id": "md-s4", "metadata": {}, "source": [ - "## 4. Custom Filename Encoding\n", + "## 6. Custom Filename Encoding in `config.yml`\n", "\n", - "Some datasets use a special convention to write certain values in file names.\n", - "For example, a negative spin value like `-0.94` might be stored as `m0.94`\n", - "in the filename, where the `m` prefix means \"minus\".\n", - "\n", - "Hallmark handles this with an encoding config: you write a regular expression\n", - "that describes the pattern, and `hallmark` applies it before reading the\n", - "parameter values. You store this config in `.hm/config.yml`." + "For regex substitutions, use the single branch dataset spec under `data:`.\n", + "We use a separate repo so the branch still has only one fmt.\n" ] }, { - "cell_type": "markdown", - "id": "md-s4-create", + "cell_type": "code", + "execution_count": null, "metadata": {}, + "outputs": [], "source": [ - "### Create files and add the encoding config" + "repo_enc = Repo.init(\"repo_enc\")\n" ] }, { "cell_type": "code", - "execution_count": 19, - "id": "code-touch-spin", - "metadata": { - "execution": { - "iopub.execute_input": "2026-04-02T23:09:21.047893Z", - "iopub.status.busy": "2026-04-02T23:09:21.047767Z", - "iopub.status.idle": "2026-04-02T23:09:21.072502Z", - "shell.execute_reply": "2026-04-02T23:09:21.072124Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallma" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "rk/demo/repo1 ~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "/github/hallmark/demo\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "~/Library/CloudStorage/GoogleDrive-ramadithya.research@gmail.com/My Drive/uni/research/github/hallma" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "rk/demo\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "repo1/am0.75_i0.h5\n", - "repo1/am0.75_i30.h5\n", - "repo1/am0.75_i60.h5\n", - "repo1/am0.94_i0.h5\n", - "repo1/am0.94_i30.h5\n", - "re" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "po1/am0.94_i60.h5\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "%%bash\n", - "# create files with negative spin values using the m-prefix convention\n", - "pushd repo1\n", - "for f in am0.75_i0.h5 am0.75_i30.h5 am0.75_i60.h5 am0.94_i0.h5 am0.94_i30.h5 am0.94_i60.h5; do echo \"$f\" > \"$f\"; done\n", - "popd\n", - "ls repo1/am*.h5" + "pushd repo_enc\n", + "for f in am0.75_i0.h5 am0.75_i30.h5 am0.75_i60.h5 am0.94_i0.h5 am0.94_i30.h5 am0.94_i60.h5; do echo \"$f\" > \"$f\"; done\n", + "popd\n" ] }, { "cell_type": "code", - "execution_count": 20, - "id": "code-write-config", - "metadata": { - "execution": { - "iopub.execute_input": "2026-04-02T23:09:21.074248Z", - "iopub.status.busy": "2026-04-02T23:09:21.074158Z", - "iopub.status.idle": "2026-04-02T23:09:21.099065Z", - "shell.execute_reply": "2026-04-02T23:09:21.098487Z" - } - }, + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "%%bash\n", - "# write the encoding config to .hm/config.yml\n", - "# the regex m([0-9]+(\\.[0-9]+)?) matches the m-prefix and captures the number\n", - "cat > repo1/.hm/config.yml <<'EOF'\n", - "encodings:\n", + "cat > repo_enc/.hm/config.yml <<'EOF'\n", + "data:\n", " - fmt: \"a{aspin}_i{i}.h5\"\n", " encoding:\n", - " aspin: \"m([0-9]+(\\\\.[0-9]+)?)\"\n", - "EOF" + " aspin: \"m([0-9]+(\\.[0-9]+)?|\\.[0-9]+)\"\n", + "EOF\n" ] }, { "cell_type": "code", - "execution_count": 21, - "id": "code-reload-config", - "metadata": { - "execution": { - "iopub.execute_input": "2026-04-02T23:09:21.100614Z", - "iopub.status.busy": "2026-04-02T23:09:21.100508Z", - "iopub.status.idle": "2026-04-02T23:09:21.111890Z", - "shell.execute_reply": "2026-04-02T23:09:21.111492Z" - } - }, + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ - "# reload the config so the repo object knows about the new encoding\n", - "repo1.state.config = repo1.dothm.load_yml(\"config\")" + "repo_enc.state.config = repo_enc.dothm.load_yml(\"config\")\n", + "repo_enc.state.config\n" ] }, { "cell_type": "code", - "execution_count": 22, - "id": "code-add-encoding", - "metadata": { - "execution": { - "iopub.execute_input": "2026-04-02T23:09:21.113669Z", - "iopub.status.busy": "2026-04-02T23:09:21.113569Z", - "iopub.status.idle": "2026-04-02T23:09:21.138248Z", - "shell.execute_reply": "2026-04-02T23:09:21.137828Z" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
pathaspinisha1
0a0.75_i0.h50.750.0da39a3ee5e6b4b0d3255bfef95601890afd80709
1a0.75_i120.h50.75120.0da39a3ee5e6b4b0d3255bfef95601890afd80709
2a0.75_i150.h50.75150.0da39a3ee5e6b4b0d3255bfef95601890afd80709
3a0.75_i180.h50.75180.0da39a3ee5e6b4b0d3255bfef95601890afd80709
4a0.75_i30.h50.7530.0da39a3ee5e6b4b0d3255bfef95601890afd80709
5a0.75_i60.h50.7560.0da39a3ee5e6b4b0d3255bfef95601890afd80709
6a0.75_i90.h50.7590.0da39a3ee5e6b4b0d3255bfef95601890afd80709
7a0.94_i0.h50.940.0da39a3ee5e6b4b0d3255bfef95601890afd80709
8a0.94_i120.h50.94120.0da39a3ee5e6b4b0d3255bfef95601890afd80709
9a0.94_i150.h50.94150.0da39a3ee5e6b4b0d3255bfef95601890afd80709
10a0.94_i180.h50.94180.0da39a3ee5e6b4b0d3255bfef95601890afd80709
11a0.94_i30.h50.9430.0da39a3ee5e6b4b0d3255bfef95601890afd80709
12a0.94_i60.h50.9460.0da39a3ee5e6b4b0d3255bfef95601890afd80709
13a0.94_i90.h50.9490.0da39a3ee5e6b4b0d3255bfef95601890afd80709
14a0_i0.h50.000.0da39a3ee5e6b4b0d3255bfef95601890afd80709
15a0_i120.h50.00120.0da39a3ee5e6b4b0d3255bfef95601890afd80709
16a0_i150.h50.00150.0da39a3ee5e6b4b0d3255bfef95601890afd80709
17a0_i180.h50.00180.0da39a3ee5e6b4b0d3255bfef95601890afd80709
18a0_i30.h50.0030.0da39a3ee5e6b4b0d3255bfef95601890afd80709
19a0_i60.h50.0060.0da39a3ee5e6b4b0d3255bfef95601890afd80709
20a0_i90.h50.0090.0da39a3ee5e6b4b0d3255bfef95601890afd80709
21am0.75_i0.h5-0.750.0da39a3ee5e6b4b0d3255bfef95601890afd80709
22am0.75_i30.h5-0.7530.0da39a3ee5e6b4b0d3255bfef95601890afd80709
23am0.75_i60.h5-0.7560.0da39a3ee5e6b4b0d3255bfef95601890afd80709
24am0.94_i0.h5-0.940.0da39a3ee5e6b4b0d3255bfef95601890afd80709
25am0.94_i30.h5-0.9430.0da39a3ee5e6b4b0d3255bfef95601890afd80709
26am0.94_i60.h5-0.9460.0da39a3ee5e6b4b0d3255bfef95601890afd80709
\n", - "
" - ], - "text/plain": [ - " path aspin i sha1\n", - "0 a0.75_i0.h5 0.75 0.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "1 a0.75_i120.h5 0.75 120.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "2 a0.75_i150.h5 0.75 150.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "3 a0.75_i180.h5 0.75 180.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "4 a0.75_i30.h5 0.75 30.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "5 a0.75_i60.h5 0.75 60.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "6 a0.75_i90.h5 0.75 90.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "7 a0.94_i0.h5 0.94 0.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "8 a0.94_i120.h5 0.94 120.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "9 a0.94_i150.h5 0.94 150.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "10 a0.94_i180.h5 0.94 180.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "11 a0.94_i30.h5 0.94 30.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "12 a0.94_i60.h5 0.94 60.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "13 a0.94_i90.h5 0.94 90.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "14 a0_i0.h5 0.00 0.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "15 a0_i120.h5 0.00 120.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "16 a0_i150.h5 0.00 150.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "17 a0_i180.h5 0.00 180.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "18 a0_i30.h5 0.00 30.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "19 a0_i60.h5 0.00 60.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "20 a0_i90.h5 0.00 90.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "21 am0.75_i0.h5 -0.75 0.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "22 am0.75_i30.h5 -0.75 30.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "23 am0.75_i60.h5 -0.75 60.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "24 am0.94_i0.h5 -0.94 0.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "25 am0.94_i30.h5 -0.94 30.0 da39a3ee5e6b4b0d3255bfef95601890afd80709\n", - "26 am0.94_i60.h5 -0.94 60.0 da39a3ee5e6b4b0d3255bfef95601890afd80709" - ] - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ - "# add all files \u2014 m0.94 in the filename is decoded to -0.94 in the table\n", - "pf_spin = repo1.add(\"a{aspin}_i{i}.h5\", encoding=True)\n", - "pf_spin" + "pf_spin = repo_enc.add(\"a{aspin}_i{i}.h5\", encoding=True)\n", + "pf_spin\n" ] }, { "cell_type": "code", - "execution_count": 23, - "id": "code-commit-spin", - "metadata": { - "execution": { - "iopub.execute_input": "2026-04-02T23:09:21.139430Z", - "iopub.status.busy": "2026-04-02T23:09:21.139347Z", - "iopub.status.idle": "2026-04-02T23:09:21.174031Z", - "shell.execute_reply": "2026-04-02T23:09:21.173468Z" - } - }, - "outputs": [ - { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ - "repo1.commit(\"add negative spin files with encoding\")" + "! echo \"=== encoded config.yml ===\" && cat repo_enc/.hm/config.yml && echo \"\" && echo \"=== encoded data.tsv ===\" && cat repo_enc/.hm/data.tsv\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "repo_enc.commit(\"add negative spin files with encoding\")\n" ] }, { "cell_type": "markdown", - "id": "md-s4-pf", "metadata": {}, "source": [ - "### Discover encoded files with ParaFrame\n", - "\n", - "You can also use `ParaFrame.parse()` directly with encoding.\n", - "Pass the encoding config from `repo1.state.config` so it knows the regex." + "You can also parse encoded files directly with `ParaFrame.parse()`.\n" ] }, { "cell_type": "code", - "execution_count": 24, - "id": "code-pf-encoding", - "metadata": { - "execution": { - "iopub.execute_input": "2026-04-02T23:09:21.175560Z", - "iopub.status.busy": "2026-04-02T23:09:21.175454Z", - "iopub.status.idle": "2026-04-02T23:09:21.191463Z", - "shell.execute_reply": "2026-04-02T23:09:21.190918Z" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
pathaspini
0a0.75_i0.h50.750.0
1a0.75_i120.h50.75120.0
2a0.75_i150.h50.75150.0
3a0.75_i180.h50.75180.0
4a0.75_i30.h50.7530.0
5a0.75_i60.h50.7560.0
6a0.75_i90.h50.7590.0
7a0.94_i0.h50.940.0
8a0.94_i120.h50.94120.0
9a0.94_i150.h50.94150.0
10a0.94_i180.h50.94180.0
11a0.94_i30.h50.9430.0
12a0.94_i60.h50.9460.0
13a0.94_i90.h50.9490.0
14a0_i0.h50.000.0
15a0_i120.h50.00120.0
16a0_i150.h50.00150.0
17a0_i180.h50.00180.0
18a0_i30.h50.0030.0
19a0_i60.h50.0060.0
20a0_i90.h50.0090.0
21am0.75_i0.h5-0.750.0
22am0.75_i30.h5-0.7530.0
23am0.75_i60.h5-0.7560.0
24am0.94_i0.h5-0.940.0
25am0.94_i30.h5-0.9430.0
26am0.94_i60.h5-0.9460.0
\n", - "
" - ], - "text/plain": [ - " path aspin i\n", - "0 a0.75_i0.h5 0.75 0.0\n", - "1 a0.75_i120.h5 0.75 120.0\n", - "2 a0.75_i150.h5 0.75 150.0\n", - "3 a0.75_i180.h5 0.75 180.0\n", - "4 a0.75_i30.h5 0.75 30.0\n", - "5 a0.75_i60.h5 0.75 60.0\n", - "6 a0.75_i90.h5 0.75 90.0\n", - "7 a0.94_i0.h5 0.94 0.0\n", - "8 a0.94_i120.h5 0.94 120.0\n", - "9 a0.94_i150.h5 0.94 150.0\n", - "10 a0.94_i180.h5 0.94 180.0\n", - "11 a0.94_i30.h5 0.94 30.0\n", - "12 a0.94_i60.h5 0.94 60.0\n", - "13 a0.94_i90.h5 0.94 90.0\n", - "14 a0_i0.h5 0.00 0.0\n", - "15 a0_i120.h5 0.00 120.0\n", - "16 a0_i150.h5 0.00 150.0\n", - "17 a0_i180.h5 0.00 180.0\n", - "18 a0_i30.h5 0.00 30.0\n", - "19 a0_i60.h5 0.00 60.0\n", - "20 a0_i90.h5 0.00 90.0\n", - "21 am0.75_i0.h5 -0.75 0.0\n", - "22 am0.75_i30.h5 -0.75 30.0\n", - "23 am0.75_i60.h5 -0.75 60.0\n", - "24 am0.94_i0.h5 -0.94 0.0\n", - "25 am0.94_i30.h5 -0.94 30.0\n", - "26 am0.94_i60.h5 -0.94 60.0" - ] - }, - "execution_count": 24, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ - "encodings = repo1.state.config.get(\"encodings\", [])\n", + "encodings = repo_enc.state.config.get(\"data\", [])\n", "\n", "pf_spin = ParaFrame.parse(\n", " \"a{aspin}_i{i}.h5\",\n", " encodings=encodings,\n", - " base_path=\"repo1\",\n", + " base_path=\"repo_enc\",\n", " encoding=True,\n", ")\n", - "pf_spin" + "pf_spin\n" ] }, { "cell_type": "code", - "execution_count": 25, - "id": "code-filter-spin", - "metadata": { - "execution": { - "iopub.execute_input": "2026-04-02T23:09:21.192942Z", - "iopub.status.busy": "2026-04-02T23:09:21.192838Z", - "iopub.status.idle": "2026-04-02T23:09:21.204692Z", - "shell.execute_reply": "2026-04-02T23:09:21.204364Z" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
pathaspini
21am0.75_i0.h5-0.750.0
22am0.75_i30.h5-0.7530.0
23am0.75_i60.h5-0.7560.0
24am0.94_i0.h5-0.940.0
25am0.94_i30.h5-0.9430.0
26am0.94_i60.h5-0.9460.0
\n", - "
" - ], - "text/plain": [ - " path aspin i\n", - "21 am0.75_i0.h5 -0.75 0.0\n", - "22 am0.75_i30.h5 -0.75 30.0\n", - "23 am0.75_i60.h5 -0.75 60.0\n", - "24 am0.94_i0.h5 -0.94 0.0\n", - "25 am0.94_i30.h5 -0.94 30.0\n", - "26 am0.94_i60.h5 -0.94 60.0" - ] - }, - "execution_count": 25, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ - "# filter using the decoded float values \u2014 not the raw m-prefix strings\n", - "pf_spin(aspin=[-0.75, -0.94])" + "pf_spin(aspin=[-0.75, -0.94])\n" ] } ], diff --git a/mod/hallmark/cli.py b/mod/hallmark/cli.py index 718e9d5..592de6a 100644 --- a/mod/hallmark/cli.py +++ b/mod/hallmark/cli.py @@ -70,19 +70,75 @@ def info(repo): click.echo(f'hallmark worktree: "{repo.worktree}"') -@hallmark.command(short_help="Add files to hallmark index using a Python f-string.") -@click.argument("fstring") +@hallmark.command(short_help="Show worktree and staged hallmark state.") @click.pass_obj -def add(repo, fstring): - """Add files discovered via a Python format string to the hallmark index. - - This is analogous to `git add FILE`, which adds file contents to - the "index" (also known as the "staging area"). - Instead of specifying file names directly, this function uses a - Python format string (i.e., an f-string) to discover and add - matching files to the hallmark index. +def status(repo): + """Show hallmark status for the current branch and worktree.""" + snapshot = repo.status() + + click.echo(f'On branch {snapshot["branch"]}') + + staged = snapshot["staged"] + worktree = snapshot["worktree"] + untracked = snapshot["untracked"] + + def emit_section(title, entries, fg): + if not entries: + return + click.echo("") + click.secho(title, fg=fg) + for label, paths in entries: + for path in paths: + click.echo(" " + click.style(f"{label}: {path}", fg=fg)) + + emit_section( + "Changes to be committed:", + [ + ("new file", staged["added"]), + ("modified", staged["modified"]), + ("deleted", staged["deleted"]), + ], + "green", + ) + emit_section( + "Changes not staged for commit:", + [ + ("modified", worktree["modified"]), + ("deleted", worktree["deleted"]), + ], + "red", + ) + if untracked: + click.echo("") + click.secho("Untracked files:", fg="red") + for path in untracked: + click.echo(" " + click.style(path, fg="red")) + + if not any([staged["added"], staged["modified"], staged["deleted"], + worktree["modified"], worktree["deleted"], untracked]): + click.echo("") + click.echo("nothing to commit, working tree clean") + + +@hallmark.command(short_help="Add files to hallmark index.") +@click.argument("inputs", nargs=-1, required=True) +@click.pass_obj +def add(repo, inputs): + """Add files to the hallmark index. + + `hallmark add FORMAT` uses the branch format string workflow. + `hallmark add "."` rebuilds the manifest from current files that match + the branch `fmt` in `config.yml`. + Explicit path inputs such as shell-expanded `*` are not supported yet + with the parameter-based manifest format. """ - pf = repo.add(fstring) + try: + if len(inputs) == 1: + pf = repo.add(inputs[0]) + else: + pf = repo.add_paths(list(inputs)) + except (RuntimeError, ValueError, FileNotFoundError) as e: + raise ClickException(str(e)) if pf.empty: click.echo("No files matched the format string.") @@ -91,6 +147,39 @@ def add(repo, fstring): click.echo(pf.path.to_string(index=False, header=False)) +@hallmark.command("set-config", short_help="Update hallmark branch config.") +@click.option("--fmt") +@click.option("--remote-name") +@click.option("--remote-url") +@click.option("--encoding", "encodings", multiple=True) +@click.pass_obj +def set_config(repo, fmt, remote_name, remote_url, encodings): + """Update the current branch config.yml.""" + if not any([fmt, remote_name, remote_url, encodings]): + raise ClickException("No config changes requested.") + + encoding_updates = {} + for item in encodings: + if "=" not in item: + raise ClickException('encoding values must use FIELD=REGEX') + field, regex = item.split("=", 1) + if not field: + raise ClickException('encoding values must use FIELD=REGEX') + encoding_updates[field] = regex + + try: + repo.set_config( + fmt=fmt, + remote_name=remote_name, + remote_url=remote_url, + encoding_updates=encoding_updates or None, + ) + except (RuntimeError, ValueError, FileNotFoundError) as e: + raise ClickException(str(e)) + + click.echo("Updated hallmark config.") + + @hallmark.command(short_help="Commit changes to the repository.") @click.option("-m", "message", required=True) @click.pass_obj diff --git a/mod/hallmark/dothm.py b/mod/hallmark/dothm.py index 82450f5..f717ce7 100644 --- a/mod/hallmark/dothm.py +++ b/mod/hallmark/dothm.py @@ -63,6 +63,20 @@ def init(cls, *args, **kwargs) -> "Dothm": dothm.index.commit("Initial commit: local `.hm` repository") return dothm + @staticmethod + def config_template() -> str: + return """# Edit this file only if your branch needs regex substitutions or a preset remote. +# For simple names, you can just run: hallmark add "a{a}_i{i}.h5" +data: + - + # fmt: "{release}_{source}_{year}_{doy:03d}_{band}.uvfits" + encoding: + # aspin: m([0-9]+(\\.[0-9]+)?|\\.[0-9]+) +remote: + # name: origin + # url: https://example.com/path/to/data/ +""" + def link(self, path: Path | str, branch: str | None = None): from git.exc import GitCommandError @@ -97,7 +111,7 @@ def dump_yml(self, data: dict, stem: Path | str) -> None: yaml.dump(data, f, sort_keys=False) def load_tsv(self, stem: Path | str) -> pd.DataFrame: - return pd.read_csv((self.path/stem).with_suffix(".tsv"), sep="\t") + return pd.read_csv((self.path/stem).with_suffix(".tsv"), sep="\t", dtype=str) def dump_tsv(self, data: pd.DataFrame, stem: Path | str) -> None: data.to_csv((self.path/stem).with_suffix(".tsv"), sep="\t", index=False) diff --git a/mod/hallmark/repo.py b/mod/hallmark/repo.py index 2fa8131..76f2d06 100644 --- a/mod/hallmark/repo.py +++ b/mod/hallmark/repo.py @@ -15,7 +15,6 @@ from __future__ import annotations -from io import StringIO import os from contextlib import contextmanager from dataclasses import dataclass @@ -23,13 +22,17 @@ from pathlib import Path from hashlib import sha1 -import pandas as pd +from git.exc import GitCommandError from .state import State from .dothm import Dothm from .worktree import Worktree from .paraframe import ParaFrame from .objects import Objects +from .repo_config import branch_encodings, branch_fmt, path_from_row, row_to_path, set_branch_fmt, set_config as repo_set_config +from .repo_manifest import manifest_frame_from_pf, manifest_map +from .repo_state import load_branch_data, load_head_state +from .repo_worktree import effective_cwd, ensure_clean_tracked_files, filtered_paraframe, tracked_paths @contextmanager @@ -67,6 +70,7 @@ def __init__(self, path: Path | str) -> None: self.dothm = Dothm(dothm_path) self.worktree = worktree_path and Worktree(worktree_path) self.state = self.dothm.load() + self.paraframe_cls = ParaFrame common = Path(self.dothm.common_dir).resolve().parent self.objects = Objects(common) @@ -78,7 +82,11 @@ def __init__(self, path: Path | str) -> None: @classmethod def init(cls, path: Path | str) -> "Repo": dothm_path, worktree_path = cls.lwpaths(path) - Dothm.init(dothm_path).dump(State()) + dothm = Dothm.init(dothm_path) + (dothm.path / "config.yml").write_text(Dothm.config_template(), encoding="utf-8") + dothm.dump_yml({}, "meta") + dothm.dump_tsv(State().data, "data") + dothm.index.add(["config.yml", "meta.yml", "data.tsv"]) Objects(dothm_path) worktree_path and Worktree.init(worktree_path) return cls(path) @@ -91,63 +99,118 @@ def checksum(path: Path, chunk_size: int = 1024 * 1024) -> str: digest.update(block) return digest.hexdigest() - def _tracked_paths(self) -> set[Path]: - return {Path(path) for path in self.state.data["path"].astype(str)} - - def _ensure_clean_tracked_files(self) -> None: - if self.worktree is None: - raise RuntimeError("cannot checkout without a worktree") - - for _, row in self.state.data.iterrows(): - path = self.worktree / row["path"] - if not path.exists(): - raise RuntimeError( - f'tracked file "{row["path"]}" is missing; commit or restore it before checkout') - if self.checksum(path) != row["sha1"]: - raise RuntimeError( - f'tracked file "{row["path"]}" has uncommitted changes; commit them before checkout') - - if self.dothm.index.diff("HEAD"): - raise RuntimeError( - "you have uncommitted hallmark state changes — commit them before checkout") - - def _load_branch_data(self, branch: str) -> State: - if branch in {head.name for head in self.dothm.heads}: - data = self.dothm.git.show(f"{branch}:data.tsv") - frame = State().data if not data.strip() else None - if frame is None: - parsed = pd.read_csv(StringIO(data), sep="\t") - else: - parsed = frame - return State(self.state.config, self.state.meta, parsed) - - return State( - dict(self.state.config), - dict(self.state.meta), - self.state.data.copy(), + def add_paths(self, paths: list[Path | str]) -> ParaFrame: + raise RuntimeError( + 'explicit path add is not supported while data.tsv stores only sha1 plus fmt fields') + + def set_config( + self, + *, + fmt: str | None = None, + remote_name: str | None = None, + remote_url: str | None = None, + encoding_updates: dict[str, str] | None = None, + ) -> dict: + repo_set_config( + self, + fmt=fmt, + remote_name=remote_name, + remote_url=remote_url, + encoding_updates=encoding_updates, + ) + self.dothm.dump(self.state) + return self.state.config + + def status(self) -> dict[str, object]: + head_state = load_head_state(self) + head_map = manifest_map(head_state) + staged_map = manifest_map(self.state) + + staged_added = sorted(path for path in staged_map if path not in head_map) + staged_deleted = sorted(path for path in head_map if path not in staged_map) + staged_modified = sorted( + path for path in staged_map + if path in head_map and staged_map[path] != head_map[path] ) + worktree_modified: list[str] = [] + worktree_deleted: list[str] = [] + tracked_paths = set(staged_map) + + if self.worktree is not None: + for path, staged_sha1 in staged_map.items(): + full_path = self.worktree / path + if not full_path.exists(): + worktree_deleted.append(path) + elif self.checksum(full_path) != staged_sha1: + worktree_modified.append(path) + + untracked = sorted( + str(path.relative_to(self.worktree)) + for path in effective_cwd(self).rglob("*") + if path.is_file() + and ".hm" not in path.relative_to(self.worktree).parts + and str(path.relative_to(self.worktree)) not in tracked_paths + ) + else: + untracked = [] + + return { + "branch": self.dothm.active_branch.name, + "staged": { + "added": staged_added, + "modified": staged_modified, + "deleted": staged_deleted, + }, + "worktree": { + "modified": sorted(worktree_modified), + "deleted": sorted(worktree_deleted), + }, + "untracked": untracked, + } + def add(self, fstr: str, encoding: bool = False) -> ParaFrame: if self.worktree is None: raise RuntimeError( "cannot add files in a bare repository without a worktree") + if fstr == ".": + fmt = branch_fmt(self) + with chdir(self.worktree): + pf = ParaFrame.parse( + fmt, + base_path=self.worktree, + encodings=branch_encodings(self) if encoding else None, + encoding=encoding, + ) + pf = filtered_paraframe(self, pf) + if not pf.empty: + pf["sha1"] = [ + self.checksum(self.worktree / Path(path)) + for path in pf["path"].astype(str) + ] + self.state.replace(manifest_frame_from_pf(pf, fmt)) + self.dothm.dump(self.state) + return pf.drop(columns=["sha1"], errors="ignore") + + set_branch_fmt(self, fstr) + with chdir(self.worktree): pf = ParaFrame.parse( fstr, base_path = self.worktree, - encodings = self.state.config.get("encodings", []) + encodings = branch_encodings(self) if encoding else None, encoding = encoding, ) if not pf.empty: pf["sha1"] = [ - self.checksum(self.worktree / path) + self.checksum(self.worktree / Path(path)) for path in pf["path"].astype(str) ] - self.state.update(pf) + self.state.update(manifest_frame_from_pf(pf, fstr)) self.dothm.dump(self.state) return pf.drop(columns=["sha1"], errors="ignore") @@ -157,7 +220,7 @@ def commit(self, msg: str, allow_empty: bool = False) -> bool: if allow_empty or self.dothm.index.diff("HEAD"): for _, row in self.state.data.iterrows(): - self.objects.store(self.worktree / row["path"], row["sha1"]) + self.objects.store(self.worktree / path_from_row(self, row), row["sha1"]) self.dothm.index.commit(msg) return True else: @@ -169,22 +232,22 @@ def checkout(self, target_branch: str) -> bool: if self.worktree is None: raise RuntimeError("cannot checkout without a worktree") - self._ensure_clean_tracked_files() + ensure_clean_tracked_files(self) existing = {head.name for head in self.dothm.heads} new_branch = target_branch not in existing - current_tracked = self._tracked_paths() - target_state = self._load_branch_data(target_branch) + current_tracked = tracked_paths(self) + target_state = load_branch_data(self, target_branch) - for rel in target_state.data["path"].astype(str): - rel_path = Path(rel) + for _, row in target_state.data.iterrows(): + rel_path = row_to_path(row, target_state.config["data"][0]["fmt"]) if rel_path not in current_tracked and (self.worktree / rel_path).exists(): raise RuntimeError( - f'target tracked path "{rel}" already exists as an untracked file') + f'target tracked path "{rel_path}" already exists as an untracked file') # remove current tracked files from worktree for _, row in self.state.data.iterrows(): - path = self.worktree / row["path"] + path = self.worktree / path_from_row(self, row) if path.exists(): path.unlink() @@ -199,7 +262,7 @@ def checkout(self, target_branch: str) -> bool: # restore files from objects store via hardlinks for _, row in self.state.data.iterrows(): - self.objects.restore(row["sha1"], self.worktree / row["path"]) + self.objects.restore(row["sha1"], self.worktree / path_from_row(self, row)) return True @@ -234,8 +297,8 @@ def add_worktree(self, target_branch: str) -> bool: raise RuntimeError(f'failed to create worktree for branch "{target_branch}": {e}') target_state = linked_dothm.load() - for rel in target_state.data["path"]: - rel_path = Path(rel) + for _, row in target_state.data.iterrows(): + rel_path = row_to_path(row, target_state.config["data"][0]["fmt"]) src = source / rel_path dest = target / rel_path dest.parent.mkdir(parents=True, exist_ok=True) diff --git a/mod/hallmark/repo_config.py b/mod/hallmark/repo_config.py new file mode 100644 index 0000000..5bd87a0 --- /dev/null +++ b/mod/hallmark/repo_config.py @@ -0,0 +1,119 @@ +from __future__ import annotations + +from string import Formatter +from pathlib import Path + + +def ensure_branch_data_spec(config: dict) -> dict: + data = config.get("data") + if not isinstance(data, list) or len(data) != 1 or not isinstance(data[0], dict): + config["data"] = [{}] + return config["data"][0] + + +def branch_data_spec(repo) -> dict: + data = repo.state.config.get("data") + if not isinstance(data, list) or len(data) != 1 or not isinstance(data[0], dict): + raise RuntimeError('branch config must define exactly one entry under "data" in config.yml') + return data[0] + + +def branch_fmt(repo) -> str: + fmt = branch_data_spec(repo).get("fmt") + if not isinstance(fmt, str) or not fmt.strip(): + raise RuntimeError('branch config must define one non-empty data[0].fmt in config.yml') + return fmt + + +def set_branch_fmt(repo, fmt: str) -> None: + set_config(repo, fmt=fmt) + + +def set_config( + repo, + *, + fmt: str | None = None, + remote_name: str | None = None, + remote_url: str | None = None, + encoding_updates: dict[str, str] | None = None, +) -> dict: + config = repo.state.config + + spec = ensure_branch_data_spec(config) + updated_spec = {} + + if fmt is not None: + updated_spec["fmt"] = fmt + elif "fmt" in spec: + updated_spec["fmt"] = spec["fmt"] + + encoding_value = spec.get("encoding") + if encoding_updates: + if not isinstance(encoding_value, dict): + encoding_value = {} + encoding_value = {**encoding_value, **encoding_updates} + if "encoding" in spec or encoding_updates is not None: + updated_spec["encoding"] = encoding_value + + for key, value in spec.items(): + if key not in {"fmt", "encoding"}: + updated_spec[key] = value + config["data"][0] = updated_spec + + if remote_name is not None or remote_url is not None: + remote = config.get("remote") + if not isinstance(remote, dict): + remote = {} + + updated_remote = {} + if remote_name is not None: + updated_remote["name"] = remote_name + elif "name" in remote: + updated_remote["name"] = remote["name"] + + if remote_url is not None: + updated_remote["url"] = remote_url + elif "url" in remote: + updated_remote["url"] = remote["url"] + + for key, value in remote.items(): + if key not in {"name", "url"}: + updated_remote[key] = value + config["remote"] = updated_remote + + return config + + +def branch_encodings(repo) -> list[dict]: + spec = branch_data_spec(repo) + return [spec] if isinstance(spec.get("encoding"), dict) else [] + + +def fmt_fields(fmt: str) -> list[str]: + fields: list[str] = [] + for _, field_name, _, _ in Formatter().parse(fmt): + if field_name and field_name not in fields: + fields.append(field_name) + return fields + + +def coerce_fmt_value(value: str, spec: str): + if not spec: + return value + if spec.endswith("d"): + return int(float(value)) + if spec[-1] in {"f", "F", "g", "G", "e", "E"}: + return float(value) + return value + + +def row_to_path(row, fmt: str) -> Path: + values = {} + for _, field_name, format_spec, _ in Formatter().parse(fmt): + if field_name: + values[field_name] = coerce_fmt_value(str(row[field_name]), format_spec) + return Path(fmt.format(**values)) + + +def path_from_row(repo, row, fmt: str | None = None) -> Path: + return row_to_path(row, fmt or branch_fmt(repo)) diff --git a/mod/hallmark/repo_manifest.py b/mod/hallmark/repo_manifest.py new file mode 100644 index 0000000..9d440fc --- /dev/null +++ b/mod/hallmark/repo_manifest.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +from pathlib import Path + +import parse +import pandas as pd + +from .repo_config import fmt_fields, row_to_path + + +def manifest_frame_from_pf(pf, fmt: str) -> pd.DataFrame: + if pf.empty: + return pd.DataFrame(columns=["sha1", *fmt_fields(fmt)]) + + parser = parse.compile(fmt) + rows = [] + for _, row in pf.iterrows(): + parsed = parser.parse(str(row["path"])) + if parsed is None: + raise RuntimeError(f'failed to parse "{row["path"]}" using branch fmt "{fmt}"') + rows.append({"sha1": row["sha1"], **{k: str(v) for k, v in parsed.named.items()}}) + return pd.DataFrame(rows, columns=["sha1", *fmt_fields(fmt)]) + + +def manifest_map(state) -> dict[str, str]: + if state.data.empty: + return {} + data = state.config.get("data") + if not isinstance(data, list) or len(data) != 1 or not isinstance(data[0], dict): + return {} + fmt = data[0].get("fmt") + if not isinstance(fmt, str) or not fmt.strip(): + return {} + return { + str(row_to_path(row, fmt)): str(row["sha1"]) + for _, row in state.data.iterrows() + } + + +def frame_from_paths(repo, rel_paths: list[Path]): + records = [{"path": str(path)} for path in rel_paths] + pf = repo.paraframe_cls(records, base_path=repo.worktree) + if not pf.empty: + pf["sha1"] = [repo.checksum(repo.worktree / path) for path in rel_paths] + return pf diff --git a/mod/hallmark/repo_state.py b/mod/hallmark/repo_state.py new file mode 100644 index 0000000..7b4ba9e --- /dev/null +++ b/mod/hallmark/repo_state.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +from io import StringIO + +import pandas as pd +import yaml +from git.exc import GitCommandError + +from .state import State + + +def load_branch_config(repo, branch: str) -> dict: + try: + return yaml.safe_load(repo.dothm.git.show(f"{branch}:config.yml")) or {} + except GitCommandError: + return dict(repo.state.config) + + +def load_branch_meta(repo, branch: str) -> dict: + try: + return yaml.safe_load(repo.dothm.git.show(f"{branch}:meta.yml")) or {} + except GitCommandError: + return dict(repo.state.meta) + + +def load_branch_data(repo, branch: str) -> State: + if branch in {head.name for head in repo.dothm.heads}: + data = repo.dothm.git.show(f"{branch}:data.tsv") + frame = State().data if not data.strip() else None + if frame is None: + parsed = pd.read_csv(StringIO(data), sep="\t", dtype=str) + else: + parsed = frame + return State( + load_branch_config(repo, branch), + load_branch_meta(repo, branch), + parsed, + ) + + return State( + dict(repo.state.config), + dict(repo.state.meta), + repo.state.data.copy(), + ) + + +def load_head_state(repo) -> State: + try: + data = repo.dothm.git.show("HEAD:data.tsv") + except GitCommandError: + return State( + dict(repo.state.config), + dict(repo.state.meta), + State().data.copy(), + ) + + if data.strip(): + parsed = pd.read_csv(StringIO(data), sep="\t", dtype=str) + else: + parsed = State().data.copy() + + return State( + load_branch_config(repo, "HEAD"), + load_branch_meta(repo, "HEAD"), + parsed, + ) diff --git a/mod/hallmark/repo_worktree.py b/mod/hallmark/repo_worktree.py new file mode 100644 index 0000000..e8d7594 --- /dev/null +++ b/mod/hallmark/repo_worktree.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +import os +from pathlib import Path + +from .repo_config import branch_fmt, path_from_row + + +def effective_cwd(repo) -> Path: + if repo.worktree is None: + raise RuntimeError("cannot inspect files in a bare repository without a worktree") + + cwd = Path.cwd().resolve() + worktree = Path(repo.worktree).resolve() + try: + cwd.relative_to(worktree) + except ValueError: + return worktree + return cwd + + +def filtered_paraframe(repo, pf): + root = effective_cwd(repo) + worktree = Path(repo.worktree) + if root == worktree: + return pf + + prefix = str(root.relative_to(worktree)) + mask = pf["path"].astype(str).str.startswith(prefix + os.sep) + return pf[mask] + + +def tracked_paths(repo) -> set[Path]: + fmt = branch_fmt(repo) + return {path_from_row(repo, row, fmt) for _, row in repo.state.data.iterrows()} + + +def ensure_clean_tracked_files(repo) -> None: + if repo.worktree is None: + raise RuntimeError("cannot checkout without a worktree") + + for _, row in repo.state.data.iterrows(): + rel_path = path_from_row(repo, row) + path = repo.worktree / rel_path + if not path.exists(): + raise RuntimeError( + f'tracked file "{rel_path}" is missing; commit or restore it before checkout') + if repo.checksum(path) != row["sha1"]: + raise RuntimeError( + f'tracked file "{rel_path}" has uncommitted changes; commit them before checkout') + + if repo.dothm.index.diff("HEAD"): + raise RuntimeError( + "you have uncommitted hallmark state changes — commit them before checkout") diff --git a/mod/hallmark/state.py b/mod/hallmark/state.py index fb5e19a..7d52966 100644 --- a/mod/hallmark/state.py +++ b/mod/hallmark/state.py @@ -18,7 +18,7 @@ import pandas as pd -PERSISTED_COLUMNS = ["sha1", "path"] +COLUMNS = ["sha1"] @dataclass @@ -35,20 +35,41 @@ class State: config: dict = field(default_factory=dict) meta: dict = field(default_factory=dict) data: pd.DataFrame = field( - default_factory=lambda: pd.DataFrame(columns=PERSISTED_COLUMNS) + default_factory=lambda: pd.DataFrame(columns=COLUMNS) ) def update(self, pf): if pf.empty: - incoming = pd.DataFrame(columns=PERSISTED_COLUMNS) + incoming = pd.DataFrame(columns=self.data.columns if len(self.data.columns) else COLUMNS) else: - incoming = pf.loc[:, PERSISTED_COLUMNS].copy() + incoming_columns = ["sha1"] + [ + col for col in pf.columns + if col not in {"sha1", "path"} + ] + incoming = pf.loc[:, incoming_columns].copy() + for column in incoming.columns: + if column != "sha1": + incoming[column] = incoming[column].astype(str) merged = pd.concat([self.data, incoming], ignore_index=True, sort=False) - # If the same path is added again (e.g., file content changed - # and a new sha1 is computed), keep only the newest row for - # that key. - deduped = merged.drop_duplicates(subset=["path"], keep="last") + key_columns = [column for column in merged.columns if column != "sha1"] + if key_columns: + deduped = merged.drop_duplicates(subset=key_columns, keep="last") + else: + deduped = merged + + self.data = deduped.loc[:, ["sha1", *key_columns]] - self.data = deduped.loc[:, PERSISTED_COLUMNS] + def replace(self, pf): + if pf.empty: + self.data = pd.DataFrame(columns=self.data.columns if len(self.data.columns) else COLUMNS) + else: + columns = ["sha1"] + [ + col for col in pf.columns + if col not in {"sha1", "path"} + ] + self.data = pf.loc[:, columns].copy() + for column in self.data.columns: + if column != "sha1": + self.data[column] = self.data[column].astype(str) diff --git a/test/conftest.py b/test/conftest.py index a6995a0..f8fdb84 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -22,10 +22,8 @@ def _write_text_files(root: Path, files: list[str]) -> None: for name in files: (root / name).write_text("test\n", encoding="utf-8") -# Write encoding into config.yml for encoded pf -def _write_config_with_encodings(repo: Repo) -> None: - config = repo.dothm.load_yml("config") or {} - config["encodings"] = [ +def _encoded_data_spec() -> list[dict]: + return [ { "fmt": "a{aspin}_i{i}.h5", "encoding": { @@ -33,8 +31,6 @@ def _write_config_with_encodings(repo: Repo) -> None: }, } ] - repo.dothm.dump_yml(config, "config") - repo.state.config = repo.dothm.load_yml("config") @pytest.fixture(scope="session") @@ -50,7 +46,7 @@ def hallmark_test_suite_dictionary(tmp_path_factory): # Actually write out listed files in the temporary directory _write_text_files(repo_path, Standard_files) _write_text_files(repo_path, Encoded_files) - _write_config_with_encodings(repo) + encoded_specs = _encoded_data_spec() # Create paraframes, glob files, glob pattern and repo behavior objects standard_pf = ParaFrame.parse("a{a}_i{i}.h5", base_path=repo.worktree) @@ -58,7 +54,7 @@ def hallmark_test_suite_dictionary(tmp_path_factory): encoded_pf = ParaFrame.parse( "a{aspin}_i{i}.h5", base_path=repo.worktree, - encodings=repo.state.config.get("encodings", []), + encodings=encoded_specs, encoding=True, ) @@ -71,7 +67,7 @@ def hallmark_test_suite_dictionary(tmp_path_factory): encoded_globbed_files, encoded_glob_pattern = ParaFrame.glob_search( "a{aspin}_i{i}.h5", base_path=repo.worktree, - encodings=repo.state.config.get("encodings", []), + encodings=encoded_specs, encoding=True, return_pattern=True, ) @@ -92,4 +88,4 @@ def hallmark_test_suite_dictionary(tmp_path_factory): "add_result": add_result, "commit_result": commit_result, "repo_path": repo_path, - } \ No newline at end of file + } diff --git a/test/test_cli.py b/test/test_cli.py index 86f3195..1fa355d 100644 --- a/test/test_cli.py +++ b/test/test_cli.py @@ -66,17 +66,126 @@ def test_cli(): assert result.exit_code == 0 assert 'Switched to branch "experiment".' in result.output - Path("b0_i45.h5").write_text("branch data\n", encoding="utf-8") - result = runner.invoke(hallmark, ["add", "b{spin}_i{inc}.h5"]) + Path("a0_i0.h5").unlink() + Path("a0_i30.h5").unlink() + Path("a0_i60.h5").unlink() + Path("a0_i90.h5").unlink() + Path("a0.75_i0.h5").unlink() + Path("a0.75_i30.h5").unlink() + Path("a0.75_i60.h5").unlink() + Path("a0.75_i90.h5").unlink() + Path("a0.975_i0.h5").unlink() + Path("a0.975_i30.h5").unlink() + Path("a0.975_i60.h5").unlink() + Path("a0.975_i90.h5").unlink() + Path("a1_i45.h5").write_text("a1_i45.h5\n", encoding="utf-8") + result = runner.invoke(hallmark, ["add", "."]) assert result.exit_code == 0 result = runner.invoke(hallmark, ["commit", "-m", "Commit experiment"]) assert result.exit_code == 0 result = runner.invoke(hallmark, ["checkout", "main"]) assert result.exit_code == 0 - assert not Path("b0_i45.h5").exists() + assert not Path("a1_i45.h5").exists() Path("a0_i0.h5").write_text("dirty\n", encoding="utf-8") result = runner.invoke(hallmark, ["checkout", "experiment"]) assert result.exit_code != 0 assert "has uncommitted changes" in result.output + + +def test_cli_add_dot_and_explicit_paths(): + runner = CliRunner() + with runner.isolated_filesystem(): + runner.invoke(hallmark, ["init", "repo"]) + + with chdir("repo"): + Path("a0_i0.h5").write_text("a0_i0.h5\n", encoding="utf-8") + Path("a0_i30.h5").write_text("a0_i30.h5\n", encoding="utf-8") + + result = runner.invoke(hallmark, ["add", "a{a}_i{i}.h5"]) + assert result.exit_code == 0 + + Path("a0_i0.h5").unlink() + Path("a1_i45.h5").write_text("a1_i45.h5\n", encoding="utf-8") + result = runner.invoke(hallmark, ["add", "."]) + assert result.exit_code == 0 + manifest = Path(".hm/data.tsv").read_text(encoding="utf-8") + assert "a0_i0.h5" not in manifest + assert "\t1\t45" in manifest or ",1,45" not in manifest + + Path("top1.h5").write_text("top1.h5\n", encoding="utf-8") + Path("top2.h5").write_text("top2.h5\n", encoding="utf-8") + result = runner.invoke(hallmark, ["add", "top1.h5", "top2.h5"]) + assert result.exit_code != 0 + assert "explicit path add is not supported" in result.output + + +def test_cli_status(): + runner = CliRunner() + with runner.isolated_filesystem(): + runner.invoke(hallmark, ["init", "repo"]) + + with chdir("repo"): + Path("a0_i0.h5").write_text("a0_i0.h5\n", encoding="utf-8") + Path("a0_i30.h5").write_text("a0_i30.h5\n", encoding="utf-8") + runner.invoke(hallmark, ["add", "a{a}_i{i}.h5"]) + runner.invoke(hallmark, ["commit", "-m", "Commit test"]) + + Path("a0_i0.h5").write_text("changed\n", encoding="utf-8") + Path("a0_i30.h5").unlink() + Path("untracked.h5").write_text("untracked\n", encoding="utf-8") + + result = runner.invoke(hallmark, ["status"]) + assert result.exit_code == 0 + assert "On branch main" in result.output + assert "Changes not staged for commit:" in result.output + assert "modified: a0_i0.h5" in result.output + assert "deleted: a0_i30.h5" in result.output + assert "Untracked files:" in result.output + assert "untracked.h5" in result.output + + +def test_cli_set_config_and_add_dot(): + runner = CliRunner() + with runner.isolated_filesystem(): + runner.invoke(hallmark, ["init", "repo"]) + + with chdir("repo"): + result = runner.invoke( + hallmark, + [ + "set-config", + "--fmt", "b{a}_i{i}.h5", + "--remote-name", "origin", + "--remote-url", "https://example.com/path", + "--encoding", r"aspin=m([0-9]+(\.[0-9]+)?|\.[0-9]+)", + ], + ) + assert result.exit_code == 0 + assert "Updated hallmark config." in result.output + + Path("b0_i0.h5").write_text("b0_i0.h5\n", encoding="utf-8") + Path("b0_i30.h5").write_text("b0_i30.h5\n", encoding="utf-8") + + result = runner.invoke(hallmark, ["add", "."]) + assert result.exit_code == 0 + + manifest = Path(".hm/data.tsv").read_text(encoding="utf-8") + assert "sha1\ta\ti" in manifest + config = Path(".hm/config.yml").read_text(encoding="utf-8") + assert "fmt: b{a}_i{i}.h5" in config + assert "name: origin" in config + assert "url: https://example.com/path" in config + assert r"aspin: m([0-9]+(\.[0-9]+)?|\.[0-9]+)" in config + + +def test_cli_set_config_rejects_malformed_encoding(): + runner = CliRunner() + with runner.isolated_filesystem(): + runner.invoke(hallmark, ["init", "repo"]) + + with chdir("repo"): + result = runner.invoke(hallmark, ["set-config", "--encoding", "aspin"]) + assert result.exit_code != 0 + assert "FIELD=REGEX" in result.output diff --git a/test/test_hallmark.py b/test/test_hallmark.py index c6137e5..4361d93 100644 --- a/test/test_hallmark.py +++ b/test/test_hallmark.py @@ -153,8 +153,173 @@ def test_repo_add_persists_only_sha1_and_path(tmp_path): assert list(result.columns) == ["path", "a", "i"] persisted = repo.dothm.load_tsv("data") - assert list(persisted.columns) == ["sha1", "path"] - assert sorted(persisted["path"]) == ["a0_i0.h5", "a0_i30.h5"] + assert repo.state.config["data"] == [{"fmt": "a{a}_i{i}.h5", "encoding": None}] + assert list(persisted.columns) == ["sha1", "a", "i"] + assert persisted.to_dict(orient="records") == [ + {"sha1": Repo.checksum(repo.worktree / "a0_i0.h5"), "a": "0", "i": "0"}, + {"sha1": Repo.checksum(repo.worktree / "a0_i30.h5"), "a": "0", "i": "30"}, + ] + + +def test_repo_add_dot_replaces_manifest_with_current_tree(tmp_path): + repo = Repo.init(tmp_path / "repo") + _write_files(repo.worktree, ["a0_i0.h5", "a0_i30.h5"]) + repo.add("a{a}_i{i}.h5") + (repo.worktree / "a1_i45.h5").write_text("a1_i45.h5\n", encoding="utf-8") + + result = repo.add(".") + + assert sorted(result["path"]) == ["a0_i0.h5", "a0_i30.h5", "a1_i45.h5"] + persisted = repo.dothm.load_tsv("data") + assert persisted.to_dict(orient="records") == [ + {"sha1": Repo.checksum(repo.worktree / "a0_i0.h5"), "a": "0", "i": "0"}, + {"sha1": Repo.checksum(repo.worktree / "a0_i30.h5"), "a": "0", "i": "30"}, + {"sha1": Repo.checksum(repo.worktree / "a1_i45.h5"), "a": "1", "i": "45"}, + ] + + +def test_repo_add_dot_removes_deleted_files_from_manifest(tmp_path): + repo = Repo.init(tmp_path / "repo") + _write_files(repo.worktree, ["a0_i0.h5", "a0_i30.h5"]) + repo.add("a{a}_i{i}.h5") + + (repo.worktree / "a0_i0.h5").unlink() + repo.add(".") + + assert repo.state.data.to_dict(orient="records") == [ + {"sha1": Repo.checksum(repo.worktree / "a0_i30.h5"), "a": "0", "i": "30"} + ] + + +def test_repo_add_pattern_keeps_deleted_manifest_rows(tmp_path): + repo = Repo.init(tmp_path / "repo") + _write_files(repo.worktree, ["a0_i0.h5", "a0_i30.h5"]) + repo.add("a{a}_i{i}.h5") + original_sha = repo.state.data.loc[ + (repo.state.data["a"] == "0") & (repo.state.data["i"] == "0"), + "sha1", + ].iloc[0] + + (repo.worktree / "a0_i0.h5").unlink() + repo.add("a{a}_i{i}.h5") + + assert repo.state.data.to_dict(orient="records") == [ + {"sha1": original_sha, "a": "0", "i": "0"}, + {"sha1": Repo.checksum(repo.worktree / "a0_i30.h5"), "a": "0", "i": "30"}, + ] + + +def test_repo_add_paths_is_not_supported_yet(tmp_path): + repo = Repo.init(tmp_path / "repo") + _write_files(repo.worktree, ["a0_i0.h5", "a0_i30.h5", "b0_i45.h5"]) + repo.add("a{a}_i{i}.h5") + + with pytest.raises(RuntimeError, match="explicit path add is not supported"): + repo.add_paths(["b0_i45.h5"]) + + +def test_repo_add_preserves_config_order_and_remote_key(tmp_path): + repo = Repo.init(tmp_path / "repo") + _write_files(repo.worktree, ["a0_i0.h5"]) + + repo.add("a{a}_i{i}.h5") + + config_text = (repo.dothm.path / "config.yml").read_text(encoding="utf-8") + assert "data:\n- fmt: a{a}_i{i}.h5\n encoding: null\n" in config_text + assert "remote: null\n" in config_text + + +def test_repo_set_config_updates_only_requested_fields(tmp_path): + repo = Repo.init(tmp_path / "repo") + + repo.set_config(fmt="b{a}_i{i}.h5", remote_name="origin") + + assert repo.state.config == { + "data": [{"fmt": "b{a}_i{i}.h5", "encoding": None}], + "remote": {"name": "origin"}, + } + + +def test_repo_set_config_preserves_encoding_and_updates_remote(tmp_path): + repo = Repo.init(tmp_path / "repo") + repo.state.config = { + "data": [ + { + "fmt": "{mag:d}a{aspin}_w{win:d}.h5", + "encoding": { + "aspin": r"m([0-9]+(\.[0-9]+)?|\.[0-9]+)" + }, + } + ], + "remote": {"name": "origin"}, + } + repo.dothm.dump(repo.state) + + repo.set_config(fmt="b{a}_i{i}.h5", remote_url="https://example.com/path") + + assert repo.state.config == { + "data": [ + { + "fmt": "b{a}_i{i}.h5", + "encoding": { + "aspin": r"m([0-9]+(\.[0-9]+)?|\.[0-9]+)" + }, + } + ], + "remote": {"name": "origin", "url": "https://example.com/path"}, + } + + +def test_repo_set_config_creates_or_updates_encoding_map(tmp_path): + repo = Repo.init(tmp_path / "repo") + + repo.set_config(encoding_updates={"aspin": r"m([0-9]+(\.[0-9]+)?|\.[0-9]+)"}) + + assert repo.state.config == { + "data": [ + { + "encoding": { + "aspin": r"m([0-9]+(\.[0-9]+)?|\.[0-9]+)" + }, + } + ], + "remote": None, + } + + +def test_repo_status_reports_staged_worktree_and_untracked_changes(tmp_path): + repo = Repo.init(tmp_path / "repo") + _write_files(repo.worktree, ["a0_i0.h5", "a0_i30.h5"]) + repo.add("a{a}_i{i}.h5") + repo.commit("main data") + + (repo.worktree / "a0_i0.h5").write_text("changed\n", encoding="utf-8") + (repo.worktree / "a0_i30.h5").unlink() + (repo.worktree / "extra.h5").write_text("extra\n", encoding="utf-8") + + snapshot = repo.status() + + assert snapshot["branch"] == "main" + assert snapshot["staged"] == {"added": [], "modified": [], "deleted": []} + assert snapshot["worktree"]["modified"] == ["a0_i0.h5"] + assert snapshot["worktree"]["deleted"] == ["a0_i30.h5"] + assert snapshot["untracked"] == ["extra.h5"] + + +def test_repo_status_reports_staged_manifest_changes(tmp_path): + repo = Repo.init(tmp_path / "repo") + _write_files(repo.worktree, ["a0_i0.h5", "a0_i30.h5"]) + repo.add("a{a}_i{i}.h5") + repo.commit("main data") + + (repo.worktree / "a1_i45.h5").write_text("a1_i45.h5\n", encoding="utf-8") + repo.add(".") + + snapshot = repo.status() + + assert snapshot["staged"]["added"] == ["a1_i45.h5"] + assert snapshot["staged"]["modified"] == [] + assert snapshot["staged"]["deleted"] == [] def test_checkout_rewrites_tracked_files_and_shares_objects(tmp_path): @@ -167,23 +332,25 @@ def test_checkout_rewrites_tracked_files_and_shares_objects(tmp_path): assert len(main_objects) == 2 repo.checkout("experiment") - _write_files(repo.worktree, ["b0_i45.h5", "b1_i90.h5"]) - repo.add("b{spin}_i{inc}.h5") + (repo.worktree / "a0_i0.h5").unlink() + (repo.worktree / "a0_i30.h5").unlink() + _write_files(repo.worktree, ["a1_i45.h5", "a1_i90.h5"]) + repo.add(".") repo.commit("experiment data") - experiment_files = sorted(path.name for path in Path(repo.worktree).glob("*.h5")) - assert experiment_files == ["a0_i0.h5", "a0_i30.h5", "b0_i45.h5", "b1_i90.h5"] + experiment_files = sorted(path.name for path in Path(str(repo.worktree)).glob("*.h5")) + assert experiment_files == ["a1_i45.h5", "a1_i90.h5"] repo.checkout("main") - main_files = sorted(path.name for path in Path(repo.worktree).glob("*.h5")) + main_files = sorted(path.name for path in Path(str(repo.worktree)).glob("*.h5")) assert main_files == ["a0_i0.h5", "a0_i30.h5"] objects_after = [p for p in (repo.dothm.path / "objects").rglob("*") if p.is_file()] assert len(objects_after) == 4 repo.checkout("experiment") - roundtrip_files = sorted(path.name for path in Path(repo.worktree).glob("*.h5")) - assert roundtrip_files == ["a0_i0.h5", "a0_i30.h5", "b0_i45.h5", "b1_i90.h5"] + roundtrip_files = sorted(path.name for path in Path(str(repo.worktree)).glob("*.h5")) + assert roundtrip_files == ["a1_i45.h5", "a1_i90.h5"] def test_checkout_leaves_untracked_files(tmp_path): @@ -193,15 +360,16 @@ def test_checkout_leaves_untracked_files(tmp_path): repo.commit("main data") repo.checkout("experiment") - _write_files(repo.worktree, ["b0_i45.h5"]) - repo.add("b{spin}_i{inc}.h5") + (repo.worktree / "a0_i0.h5").unlink() + _write_files(repo.worktree, ["a1_i45.h5"]) + repo.add(".") repo.commit("experiment data") (repo.worktree / "notes.txt").write_text("keep me\n", encoding="utf-8") repo.checkout("main") assert (repo.worktree / "notes.txt").read_text(encoding="utf-8") == "keep me\n" - assert sorted(path.name for path in Path(repo.worktree).glob("*.h5")) == ["a0_i0.h5"] + assert sorted(path.name for path in Path(str(repo.worktree)).glob("*.h5")) == ["a0_i0.h5"] def test_checkout_aborts_on_dirty_tracked_file(tmp_path): @@ -224,12 +392,13 @@ def test_checkout_aborts_on_untracked_path_conflict(tmp_path): repo.commit("main data") repo.checkout("experiment") - _write_files(repo.worktree, ["b0_i45.h5"]) - repo.add("b{spin}_i{inc}.h5") + (repo.worktree / "a0_i0.h5").unlink() + _write_files(repo.worktree, ["a1_i45.h5"]) + repo.add(".") repo.commit("experiment data") repo.checkout("main") - (repo.worktree / "b0_i45.h5").write_text("untracked blocker\n", encoding="utf-8") + (repo.worktree / "a1_i45.h5").write_text("untracked blocker\n", encoding="utf-8") - with pytest.raises(RuntimeError, match='target tracked path "b0_i45.h5" already exists as an untracked file'): + with pytest.raises(RuntimeError, match='target tracked path "a1_i45.h5" already exists as an untracked file'): repo.checkout("experiment") From ca6827664822e8449681ae965021c5b45f830cc7 Mon Sep 17 00:00:00 2001 From: Nayera Abdessalam Date: Mon, 20 Apr 2026 17:58:49 -0700 Subject: [PATCH 14/20] implemented clone feature with downloading files from cyverse --- mod/hallmark/cli.py | 9 +++++--- mod/hallmark/dothm.py | 22 ++++++++++++++---- mod/hallmark/downloader.py | 31 +++++++++++++++++++++++-- mod/hallmark/error.py | 46 ++++++++++++++++++++++++++++++++++++- mod/hallmark/repo.py | 4 ++-- test/test_cli.py | 27 ++++++++++++++++++++++ test/test_downloader.py | 47 ++++++++++++++++++++++++++++++++++++++ 7 files changed, 173 insertions(+), 13 deletions(-) create mode 100644 test/test_downloader.py diff --git a/mod/hallmark/cli.py b/mod/hallmark/cli.py index e5a6f28..e6292dc 100644 --- a/mod/hallmark/cli.py +++ b/mod/hallmark/cli.py @@ -21,7 +21,7 @@ from click import ClickException from git.exc import GitError -from .error import DothmError +from .error import CloneError from .downloader import DownloadProgress from . import Repo # from "__init__.py" @@ -200,5 +200,8 @@ def progress_callback(progress: DownloadProgress): f"Failed to download {results['failed']} file(s)" ) - except (DothmError, GitError) as e: - raise ClickException(f'Clone failed: {e}') \ No newline at end of file + except CloneError as e: + click.echo(str(e), err=True) + raise SystemExit(1) + except GitError as e: + raise ClickException(str(e)) diff --git a/mod/hallmark/dothm.py b/mod/hallmark/dothm.py index 131a2e9..b50606f 100644 --- a/mod/hallmark/dothm.py +++ b/mod/hallmark/dothm.py @@ -23,7 +23,7 @@ import yaml from .state import State -from .error import DothmError +from .error import CloneError, DothmError class Dothm(Repo): @@ -62,7 +62,12 @@ def init(cls, *args, **kwargs) -> "Dothm": return dothm @classmethod - def clone(cls, url: str, to_path: Path | str) -> "Dothm": + def clone( + cls, + url: str, + to_path: Path | str, + display_path: Path | str | None = None, + ) -> "Dothm": to_path = Path(to_path) try: @@ -73,11 +78,18 @@ def clone(cls, url: str, to_path: Path | str) -> "Dothm": required_files = ["config.yml", "meta.yml", "data.tsv"] for file in required_files: if not (dothm.path / file).exists(): - raise DothmError(f'Cloned repository missing required file: {file}') + raise CloneError( + f'Cloned repository missing required file: {file}' + ) return dothm except GitCommandError as e: - raise DothmError(f'Failed to clone from "{url}": {e}') - except DothmError: + raise CloneError.from_git_command( + e, + fallback=f'Failed to clone from "{url}"', + clone_path=to_path, + display_path=display_path, + ) from e + except CloneError: raise def link(self, path: Path | str, branch: str | None = None): diff --git a/mod/hallmark/downloader.py b/mod/hallmark/downloader.py index 2307f05..2cf54f7 100644 --- a/mod/hallmark/downloader.py +++ b/mod/hallmark/downloader.py @@ -53,6 +53,32 @@ class DownloadError(HallmarkError): """Raised when remote data download fails.""" +def _resolve_remote_path(row: pd.Series, data_config: list[dict]) -> Path: + """Resolve a downloadable relative path from state metadata.""" + if "path" in row.index and pd.notna(row["path"]): + return Path(str(row["path"])) + + for entry in data_config: + fmt = entry.get("fmt") + if not fmt: + continue + + try: + return Path(fmt.format(**row.to_dict())) + except KeyError: + continue + except ValueError as exc: + raise DownloadError( + f"Invalid data format {fmt!r} for remote download: {exc}" + ) from exc + + available = ", ".join(map(str, row.index.tolist())) + raise DownloadError( + "Unable to resolve download path from repository metadata. " + f"Available columns: {available}" + ) + + class CyVerseDownloader: """Efficient downloader for CyVerse and HTTP-based remote sources. @@ -339,12 +365,13 @@ def download_remote_data( # Get list of files to download from data.tsv files_to_download = [] data_df = repo.state.data + data_config = repo.state.config.get("data", []) if data_df.empty: return {'succeeded': 0, 'failed': 0, 'total_bytes': 0, 'errors': []} for _, row in data_df.iterrows(): - rel_path = Path(row['path']) + rel_path = _resolve_remote_path(row, data_config) destination = worktree_path / rel_path # Construct remote URL @@ -365,4 +392,4 @@ def download_remote_data( max_workers=max_workers, progress_callback=progress_callback ) as downloader: - return downloader.download_batch(files_to_download) \ No newline at end of file + return downloader.download_batch(files_to_download) diff --git a/mod/hallmark/error.py b/mod/hallmark/error.py index f6ab968..d71f068 100644 --- a/mod/hallmark/error.py +++ b/mod/hallmark/error.py @@ -16,7 +16,9 @@ """Error hierarchy for Hallmark.""" -from git.exc import GitError +from pathlib import Path + +from git.exc import GitError, GitCommandError class HallmarkError(RuntimeError): @@ -25,3 +27,45 @@ class HallmarkError(RuntimeError): class DothmError(HallmarkError, GitError): """Raised for `.hm` repository validation and access failures.""" + +class CloneError(HallmarkError, GitError): + """Raised for hallmark clone failures.""" + + @staticmethod + def _clean_message( + message: str, + clone_path: Path | str | None = None, + display_path: Path | str | None = None, + ) -> str: + text = str(message).strip() + if "fatal:" in text: + text = text[text.index("fatal:"):].strip(" '") + + if clone_path is not None and display_path is not None: + clone_path = Path(clone_path) + display_path = Path(display_path) + candidates = { + str(clone_path), + str(clone_path.resolve()), + } + for candidate in candidates: + text = text.replace(candidate, str(display_path)) + + return text + + @classmethod + def from_git_command( + cls, + error: GitCommandError, + fallback: str | None = None, + clone_path: Path | str | None = None, + display_path: Path | str | None = None, + ) -> "CloneError": + message = error.stderr or fallback or str(error) + return cls( + cls._clean_message( + message, + clone_path=clone_path, + display_path=display_path, + ) + ) diff --git a/mod/hallmark/repo.py b/mod/hallmark/repo.py index 6c05b59..f6cc386 100644 --- a/mod/hallmark/repo.py +++ b/mod/hallmark/repo.py @@ -84,7 +84,7 @@ def init(cls, path: Path | str) -> "Repo": def clone(cls, url: str, path: Path | str) -> "Repo": dothm_path, worktree_path = cls.lwpaths(path) - Dothm.clone(url, dothm_path) + Dothm.clone(url, dothm_path, display_path=path) # Initialize worktree if non-bare if worktree_path: @@ -173,4 +173,4 @@ def checkout(self, target_branch: str) -> bool: dest = target / rel_path dest.parent.mkdir(parents=True, exist_ok=True) copy2(src, dest) - return True \ No newline at end of file + return True diff --git a/test/test_cli.py b/test/test_cli.py index 904a494..28e9998 100644 --- a/test/test_cli.py +++ b/test/test_cli.py @@ -61,3 +61,30 @@ def test_cli(): result = runner.invoke(hallmark, ["commit", "-m", "Commit test"]) assert result.exit_code == 0 assert "Committed staged state changes." in result.output + + +def test_clone_existing_destination_reports_plain_git_stderr(): + runner = CliRunner() + with runner.isolated_filesystem(): + source = Path("source") + result = runner.invoke(hallmark, ["init", str(source)]) + assert result.exit_code == 0 + + target = Path("repo3") + existing_hm = target / ".hm" + existing_hm.mkdir(parents=True) + (existing_hm / "placeholder.txt").write_text("test\n", encoding="utf-8") + + result = runner.invoke( + hallmark, + ["clone", "--no-fetch-data", str(source / ".hm"), str(target)], + ) + + assert result.exit_code != 0 + assert not result.output.startswith("Error:") + assert "stderr:" not in result.output + assert "Clone failed:" not in result.output + assert ( + result.output.strip() + == "fatal: destination path 'repo3' already exists and is not an empty directory." + ) diff --git a/test/test_downloader.py b/test/test_downloader.py new file mode 100644 index 0000000..b7ee6ae --- /dev/null +++ b/test/test_downloader.py @@ -0,0 +1,47 @@ +from pathlib import Path + +import pandas as pd +import pytest + +from hallmark.downloader import DownloadError, _resolve_remote_path + + +def test_resolve_remote_path_uses_explicit_path(): + row = pd.Series({"path": "nested/file.txt", "sha1": "abc"}) + + assert _resolve_remote_path(row, []) == Path("nested/file.txt") + + +def test_resolve_remote_path_builds_path_from_data_format(): + row = pd.Series( + { + "sha1": "abc", + "release": "SR1", + "source": "M87", + "year": 2017, + "doy": 95, + "band": "hi", + "pipeline": "hops", + "step": "netcal", + "type": "StokesI", + } + ) + data_config = [ + { + "fmt": ( + "{release}_{source}_{year}_{doy:03d}_{band}_" + "{pipeline}_{step}_{type}.uvfits" + ) + } + ] + + assert _resolve_remote_path(row, data_config) == Path( + "SR1_M87_2017_095_hi_hops_netcal_StokesI.uvfits" + ) + + +def test_resolve_remote_path_raises_when_no_path_can_be_built(): + row = pd.Series({"sha1": "abc", "release": "SR1"}) + + with pytest.raises(DownloadError, match="Unable to resolve download path"): + _resolve_remote_path(row, [{"fmt": "{missing}.uvfits"}]) From f7c3883989bd577e28f55fe8169ddb293cc30314 Mon Sep 17 00:00:00 2001 From: Nayera Abdessalam Date: Wed, 22 Apr 2026 10:41:02 -0700 Subject: [PATCH 15/20] updated linter workflow with demo --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b5bbddd..9c19027 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,7 +31,7 @@ jobs: python -m pip install -e . - name: Lint with ruff run: | - ruff check --select E,F --exclude ./demos + ruff check --select E,F --exclude ./demo - name: Test with pytest run: | pytest From 29908ace6978bb5805daef801cbabbf15b5f9561 Mon Sep 17 00:00:00 2001 From: Nayera Abdessalam Date: Wed, 22 Apr 2026 10:57:25 -0700 Subject: [PATCH 16/20] fixed 36 linter errors, mostly lines too long --- mod/hallmark/__init__.py | 2 +- mod/hallmark/dothm.py | 24 +++++++++++++----------- mod/hallmark/downloader.py | 9 ++++++--- mod/hallmark/repo.py | 33 ++++++++++++++++++++------------- mod/hallmark/repo_config.py | 6 ++++-- mod/hallmark/repo_manifest.py | 6 ++++-- mod/hallmark/repo_worktree.py | 12 ++++++++---- mod/hallmark/state.py | 6 ++++-- test/test_cli.py | 3 ++- test/test_downloader.py | 6 ++++-- test/test_hallmark.py | 24 ++++++++++++++++-------- 11 files changed, 82 insertions(+), 49 deletions(-) diff --git a/mod/hallmark/__init__.py b/mod/hallmark/__init__.py index b7c63e7..efbd637 100644 --- a/mod/hallmark/__init__.py +++ b/mod/hallmark/__init__.py @@ -18,4 +18,4 @@ from .repo import Repo from .paraframe import ParaFrame -__all__ = ["Repo"] +__all__ = ["Repo", "ParaFrame"] diff --git a/mod/hallmark/dothm.py b/mod/hallmark/dothm.py index 00250f0..bf1e0b7 100644 --- a/mod/hallmark/dothm.py +++ b/mod/hallmark/dothm.py @@ -65,17 +65,19 @@ def init(cls, *args, **kwargs) -> "Dothm": @staticmethod def config_template() -> str: - return """# Edit this file only if your branch needs regex substitutions or a preset remote. -# For simple names, you can just run: hallmark add "a{a}_i{i}.h5" -data: - - - # fmt: "{release}_{source}_{year}_{doy:03d}_{band}.uvfits" - encoding: - # aspin: m([0-9]+(\\.[0-9]+)?|\\.[0-9]+) -remote: - # name: origin - # url: https://example.com/path/to/data/ -""" + return + """Edit this file only if your branch needs regex substitutions or + a preset remote. + # For simple names, you can just run: hallmark add "a{a}_i{i}.h5" + data: + - + # fmt: "{release}_{source}_{year}_{doy:03d}_{band}.uvfits" + encoding: + # aspin: m([0-9]+(\\.[0-9]+)?|\\.[0-9]+) + remote: + # name: origin + # url: https://example.com/path/to/data/ + """ @classmethod def clone( diff --git a/mod/hallmark/downloader.py b/mod/hallmark/downloader.py index 55706cc..a61b9f9 100644 --- a/mod/hallmark/downloader.py +++ b/mod/hallmark/downloader.py @@ -62,7 +62,8 @@ def _resolve_remote_path(row: pd.Series, data_config: list[dict]) -> Path: ) -def _verify_sha1(path: Path, expected_sha1: Optional[str], chunk_size: int = 8192) -> None: +def _verify_sha1(path: Path, expected_sha1: Optional[str], + chunk_size: int = 8192) -> None: if not expected_sha1: return @@ -134,11 +135,13 @@ def download_remote_data( rel_path = _resolve_remote_path(row, data_config) file_url = urljoin(remote_url + "/", str(rel_path)) destination = worktree_path / rel_path - sha1 = str(row["sha1"]) if "sha1" in row.index and pd.notna(row["sha1"]) else None + sha1 = str(row["sha1"]) if "sha1" in row.index and \ + pd.notna(row["sha1"]) else None files_to_download.append((file_url, destination, sha1)) results = {"succeeded": 0, "failed": 0, "total_bytes": 0, "errors": []} - progress = tqdm(total=len(files_to_download), unit="file", disable=not show_progress) + progress = tqdm(total=len(files_to_download), unit="file", + disable=not show_progress) try: with ThreadPoolExecutor(max_workers=max_workers) as executor: diff --git a/mod/hallmark/repo.py b/mod/hallmark/repo.py index cd06590..4659cbb 100644 --- a/mod/hallmark/repo.py +++ b/mod/hallmark/repo.py @@ -22,16 +22,16 @@ from pathlib import Path from typing import Optional -from git.exc import GitCommandError - from .dothm import Dothm from .error import DestinationExistsError from .objects import Objects from .paraframe import ParaFrame -from .repo_config import branch_encodings, branch_fmt, path_from_row, row_to_path, set_branch_fmt, set_config as repo_set_config +from .repo_config import branch_encodings, branch_fmt, path_from_row, row_to_path, \ + set_branch_fmt, set_config as repo_set_config from .repo_manifest import manifest_frame_from_pf, manifest_map from .repo_state import load_branch_data, load_head_state -from .repo_worktree import effective_cwd, ensure_clean_tracked_files, filtered_paraframe, tracked_paths +from .repo_worktree import effective_cwd, ensure_clean_tracked_files, \ + filtered_paraframe, tracked_paths from .state import State from .worktree import Worktree @@ -77,14 +77,15 @@ def __init__(self, path: Path | str) -> None: self.objects = Objects(common) dothm_objects = Path(dothm_path) / "objects" main_objects = common / "objects" - if dothm_objects.resolve() != main_objects.resolve() and not dothm_objects.exists(): + if dothm_objects.resolve() != main_objects.resolve() \ + and not dothm_objects.exists(): dothm_objects.symlink_to(main_objects) @classmethod def init(cls, path: Path | str) -> "Repo": dothm_path, worktree_path = cls.lwpaths(path) dothm = Dothm.init(dothm_path) - (dothm.path / "config.yml").write_text(Dothm.config_template(), encoding="utf-8") + (dothm.path/"config.yml").write_text(Dothm.config_template(),encoding="utf-8") dothm.dump_yml({}, "meta") dothm.dump_tsv(State().data, "data") dothm.index.add(["config.yml", "meta.yml", "data.tsv"]) @@ -97,7 +98,8 @@ def clone(cls, url: str, path: Path | str) -> "Repo": clone_path = Path(path) if clone_path.exists(): raise DestinationExistsError( - f"fatal: destination path '{clone_path}' already exists and is not an empty directory." + f"fatal: destination path '{clone_path}' \ + already exists and is not an empty directory." ) dothm_path, worktree_path = cls.lwpaths(path) @@ -120,7 +122,8 @@ def checksum(path: Path, chunk_size: int = 1024 * 1024) -> str: def add_paths(self, paths: list[Path | str]) -> ParaFrame: raise RuntimeError( - 'explicit path add is not supported while data.tsv stores only sha1 plus fmt fields') + 'explicit path add is not supported while data.tsv ' \ + 'stores only sha1 plus fmt fields') def set_config( self, @@ -239,7 +242,8 @@ def commit(self, msg: str, allow_empty: bool = False) -> bool: if allow_empty or self.dothm.index.diff("HEAD"): for _, row in self.state.data.iterrows(): - self.objects.store(self.worktree / path_from_row(self, row), row["sha1"]) + self.objects.store(self.worktree / path_from_row(self, row), + row["sha1"]) self.dothm.index.commit(msg) return True return False @@ -266,7 +270,8 @@ def checkout(self, target_branch: str) -> bool: rel_path = row_to_path(row, target_state.config["data"][0]["fmt"]) if rel_path not in current_tracked and (self.worktree / rel_path).exists(): raise RuntimeError( - f'target tracked path "{rel_path}" already exists as an untracked file') + f'target tracked path "{rel_path}" \ + already exists as an untracked file') # remove current tracked files from worktree for _, row in self.state.data.iterrows(): @@ -297,7 +302,8 @@ def add_worktree(self, target_branch: str) -> bool: raise ValueError("branch name must be a non-empty string") if self.worktree is None: - raise RuntimeError("cannot add a worktree in a bare repository without a worktree") + raise RuntimeError("cannot add a worktree in a bare " \ + "repository without a worktree") source = Path(self.worktree).resolve() target = source.parent / target_branch @@ -313,10 +319,11 @@ def add_worktree(self, target_branch: str) -> bool: if target_branch in existing_branches: linked_dothm = self.dothm.link(target_dothm, target_branch) else: - self.dothm.git.worktree("add", "-b", target_branch, str(target_dothm)) + self.dothm.git.worktree("add","-b",target_branch,str(target_dothm)) linked_dothm = Dothm(target_dothm) except GitCommandError as e: - raise RuntimeError(f'failed to create worktree for branch "{target_branch}": {e}') + raise RuntimeError(f'failed to create worktree for \ + branch "{target_branch}": {e}') target_state = linked_dothm.load() for _, row in target_state.data.iterrows(): diff --git a/mod/hallmark/repo_config.py b/mod/hallmark/repo_config.py index 5bd87a0..6c012e3 100644 --- a/mod/hallmark/repo_config.py +++ b/mod/hallmark/repo_config.py @@ -14,14 +14,16 @@ def ensure_branch_data_spec(config: dict) -> dict: def branch_data_spec(repo) -> dict: data = repo.state.config.get("data") if not isinstance(data, list) or len(data) != 1 or not isinstance(data[0], dict): - raise RuntimeError('branch config must define exactly one entry under "data" in config.yml') + raise RuntimeError('branch config must define exactly ' \ + 'one entry under "data" in config.yml') return data[0] def branch_fmt(repo) -> str: fmt = branch_data_spec(repo).get("fmt") if not isinstance(fmt, str) or not fmt.strip(): - raise RuntimeError('branch config must define one non-empty data[0].fmt in config.yml') + raise RuntimeError('branch config must define one ' \ + 'non-empty data[0].fmt in config.yml') return fmt diff --git a/mod/hallmark/repo_manifest.py b/mod/hallmark/repo_manifest.py index 9d440fc..72914e7 100644 --- a/mod/hallmark/repo_manifest.py +++ b/mod/hallmark/repo_manifest.py @@ -17,8 +17,10 @@ def manifest_frame_from_pf(pf, fmt: str) -> pd.DataFrame: for _, row in pf.iterrows(): parsed = parser.parse(str(row["path"])) if parsed is None: - raise RuntimeError(f'failed to parse "{row["path"]}" using branch fmt "{fmt}"') - rows.append({"sha1": row["sha1"], **{k: str(v) for k, v in parsed.named.items()}}) + raise RuntimeError(f'failed to parse "{row["path"]}" \ + using branch fmt "{fmt}"') + rows.append({"sha1": row["sha1"], **{k: str(v) + for k, v in parsed.named.items()}}) return pd.DataFrame(rows, columns=["sha1", *fmt_fields(fmt)]) diff --git a/mod/hallmark/repo_worktree.py b/mod/hallmark/repo_worktree.py index e8d7594..6da32de 100644 --- a/mod/hallmark/repo_worktree.py +++ b/mod/hallmark/repo_worktree.py @@ -8,7 +8,8 @@ def effective_cwd(repo) -> Path: if repo.worktree is None: - raise RuntimeError("cannot inspect files in a bare repository without a worktree") + raise RuntimeError("cannot inspect files in a bare repository " \ + "without a worktree") cwd = Path.cwd().resolve() worktree = Path(repo.worktree).resolve() @@ -44,11 +45,14 @@ def ensure_clean_tracked_files(repo) -> None: path = repo.worktree / rel_path if not path.exists(): raise RuntimeError( - f'tracked file "{rel_path}" is missing; commit or restore it before checkout') + f'tracked file "{rel_path}" is missing; commit or \ + restore it before checkout') if repo.checksum(path) != row["sha1"]: raise RuntimeError( - f'tracked file "{rel_path}" has uncommitted changes; commit them before checkout') + f'tracked file "{rel_path}" has uncommitted changes; \ + commit them before checkout') if repo.dothm.index.diff("HEAD"): raise RuntimeError( - "you have uncommitted hallmark state changes — commit them before checkout") + "you have uncommitted hallmark state changes — " \ + "commit them before checkout") diff --git a/mod/hallmark/state.py b/mod/hallmark/state.py index 7d52966..967dd0e 100644 --- a/mod/hallmark/state.py +++ b/mod/hallmark/state.py @@ -40,7 +40,8 @@ class State: def update(self, pf): if pf.empty: - incoming = pd.DataFrame(columns=self.data.columns if len(self.data.columns) else COLUMNS) + incoming = pd.DataFrame(columns=self.data.columns + if len(self.data.columns) else COLUMNS) else: incoming_columns = ["sha1"] + [ col for col in pf.columns @@ -63,7 +64,8 @@ def update(self, pf): def replace(self, pf): if pf.empty: - self.data = pd.DataFrame(columns=self.data.columns if len(self.data.columns) else COLUMNS) + self.data = pd.DataFrame(columns=self.data.columns + if len(self.data.columns) else COLUMNS) else: columns = ["sha1"] + [ col for col in pf.columns diff --git a/test/test_cli.py b/test/test_cli.py index a1b005c..ab9495d 100644 --- a/test/test_cli.py +++ b/test/test_cli.py @@ -240,7 +240,8 @@ def test_clone_existing_destination_reports_plain_git_stderr(): assert "Clone failed:" not in result.output assert ( result.output.strip() - == "fatal: destination path 'repo3' already exists and is not an empty directory." + == "fatal: destination path 'repo3' already exists and " + "is not an empty directory." ) diff --git a/test/test_downloader.py b/test/test_downloader.py index 96b2cab..1f5526b 100644 --- a/test/test_downloader.py +++ b/test/test_downloader.py @@ -4,7 +4,8 @@ import pandas as pd import pytest -from hallmark.downloader import DownloadError, _resolve_remote_path, download_remote_data +from hallmark.downloader import DownloadError, _resolve_remote_path, \ +download_remote_data def test_resolve_remote_path_uses_explicit_path(): @@ -76,7 +77,8 @@ def test_resolve_remote_path_raises_when_no_path_can_be_built(): _resolve_remote_path(row, [{"fmt": "{missing}.uvfits"}]) -def test_download_remote_data_builds_urls_and_destinations_from_fmt(monkeypatch, tmp_path): +def test_download_remote_data_builds_urls_and_destinations_from_fmt( + monkeypatch, tmp_path): captured = {} def fake_download_file(url, destination, sha1, chunk_size=8192): diff --git a/test/test_hallmark.py b/test/test_hallmark.py index 4361d93..2968fa9 100644 --- a/test/test_hallmark.py +++ b/test/test_hallmark.py @@ -328,7 +328,8 @@ def test_checkout_rewrites_tracked_files_and_shares_objects(tmp_path): repo.add("a{a}_i{i}.h5") repo.commit("main data") - main_objects = sorted(p.relative_to(repo.dothm.path) for p in (repo.dothm.path / "objects").rglob("*") if p.is_file()) + main_objects = sorted(p.relative_to(repo.dothm.path) + for p in (repo.dothm.path / "objects").rglob("*") if p.is_file()) assert len(main_objects) == 2 repo.checkout("experiment") @@ -338,18 +339,22 @@ def test_checkout_rewrites_tracked_files_and_shares_objects(tmp_path): repo.add(".") repo.commit("experiment data") - experiment_files = sorted(path.name for path in Path(str(repo.worktree)).glob("*.h5")) + experiment_files = sorted(path.name + for path in Path(str(repo.worktree)).glob("*.h5")) assert experiment_files == ["a1_i45.h5", "a1_i90.h5"] repo.checkout("main") - main_files = sorted(path.name for path in Path(str(repo.worktree)).glob("*.h5")) + main_files = sorted(path.name + for path in Path(str(repo.worktree)).glob("*.h5")) assert main_files == ["a0_i0.h5", "a0_i30.h5"] - objects_after = [p for p in (repo.dothm.path / "objects").rglob("*") if p.is_file()] + objects_after = [p for p in (repo.dothm.path / + "objects").rglob("*") if p.is_file()] assert len(objects_after) == 4 repo.checkout("experiment") - roundtrip_files = sorted(path.name for path in Path(str(repo.worktree)).glob("*.h5")) + roundtrip_files = sorted(path.name + for path in Path(str(repo.worktree)).glob("*.h5")) assert roundtrip_files == ["a1_i45.h5", "a1_i90.h5"] @@ -369,7 +374,8 @@ def test_checkout_leaves_untracked_files(tmp_path): repo.checkout("main") assert (repo.worktree / "notes.txt").read_text(encoding="utf-8") == "keep me\n" - assert sorted(path.name for path in Path(str(repo.worktree)).glob("*.h5")) == ["a0_i0.h5"] + assert sorted(path.name + for path in Path(str(repo.worktree)).glob("*.h5")) == ["a0_i0.h5"] def test_checkout_aborts_on_dirty_tracked_file(tmp_path): @@ -381,7 +387,8 @@ def test_checkout_aborts_on_dirty_tracked_file(tmp_path): repo.checkout("main") (repo.worktree / "a0_i0.h5").write_text("changed\n", encoding="utf-8") - with pytest.raises(RuntimeError, match='tracked file "a0_i0.h5" has uncommitted changes'): + with pytest.raises(RuntimeError, + match='tracked file "a0_i0.h5" has uncommitted changes'): repo.checkout("experiment") @@ -400,5 +407,6 @@ def test_checkout_aborts_on_untracked_path_conflict(tmp_path): (repo.worktree / "a1_i45.h5").write_text("untracked blocker\n", encoding="utf-8") - with pytest.raises(RuntimeError, match='target tracked path "a1_i45.h5" already exists as an untracked file'): + with pytest.raises(RuntimeError, + match='target tracked path "a1_i45.h5" already exists as an untracked file'): repo.checkout("experiment") From 8309801c049dcffc6ccb5e1693d051db17f16474 Mon Sep 17 00:00:00 2001 From: HaydenMarchinek Date: Wed, 22 Apr 2026 11:15:21 -0700 Subject: [PATCH 17/20] Updated dothm and repo to allow tests to pass --- mod/hallmark/dothm.py | 23 ++++++++++++----------- mod/hallmark/repo.py | 8 ++++---- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/mod/hallmark/dothm.py b/mod/hallmark/dothm.py index bf1e0b7..7f71ec1 100644 --- a/mod/hallmark/dothm.py +++ b/mod/hallmark/dothm.py @@ -66,17 +66,18 @@ def init(cls, *args, **kwargs) -> "Dothm": @staticmethod def config_template() -> str: return - """Edit this file only if your branch needs regex substitutions or - a preset remote. - # For simple names, you can just run: hallmark add "a{a}_i{i}.h5" - data: - - - # fmt: "{release}_{source}_{year}_{doy:03d}_{band}.uvfits" - encoding: - # aspin: m([0-9]+(\\.[0-9]+)?|\\.[0-9]+) - remote: - # name: origin - # url: https://example.com/path/to/data/ + """ + Edit this file only if your branch needs regex substitutions or a + preset remote. + For simple names, you can just run: hallmark add "a{a}_i{i}.h5" + data: + - + # fmt: "{release}_{source}_{year}_{doy:03d}_{band}.uvfits" + encoding: + # aspin: m([0-9]+(\\.[0-9]+)?|\\.[0-9]+) + remote: + name: origin + url: https://example.com/path/to/data/ """ @classmethod diff --git a/mod/hallmark/repo.py b/mod/hallmark/repo.py index 4659cbb..d1438e9 100644 --- a/mod/hallmark/repo.py +++ b/mod/hallmark/repo.py @@ -98,8 +98,8 @@ def clone(cls, url: str, path: Path | str) -> "Repo": clone_path = Path(path) if clone_path.exists(): raise DestinationExistsError( - f"fatal: destination path '{clone_path}' \ - already exists and is not an empty directory." + f"fatal: destination path '{clone_path}' already exists " + "and is not an empty directory." ) dothm_path, worktree_path = cls.lwpaths(path) @@ -270,8 +270,8 @@ def checkout(self, target_branch: str) -> bool: rel_path = row_to_path(row, target_state.config["data"][0]["fmt"]) if rel_path not in current_tracked and (self.worktree / rel_path).exists(): raise RuntimeError( - f'target tracked path "{rel_path}" \ - already exists as an untracked file') + f'target tracked path "{rel_path}" already exists ' + "as an untracked file") # remove current tracked files from worktree for _, row in self.state.data.iterrows(): From 7f446fc8bc33b0859b2f2bd60472995dd5f26f1c Mon Sep 17 00:00:00 2001 From: HaydenMarchinek Date: Wed, 22 Apr 2026 11:19:30 -0700 Subject: [PATCH 18/20] Fixed config_template to allow tests to pass --- mod/hallmark/dothm.py | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/mod/hallmark/dothm.py b/mod/hallmark/dothm.py index 7f71ec1..eb55ceb 100644 --- a/mod/hallmark/dothm.py +++ b/mod/hallmark/dothm.py @@ -65,20 +65,17 @@ def init(cls, *args, **kwargs) -> "Dothm": @staticmethod def config_template() -> str: - return - """ - Edit this file only if your branch needs regex substitutions or a - preset remote. - For simple names, you can just run: hallmark add "a{a}_i{i}.h5" - data: - - - # fmt: "{release}_{source}_{year}_{doy:03d}_{band}.uvfits" - encoding: - # aspin: m([0-9]+(\\.[0-9]+)?|\\.[0-9]+) - remote: - name: origin - url: https://example.com/path/to/data/ - """ + return """# Edit this file only if your branch needs regex substitutions. +# For simple names, you can just run: hallmark add "a{a}_i{i}.h5" +data: + - + # fmt: "{release}_{source}_{year}_{doy:03d}_{band}.uvfits" + encoding: + # aspin: m([0-9]+(\\.[0-9]+)?|\\.[0-9]+) +remote: + # name: origin + # url: https://example.com/path/to/data/ +""" @classmethod def clone( From 41b4d02de2e89fd5c63185183ec62b029a94e44d Mon Sep 17 00:00:00 2001 From: HaydenMarchinek Date: Wed, 22 Apr 2026 11:24:29 -0700 Subject: [PATCH 19/20] removed python3.9 functionality --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9c19027..652887e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9","3.10","3.11","3.12","3.13","3.14"] + python-version: ["3.10","3.11","3.12","3.13","3.14"] steps: - uses: actions/checkout@v4 From 3ca0b663d071d03a7c9a4b4aa6f8e85ae71e63bc Mon Sep 17 00:00:00 2001 From: Chi-kwan Chan Date: Wed, 22 Apr 2026 11:47:13 -0700 Subject: [PATCH 20/20] Make compatible with Python 3.9 --- .github/workflows/ci.yml | 6 +++--- mod/hallmark/dothm.py | 15 ++++++++------- mod/hallmark/error.py | 7 ++++--- mod/hallmark/objects.py | 3 ++- mod/hallmark/repo.py | 20 ++++++++++---------- mod/hallmark/repo_config.py | 11 ++++++----- mod/hallmark/worktree.py | 5 +++-- pyproject.toml | 1 + 8 files changed, 37 insertions(+), 31 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 652887e..16556b2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10","3.11","3.12","3.13","3.14"] + python-version: ["3.9", "3.10","3.11","3.12","3.13","3.14"] steps: - uses: actions/checkout@v4 @@ -27,11 +27,11 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install ruff pytest + python -m pip install ruff pytest python -m pip install -e . - name: Lint with ruff run: | ruff check --select E,F --exclude ./demo - name: Test with pytest run: | - pytest + pytest diff --git a/mod/hallmark/dothm.py b/mod/hallmark/dothm.py index eb55ceb..277acbb 100644 --- a/mod/hallmark/dothm.py +++ b/mod/hallmark/dothm.py @@ -17,6 +17,7 @@ from pathlib import Path from functools import cached_property +from typing import Optional, Union from git import GitCommandError, Repo import pandas as pd @@ -81,8 +82,8 @@ def config_template() -> str: def clone( cls, url: str, - to_path: Path | str, - display_path: Path | str | None = None, + to_path: Union[Path, str], + display_path: Optional[Union[Path, str]] = None, ) -> "Dothm": to_path = Path(to_path) @@ -107,7 +108,7 @@ def clone( except CloneError: raise - def link(self, path: Path | str, branch: str | None = None): + def link(self, path: Union[Path, str], branch: Optional[str] = None): cmd = self.git # has its own working directory path = Path(path).resolve() # use absolute path try: @@ -129,16 +130,16 @@ def dump(self, state: State) -> None: self.dump_tsv(state.data, "data") self.index.add(["config.yml", "meta.yml", "data.tsv"]) - def load_yml(self, stem: Path | str) -> dict: + def load_yml(self, stem: Union[Path, str]) -> dict: with open((self.path/stem).with_suffix(".yml"), "r") as f: return yaml.safe_load(f) - def dump_yml(self, data: dict, stem: Path | str) -> None: + def dump_yml(self, data: dict, stem: Union[Path, str]) -> None: with open((self.path/stem).with_suffix(".yml"), "w") as f: yaml.dump(data, f, sort_keys=False) - def load_tsv(self, stem: Path | str) -> pd.DataFrame: + def load_tsv(self, stem: Union[Path, str]) -> pd.DataFrame: return pd.read_csv((self.path/stem).with_suffix(".tsv"), sep="\t", dtype=str) - def dump_tsv(self, data: pd.DataFrame, stem: Path | str) -> None: + def dump_tsv(self, data: pd.DataFrame, stem: Union[Path, str]) -> None: data.to_csv((self.path/stem).with_suffix(".tsv"), sep="\t", index=False) diff --git a/mod/hallmark/error.py b/mod/hallmark/error.py index 0737b4b..59aee4f 100644 --- a/mod/hallmark/error.py +++ b/mod/hallmark/error.py @@ -17,6 +17,7 @@ from pathlib import Path +from typing import Optional, Union from git.exc import GitError, GitCommandError @@ -36,9 +37,9 @@ class CloneError(HallmarkError, GitError): def from_git_command( cls, error: GitCommandError, - fallback: str | None = None, - clone_path: Path | str | None = None, - display_path: Path | str | None = None, + fallback: Optional[str] = None, + clone_path: Optional[Union[Path, str]] = None, + display_path: Optional[Union[Path, str]] = None, ) -> "CloneError": text = str(error.stderr or fallback or error).strip() if "fatal:" in text: diff --git a/mod/hallmark/objects.py b/mod/hallmark/objects.py index cde9945..0c5dd30 100644 --- a/mod/hallmark/objects.py +++ b/mod/hallmark/objects.py @@ -1,10 +1,11 @@ import os import shutil from pathlib import Path +from typing import Union class Objects: - def __init__(self, path: Path | str): + def __init__(self, path: Union[Path, str]): self.root = Path(path) / "objects" def _split_checksum(self, sha1: str) -> Path: diff --git a/mod/hallmark/repo.py b/mod/hallmark/repo.py index d1438e9..03356c7 100644 --- a/mod/hallmark/repo.py +++ b/mod/hallmark/repo.py @@ -20,7 +20,7 @@ from dataclasses import dataclass from hashlib import sha1 from pathlib import Path -from typing import Optional +from typing import Dict, List, Optional, Tuple, Union from .dothm import Dothm from .error import DestinationExistsError @@ -60,13 +60,13 @@ class Repo: worktree: Optional[Worktree] = None @staticmethod - def lwpaths(path: Path | str) -> tuple[Path, Path | None]: + def lwpaths(path: Union[Path, str]) -> Tuple[Path, Optional[Path]]: path = Path(path).resolve() if path.suffix == ".hm": return path, None return path / ".hm", path - def __init__(self, path: Path | str) -> None: + def __init__(self, path: Union[Path, str]) -> None: dothm_path, worktree_path = self.lwpaths(path) self.dothm = Dothm(dothm_path) self.worktree = worktree_path and Worktree(worktree_path) @@ -82,7 +82,7 @@ def __init__(self, path: Path | str) -> None: dothm_objects.symlink_to(main_objects) @classmethod - def init(cls, path: Path | str) -> "Repo": + def init(cls, path: Union[Path, str]) -> "Repo": dothm_path, worktree_path = cls.lwpaths(path) dothm = Dothm.init(dothm_path) (dothm.path/"config.yml").write_text(Dothm.config_template(),encoding="utf-8") @@ -94,7 +94,7 @@ def init(cls, path: Path | str) -> "Repo": return cls(path) @classmethod - def clone(cls, url: str, path: Path | str) -> "Repo": + def clone(cls, url: str, path: Union[Path, str]) -> "Repo": clone_path = Path(path) if clone_path.exists(): raise DestinationExistsError( @@ -120,7 +120,7 @@ def checksum(path: Path, chunk_size: int = 1024 * 1024) -> str: digest.update(block) return digest.hexdigest() - def add_paths(self, paths: list[Path | str]) -> ParaFrame: + def add_paths(self, paths: List[Union[Path, str]]) -> ParaFrame: raise RuntimeError( 'explicit path add is not supported while data.tsv ' \ 'stores only sha1 plus fmt fields') @@ -128,10 +128,10 @@ def add_paths(self, paths: list[Path | str]) -> ParaFrame: def set_config( self, *, - fmt: str | None = None, - remote_name: str | None = None, - remote_url: str | None = None, - encoding_updates: dict[str, str] | None = None, + fmt: Optional[str] = None, + remote_name: Optional[str] = None, + remote_url: Optional[str] = None, + encoding_updates: Optional[Dict[str, str]] = None, ) -> dict: repo_set_config( self, diff --git a/mod/hallmark/repo_config.py b/mod/hallmark/repo_config.py index 6c012e3..49a3ed0 100644 --- a/mod/hallmark/repo_config.py +++ b/mod/hallmark/repo_config.py @@ -2,6 +2,7 @@ from string import Formatter from pathlib import Path +from typing import Dict, Optional def ensure_branch_data_spec(config: dict) -> dict: @@ -34,10 +35,10 @@ def set_branch_fmt(repo, fmt: str) -> None: def set_config( repo, *, - fmt: str | None = None, - remote_name: str | None = None, - remote_url: str | None = None, - encoding_updates: dict[str, str] | None = None, + fmt: Optional[str] = None, + remote_name: Optional[str] = None, + remote_url: Optional[str] = None, + encoding_updates: Optional[Dict[str, str]] = None, ) -> dict: config = repo.state.config @@ -117,5 +118,5 @@ def row_to_path(row, fmt: str) -> Path: return Path(fmt.format(**values)) -def path_from_row(repo, row, fmt: str | None = None) -> Path: +def path_from_row(repo, row, fmt: Optional[str] = None) -> Path: return row_to_path(row, fmt or branch_fmt(repo)) diff --git a/mod/hallmark/worktree.py b/mod/hallmark/worktree.py index 451c3c0..aa2dbd2 100644 --- a/mod/hallmark/worktree.py +++ b/mod/hallmark/worktree.py @@ -16,6 +16,7 @@ from __future__ import annotations from pathlib import Path +from typing import Union class Worktree(type(Path())): @@ -25,7 +26,7 @@ class Worktree(type(Path())): and later consumed by downstream software. """ - def __new__(cls, path: Path | str) -> "Worktree": + def __new__(cls, path: Union[Path, str]) -> "Worktree": path = Path(path).resolve() if path.is_dir(): return super().__new__(cls, path) @@ -35,7 +36,7 @@ def __new__(cls, path: Path | str) -> "Worktree": raise FileNotFoundError(f'Worktree "{path}" not found') @classmethod - def init(cls, path: Path | str) -> "Worktree": + def init(cls, path: Union[Path, str]) -> "Worktree": path = Path(path) path.mkdir(parents=True, exist_ok=True) return cls(path) diff --git a/pyproject.toml b/pyproject.toml index a1d3cfb..68797f4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,6 +6,7 @@ name = "hallmark" dynamic = ["version"] license = "Apache-2.0" description = "Reproducibility is the hallmark of the scientific method" +requires-python = ">=3.9" dependencies =[ "click",