From d3a0fa205d38194b50b5f17089d53f118afda22f Mon Sep 17 00:00:00 2001 From: YaqingSu Date: Mon, 13 Jul 2026 12:05:27 +0800 Subject: [PATCH 1/5] allow py-modindex.html in .gitignore --- .gitignore | 9 + .../html/_modules/hallmark/downloader.html | 275 +++++++ .../html/_modules/hallmark/paraframe.html | 442 +++++++++++ doc/_build/html/_modules/hallmark/repo.html | 695 ++++++++++++++++++ .../html/_modules/hallmark/repo_state.html | 224 ++++++ doc/_build/html/_modules/hallmark/state.html | 214 ++++++ doc/_build/html/_modules/index.html | 105 +++ doc/_build/html/api.html | 545 ++++++++++++++ doc/_build/html/design.html | 231 ++++++ doc/_build/html/genindex.html | 327 ++++++++ doc/_build/html/index.html | 196 +++++ doc/_build/html/py-modindex.html | 145 ++++ doc/_build/html/search.html | 122 +++ doc/_build/html/usecase.html | 221 ++++++ 14 files changed, 3751 insertions(+) create mode 100644 doc/_build/html/_modules/hallmark/downloader.html create mode 100644 doc/_build/html/_modules/hallmark/paraframe.html create mode 100644 doc/_build/html/_modules/hallmark/repo.html create mode 100644 doc/_build/html/_modules/hallmark/repo_state.html create mode 100644 doc/_build/html/_modules/hallmark/state.html create mode 100644 doc/_build/html/_modules/index.html create mode 100644 doc/_build/html/api.html create mode 100644 doc/_build/html/design.html create mode 100644 doc/_build/html/genindex.html create mode 100644 doc/_build/html/index.html create mode 100644 doc/_build/html/py-modindex.html create mode 100644 doc/_build/html/search.html create mode 100644 doc/_build/html/usecase.html diff --git a/.gitignore b/.gitignore index 1b3932c..38f87d8 100644 --- a/.gitignore +++ b/.gitignore @@ -16,4 +16,13 @@ __pycache__/ !*.md !*.rst + +# Ignore Sphinx build output by default _build/ + +# Allow the documentation build directory +!doc/_build/ +!doc/_build/html/ + +# Track only the Python module index +!doc/_build/html/py-modindex.html diff --git a/doc/_build/html/_modules/hallmark/downloader.html b/doc/_build/html/_modules/hallmark/downloader.html new file mode 100644 index 0000000..358476f --- /dev/null +++ b/doc/_build/html/_modules/hallmark/downloader.html @@ -0,0 +1,275 @@ + + + + + + + hallmark.downloader — hallmark 0.2 documentation + + + + + + + + + + + + + + + + + +
+
+
+ + +
+ +

Source code for hallmark.downloader

+"""Remote data file downloader for hallmark repositories."""
+
+from __future__ import annotations
+
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from dataclasses import dataclass
+import hashlib
+from pathlib import Path
+from typing import Optional
+from urllib.parse import urljoin
+
+import pandas as pd
+import requests
+from tqdm import tqdm
+
+from .error import HallmarkError
+from .repo_config import row_to_path
+
+
+

+[docs]
+@dataclass
+class DownloadProgress:
+    """Compatibility wrapper for older callers."""
+
+    filename: str
+    total_bytes: int
+    downloaded_bytes: int = 0
+
+    @property
+    def percent(self) -> float:
+        if self.total_bytes == 0:
+            return 0.0
+        return (self.downloaded_bytes / self.total_bytes) * 100

+
+
+
+

+[docs]
+class DownloadError(HallmarkError):
+    """Raised when remote data download fails."""

+
+
+
+def _resolve_remote_path(row: pd.Series, data_config: list[dict]) -> Path:
+    """Resolve a downloadable relative path from state metadata."""
+    if "path" in row.index and pd.notna(row["path"]):
+        return Path(str(row["path"]))
+
+    for entry in data_config:
+        fmt = entry.get("fmt")
+        if not fmt:
+            continue
+
+        try:
+            return row_to_path(row, fmt)
+        except KeyError:
+            continue
+        except (TypeError, ValueError) as exc:
+            raise DownloadError(
+                f"Invalid data format {fmt!r} for remote download: {exc}"
+            ) from exc
+
+    available = ", ".join(map(str, row.index.tolist()))
+    raise DownloadError(
+        "Unable to resolve download path from repository metadata. "
+        f"Available columns: {available}"
+    )
+
+
+def _verify_sha1(path: Path, expected_sha1: Optional[str], 
+                 chunk_size: int = 8192) -> None:
+    if not expected_sha1:
+        return
+
+    digest = hashlib.sha1()
+    with path.open("rb") as handle:
+        for chunk in iter(lambda: handle.read(chunk_size), b""):
+            digest.update(chunk)
+
+    if digest.hexdigest() != expected_sha1:
+        raise DownloadError(f"Checksum mismatch for {path.name}")
+
+
+def _download_file(
+    url: str,
+    destination: Path,
+    expected_sha1: Optional[str] = None,
+    chunk_size: int = 8192,
+) -> int:
+    """Download one file and return the number of bytes written."""
+    destination.parent.mkdir(parents=True, exist_ok=True)
+    temp_path = destination.with_name(destination.name + ".part")
+    temp_path.unlink(missing_ok=True)
+
+    try:
+        with requests.get(url, stream=True, timeout=(10, 30)) as response:
+            response.raise_for_status()
+            with temp_path.open("wb") as handle:
+                for chunk in response.iter_content(chunk_size=chunk_size):
+                    if chunk:
+                        handle.write(chunk)
+
+        _verify_sha1(temp_path, expected_sha1, chunk_size=chunk_size)
+        size = temp_path.stat().st_size
+        temp_path.replace(destination)
+        return size
+    except requests.RequestException as exc:
+        temp_path.unlink(missing_ok=True)
+        raise DownloadError(f"Failed to download {url}: {exc}") from exc
+    except OSError as exc:
+        temp_path.unlink(missing_ok=True)
+        raise DownloadError(f"Failed to write {destination}: {exc}") from exc
+    except DownloadError:
+        temp_path.unlink(missing_ok=True)
+        raise
+
+
+

+[docs]
+def download_remote_data(
+    repo,
+    worktree_path: Path,
+    max_workers: int = 4,
+    show_progress: bool = False,
+) -> dict:
+    """Download remote data files for a cloned repository."""
+    remote_config = repo.state.config.get("remote", {})
+    if not remote_config:
+        return {"succeeded": 0, "failed": 0, "total_bytes": 0, "errors": []}
+
+    remote_url = remote_config.get("url", "").rstrip("/")
+    if not remote_url:
+        raise DownloadError("Remote URL not configured in config.yml")
+
+    data_df = repo.state.data
+    data_config = repo.state.config.get("data", [])
+    if data_df.empty:
+        return {"succeeded": 0, "failed": 0, "total_bytes": 0, "errors": []}
+
+    files_to_download: list[tuple[str, Path, Optional[str]]] = []
+    for _, row in data_df.iterrows():
+        rel_path = _resolve_remote_path(row, data_config)
+        file_url = urljoin(remote_url + "/", str(rel_path))
+        destination = worktree_path / rel_path
+        sha1 = str(row["sha1"]) if "sha1" in row.index and \
+        pd.notna(row["sha1"]) else None
+        files_to_download.append((file_url, destination, sha1))
+
+    results = {"succeeded": 0, "failed": 0, "total_bytes": 0, "errors": []}
+    progress = tqdm(total=len(files_to_download), unit="file", 
+                    disable=not show_progress)
+
+    try:
+        with ThreadPoolExecutor(max_workers=max_workers) as executor:
+            futures = {
+                executor.submit(_download_file, url, destination, sha1): destination
+                for url, destination, sha1 in files_to_download
+            }
+            for future in as_completed(futures):
+                try:
+                    results["total_bytes"] += future.result()
+                    results["succeeded"] += 1
+                except DownloadError as exc:
+                    results["failed"] += 1
+                    results["errors"].append(str(exc))
+                finally:
+                    progress.update(1)
+    finally:
+        progress.close()
+
+    return results

+
+
+ +
+ +
+
+ +
+
+
+ ©2025, the Hallmark Authors. + + | + Powered by Sphinx 7.3.7 + & Alabaster 0.7.16 + +
+ + + + + + \ No newline at end of file diff --git a/doc/_build/html/_modules/hallmark/paraframe.html b/doc/_build/html/_modules/hallmark/paraframe.html new file mode 100644 index 0000000..b324d70 --- /dev/null +++ b/doc/_build/html/_modules/hallmark/paraframe.html @@ -0,0 +1,442 @@ + + + + + + + hallmark.paraframe — hallmark 0.2 documentation + + + + + + + + + + + + + + + + + +
+
+
+ + +
+ +

Source code for hallmark.paraframe

+# Copyright 2019 Chi-kwan Chan
+# Copyright 2019 Steward Observatory
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Utilities for discovering and parsing parameterized file collections.
+
+This module defines :class:`ParaFrame`, a subclass of
+:class:`pandas.DataFrame` that provides convenient methods for locating,
+parsing, and filtering files whose names follow parameterized naming
+conventions.
+"""
+
+from glob import glob
+from pathlib import Path
+
+import re
+import parse
+import pandas as pd
+import numpy as np
+
+from .helper_functions import find_spec_by_fmt, regex_sub, try_numeric_conversion
+
+
+

+[docs]
+class ParaFrame(pd.DataFrame):
+    """
+    A subclass of :class:`pandas.DataFrame` with additional methods for
+    parameterized file discovery and filtering.
+
+    ``ParaFrame`` behaves like a standard ``pandas.DataFrame`` while
+    providing the following additional functionality:
+
+    * ``__init__``: Initializes the class and stores ``encodings`` and
+      ``base_path`` as metadata.
+    * ``_constructor``: Returns a new ``ParaFrame`` while preserving
+      ``encodings`` and ``base_path``.
+    * ``glob_search``: Finds files matching a parameterized format string.
+    * ``parse``: Builds a ``ParaFrame`` by parsing file paths that match
+      a format pattern.
+    * ``__call__`` and ``filter``: Convenience methods for filtering rows
+      by column values.
+    """
+
+    _metadata = ["encodings", "base_path"]
+
+    def __init__(self, data=None, encodings=None, base_path=None, **kwargs):
+        """
+        Initialize a ``ParaFrame``.
+
+        Args:
+            data: Data used to initialize the ``ParaFrame``.
+            encodings (dict, optional): Filename encoding specifications.
+                Defaults to ``{}``.
+            base_path (Path or str, optional): Root directory associated with
+                the ``ParaFrame``. Defaults to the current working directory.
+            **kwargs: Additional arguments passed to
+                :class:`pandas.DataFrame`.
+        """
+        super().__init__(data, **kwargs)
+        self.encodings = encodings or {}
+        self.base_path = Path(base_path) if base_path is not None else Path.cwd()
+
+    @property
+    def _constructor(self):
+        """
+        Return the constructor used by pandas operations.
+
+        Returns:
+            callable: A constructor that preserves ``encodings`` and
+            ``base_path`` metadata.
+        """
+        def _c(*args, **kwargs):
+            kwargs.setdefault("encodings", self.encodings)
+            kwargs.setdefault("base_path", self.base_path)
+            return ParaFrame(*args, **kwargs)
+        return _c
+
+    def __call__(self, **kwds):
+        """
+        Filter the ``ParaFrame``.
+
+        Equivalent to calling :meth:`filter`.
+
+        Args:
+            **kwds: Column names and values used for filtering.
+
+        Returns:
+            pandas.DataFrame: The filtered ``ParaFrame``.
+        """
+        return self.filter(**kwds)
+
+

+[docs]
+    def filter(self, **kwargs):
+        """
+        Filter a ``ParaFrame`` by matching column values.
+
+        This method filters the current ``ParaFrame`` by applying one or more
+        conditions on its columns. Rows satisfying any of the specified
+        conditions are returned.
+
+        Args:
+            ``**kwargs``: Keyword arguments specifying column names and values to
+            filter by. Values may be scalars or sequences (lists or tuples).
+            Sequence values match any element in the sequence.
+
+        Returns:
+            pandas.DataFrame: A filtered DataFrame containing only the rows
+            that satisfy the requested conditions.
+        """
+        mask = np.zeros(len(self), dtype=bool)
+        for k, v in kwargs.items():
+            if isinstance(v, (tuple, list)):  # looking through the specified conditions
+                mask |= np.isin(np.array(self[k]), np.array(v))
+            else:
+                mask |= np.array(self[k]) == v
+        return self[mask]

+
+
+
+
+
+

+[docs]
+    @classmethod
+    def parse(
+        cls,
+        fmt,
+        *args,
+        encodings=None,
+        base_path=None,
+        debug=False,
+        encoding=False,
+        **kwargs,
+    ):
+        """
+        Build a ``ParaFrame`` by parsing file paths that match a format
+        string.
+
+        Args:
+            fmt (str):
+                Format string containing ``{parameter}`` fields.
+
+            encodings (dict):
+                Encoding specifications from ``config.yml``.
+                Defaults to ``{}``.
+
+            base_path (Path):
+                Root directory to search.
+                Defaults to ``Path.cwd()``.
+
+            debug (bool):
+                If ``True``, prints debugging information.
+                Defaults to ``False``.
+
+            encoding (bool):
+                If ``True``, applies regex encoding.
+                Defaults to ``False``.
+
+        Returns:
+            ParaFrame: A ``ParaFrame`` whose rows correspond to matched files.
+            Parsed parameters are stored as columns together with a ``path`` column.
+        """
+        base_path = Path(base_path) if base_path is not None else Path.cwd()
+
+        yaml_encodings, fmt_g, globbed_files = cls.glob_search(
+            fmt, *args,
+            encodings=encodings,
+            base_path=base_path,
+            debug=debug,
+            encoding=encoding,
+            **kwargs,
+        )
+
+        parser = parse.compile(fmt_g)
+        frame = []
+
+        # Writing the ParaFrame
+        for f in globbed_files:
+            f_short = str(Path(f).relative_to(base_path))
+            if encoding:
+                f_new = regex_sub(f_short, yaml_encodings)
+            else:
+                f_new = f_short
+
+            r = parser.parse(f_new)
+
+            if r is None:
+                print(f'Failed to parse "{f}"')
+            else:
+                frame.append({"path": f_short, **r.named})
+        # attempt to convert each parameter column to numeric
+        # if conversion fails the column stays as string
+        result = cls(frame, encodings=encodings, base_path=base_path)
+        for col in result.columns:
+            result[col] = try_numeric_conversion(result[col])
+        return result

+

+
+
+ +
+ +
+
+ +
+
+
+ ©2025, the Hallmark Authors. + + | + Powered by Sphinx 7.3.7 + & Alabaster 0.7.16 + +
+ + + + + + \ No newline at end of file diff --git a/doc/_build/html/_modules/hallmark/repo.html b/doc/_build/html/_modules/hallmark/repo.html new file mode 100644 index 0000000..7149606 --- /dev/null +++ b/doc/_build/html/_modules/hallmark/repo.html @@ -0,0 +1,695 @@ + + + + + + + hallmark.repo — hallmark 0.2 documentation + + + + + + + + + + + + + + + + + +
+
+
+ + +
+ +

Source code for hallmark.repo

+# Copyright 2025 the Hallmark Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from __future__ import annotations
+
+import os
+from contextlib import contextmanager
+from dataclasses import dataclass
+from hashlib import sha1
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple, Union
+
+from .dothm import Dothm
+from .error import CheckoutError, DestinationExistsError
+from .objects import Objects
+from .paraframe import ParaFrame
+from .repo_config import branch_encodings, branch_fmt, path_from_row, row_to_path, \
+    set_branch_fmt, set_config as repo_set_config
+from .repo_manifest import manifest_frame_from_pf, manifest_map
+from .repo_state import load_branch_data, load_head_state
+from .repo_worktree import effective_cwd, ensure_clean_tracked_files, \
+    filtered_paraframe, tracked_paths
+from .state import State
+from .worktree import Worktree
+
+
+

+[docs]
+@contextmanager
+def chdir(path):
+    '''
+    Temporarily change the current working directory. No Yields. 
+
+    Args:
+        Path: Directory to dwitch to while inside the context.
+    Returns:
+        None.
+    '''
+    old = os.getcwd()
+    os.chdir(path)
+    try:
+        yield
+    finally:
+        os.chdir(old)

+
+
+
+

+[docs]
+@dataclass(init=False)
+class Repo:
+    """
+    Hallmark repository.
+
+    This is the Python API boundary.
+    It loads the in-memory ``State`` from repository ``Dothm``, and
+    potentially populate the ``Worktree``.
+    """
+
+    state: State
+    dothm: Optional[Dothm] = None
+    worktree: Optional[Worktree] = None
+
+

+[docs]
+    @staticmethod
+    def lwpaths(path: Union[Path, str]) -> Tuple[Path, Optional[Path]]:
+        '''
+        Resolve repository and worktree paths.
+
+        Args:
+            path (Path | str): Path to either a worktree or a 
+            ``.hm`` repository.
+
+        Returns:
+            tuple[Path, Path | None]: A ``(dothm_path, worktree_path)`` tuple. 
+            If ``path`` refers to a ``.hm`` directory, ``worktree_path`` is ``None``.
+        '''
+        path = Path(path).resolve()
+        if path.suffix == ".hm":
+            return path, None
+        return path / ".hm", path

+
+
+    def __init__(self, path: Union[Path, str]) -> None:
+        '''
+        Open an existing hallmark repository.
+
+        Args: 
+            path: path to a worktree or '.hm repository'/
+        Returns:
+            none.
+        '''
+        dothm_path, worktree_path = self.lwpaths(path)
+        self.dothm = Dothm(dothm_path)
+        self.worktree = worktree_path and Worktree(worktree_path)
+        self.state = self.dothm.load()
+        self.paraframe_cls = ParaFrame
+
+        common = Path(self.dothm.common_dir).resolve().parent
+        self.objects = Objects(common)
+        dothm_objects = Path(dothm_path) / "objects"
+        main_objects = common / "objects"
+        if dothm_objects.resolve() != main_objects.resolve() \
+        and not dothm_objects.exists():
+            dothm_objects.symlink_to(main_objects)
+
+

+[docs]
+    @classmethod
+    def init(cls, path: Union[Path, str]) -> "Repo":
+        '''
+        Initialize a new hallmark repository.
+
+        Args:
+            paht(paht|string): path to initialize a worktree or ``.hm`` repository.
+
+        Returns:
+            Repo: newly created repository instance
+        '''
+        dothm_path, worktree_path = cls.lwpaths(path)
+        dothm = Dothm.init(dothm_path)
+        (dothm.path/"config.yml").write_text(Dothm.config_template(),encoding="utf-8")
+        dothm.dump_yml({}, "meta")
+        dothm.dump_tsv(State().data, "data")
+        dothm.index.add(["config.yml", "meta.yml", "data.tsv"])
+        Objects(dothm_path)
+        worktree_path and Worktree.init(worktree_path)
+        return cls(path)

+
+
+

+[docs]
+    @classmethod
+    def clone(
+        cls,
+        url: str,
+        path: Union[Path, str],
+        *,
+        fetch_data: bool = True,
+        max_workers: int = 4,
+        show_progress: bool = False,
+    ) -> "Repo":
+        '''
+        Clone a remote hallmark repository. Raises DestinationExistsError
+        if the destination path already exists. Raises DownloadError if 
+        data download is enabled and files fail to download.
+
+        Args:
+            url(string): remote repository URL.
+            path(path|string): destination path for clone.
+            fetch_data (boolean): if true, downloads associated data files. 
+            max_workers (integer): Number of parallel workers for downloading data.
+            show_progress (boolean): wether to display download progress.
+        Returns:
+            Repo: Cloned Repository instance
+        '''
+        clone_path = Path(path)
+        if clone_path.exists():
+            raise DestinationExistsError(
+                f"fatal: destination path '{clone_path}' already exists "
+                "and is not an empty directory."
+            )
+
+        dothm_path, worktree_path = cls.lwpaths(path)
+
+        Dothm.clone(url, dothm_path, display_path=path)
+
+        # Initialize worktree if non-bare
+        if worktree_path:
+            Worktree.init(worktree_path)
+
+        repo = cls(path)
+        repo.download_result = None
+
+        if fetch_data and worktree_path:
+            from .downloader import DownloadError, download_remote_data
+
+            result = download_remote_data(
+                repo,
+                worktree_path,
+                max_workers=max_workers,
+                show_progress=show_progress,
+            )
+            repo.download_result = result
+            if result["failed"]:
+                errors = result.get("errors", [])
+                details = "\n".join(f"  - {error}" for error in errors[:5])
+                remaining = result["failed"] - len(errors[:5])
+                if remaining > 0:
+                    details += f"\n  - ... {remaining} more error(s)"
+                raise DownloadError(
+                    f"Failed to download {result['failed']} file(s):\n"
+                    f"{details}"
+                )
+
+        return repo

+
+
+

+[docs]
+    @staticmethod
+    def checksum(path: Path, chunk_size: int = 1024 * 1024) -> str:
+        '''
+        Computes the SHA1 checksum of a file. 
+
+        Args:
+            path (path): path to the file to hash.
+            chunk_size (integer): size of chunks used for streaming reads.
+        Returns:
+            String: Hexadecimal SHA1 digest of the file contents.
+        '''
+        digest = sha1()
+        with path.open("rb") as f:
+            for block in iter(lambda: f.read(chunk_size), b""):
+                digest.update(block)
+        return digest.hexdigest()

+
+
+

+[docs]
+    def add_paths(self, paths: List[Union[Path, str]]) -> ParaFrame:
+        '''
+        Add explicit file paths to the repository index. Raises RuntimeError.
+        Operation not supported in Hallmark.
+        '''
+        raise RuntimeError(
+            'explicit path add is not supported while data.tsv ' \
+            'stores only sha1 plus fmt fields')

+
+
+

+[docs]
+    def set_config(
+        self,
+        *,
+        fmt: Optional[str] = None,
+        remote_name: Optional[str] = None,
+        remote_url: Optional[str] = None,
+        encoding_updates: Optional[Dict[str, str]] = None,
+    ) -> dict:
+        """
+        Update repository configuration values.
+
+        Args:
+            fmt (str, optional): Data format specification.
+            remote_name (str, optional): Name of the remote repository.
+            remote_url (str, optional): URL of the remote repository.
+            encoding_updates (dict[str, str], optional): Updates to encoding rules.
+
+        Returns:
+            dict: Updated configuration dictionary.
+        """
+        repo_set_config(
+            self,
+            fmt=fmt,
+            remote_name=remote_name,
+            remote_url=remote_url,
+            encoding_updates=encoding_updates,
+        )
+        self.dothm.dump(self.state)
+        return self.state.config

+
+
+

+[docs]
+    def status(self) -> dict[str, object]:
+        """
+        Return repository status information. Includes staged changes, 
+        orktree modifications, deletions, and untracked files.
+
+        Args: 
+            none?
+
+        Returns:
+            dict[str, object]: Status summary including:
+            - branch (str)
+            - staged changes (dict)
+            - worktree changes (dict)
+            - untracked files (list[str])
+        """
+        head_state = load_head_state(self)
+        head_map = manifest_map(head_state)
+        staged_map = manifest_map(self.state)
+        state_changes = sorted({
+            diff.a_path or diff.b_path
+            for diff in self.dothm.index.diff("HEAD")
+            if diff.a_path or diff.b_path
+        })
+
+        staged_added = sorted(path for path in staged_map if path not in head_map)
+        staged_deleted = sorted(path for path in head_map if path not in staged_map)
+        staged_modified = sorted(
+            path for path in staged_map
+            if path in head_map and staged_map[path] != head_map[path]
+        )
+
+        worktree_modified: list[str] = []
+        worktree_deleted: list[str] = []
+        tracked_paths = set(staged_map)
+
+        if self.worktree is not None:
+            for path, staged_sha1 in staged_map.items():
+                full_path = self.worktree / path
+                if not full_path.exists():
+                    worktree_deleted.append(path)
+                elif self.checksum(full_path) != staged_sha1:
+                    worktree_modified.append(path)
+
+            untracked = sorted(
+                str(path.relative_to(self.worktree))
+                for path in effective_cwd(self).rglob("*")
+                if path.is_file()
+                and ".hm" not in path.relative_to(self.worktree).parts
+                and str(path.relative_to(self.worktree)) not in tracked_paths
+            )
+        else:
+            untracked = []
+
+        return {
+            "branch": self.dothm.active_branch.name,
+            "staged": {
+                "state": state_changes,
+                "added": staged_added,
+                "modified": staged_modified,
+                "deleted": staged_deleted,
+            },
+            "worktree": {
+                "modified": sorted(worktree_modified),
+                "deleted": sorted(worktree_deleted),
+            },
+            "untracked": untracked,
+        }

+
+
+

+[docs]
+    def add(self, fstr: str, encoding: bool = False) -> ParaFrame:
+        '''
+        Stage files or updated repository indecing from the worktree.
+
+        Args:
+            fstr (string): Format string or "." for full directory scan.
+            encoding (boolean): Whether to apply encoding rules.
+        Returns:
+            paraframe Parsed and filtered file index (without checksums).
+        '''
+        if self.worktree is None:
+            raise RuntimeError(
+                "cannot add files in a bare repository without a worktree")
+
+        if fstr == ".":
+            fmt = branch_fmt(self)
+            with chdir(self.worktree):
+                pf = ParaFrame.parse(
+                    fmt,
+                    base_path=self.worktree,
+                    encodings=branch_encodings(self) if encoding else None,
+                    encoding=encoding,
+                )
+            pf = filtered_paraframe(self, pf)
+            if not pf.empty:
+                pf["sha1"] = [
+                    self.checksum(self.worktree / Path(path))
+                    for path in pf["path"].astype(str)
+                ]
+            self.state.replace(manifest_frame_from_pf(pf, fmt))
+            self.dothm.dump(self.state)
+            return pf.drop(columns=["sha1"], errors="ignore")
+
+        try:
+            previous_fmt = branch_fmt(self)
+        except RuntimeError:
+            previous_fmt = None
+
+        set_branch_fmt(self, fstr)
+
+        with chdir(self.worktree):
+            pf = ParaFrame.parse(
+                fstr,
+                base_path = self.worktree,
+                encodings = branch_encodings(self)
+                            if encoding else None,
+                encoding = encoding,
+                )
+
+        if not pf.empty:
+            pf["sha1"] = [
+                self.checksum(self.worktree / Path(path))
+                for path in pf["path"].astype(str)
+            ]
+
+        manifest = manifest_frame_from_pf(pf, fstr)
+        if previous_fmt is None or previous_fmt != fstr:
+            self.state.replace(manifest)
+        else:
+            self.state.update(manifest)
+        self.dothm.dump(self.state)
+        return pf.drop(columns=["sha1"], errors="ignore")

+
+
+

+[docs]
+    def commit(self, msg: str, allow_empty: bool = False) -> bool:
+        '''
+        Commit staged changes to the repository. Raises ValueError if commit
+        message is empty or invalid. 
+
+        Args:
+            msg (string): commit message.
+            allow_empty (boolean): Allow comitting even if no changes exists.
+        Returns:
+            boolean: True if a commit was created, false otherwise. 
+        '''
+        if not isinstance(msg, str) or not msg.strip():
+            raise ValueError("commit message must be a non-empty string")
+
+        if allow_empty or self.dothm.index.diff("HEAD"):
+            for _, row in self.state.data.iterrows():
+                self.objects.store(self.worktree / path_from_row(self, row), 
+                                   row["sha1"])
+            self.dothm.index.commit(msg)
+            return True
+        return False

+
+
+

+[docs]
+    def log(self) -> str:
+        '''
+        Return commit history log.
+
+        Returns:
+            string: Git log output, or an empty string if no valid HEAD exists.
+        '''
+        if not self.dothm.head.is_valid():
+            return ""
+        return self.dothm.git.log()

+
+
+

+[docs]
+    def branches(self) -> dict[str, object]:
+        '''
+        List repository branches. 
+
+        Returns:
+            dictionary[string, object]: Dictionary containing:
+                - ``current`` (string): Active branch name
+                - ``names``: All branch names
+        '''
+        current = self.dothm.active_branch.name
+        names = sorted(head.name for head in self.dothm.heads)
+        return {"current": current, "names": names}

+
+
+

+[docs]
+    def checkout(self, target_branch: str) -> bool:
+        '''
+        Switch to a different branch and update the worktree. Raises ValueError if
+        branch name is invalid. Raises CheckoutError if the workign directoary
+        is not clean or checkout can't be completed safely.
+
+        Args:
+            target_branch (string): Branch to switch to.
+        Returns: 
+            boolean: True if checkout succeeds.
+
+
+        '''
+        if not isinstance(target_branch, str) or not target_branch.strip():
+            raise ValueError("branch name must be a non-empty string")
+
+        if self.worktree is None:
+            raise CheckoutError("cannot checkout without a worktree")
+        ensure_clean_tracked_files(self)
+
+        existing = {head.name for head in self.dothm.heads}
+        new_branch = target_branch not in existing
+        current_tracked = tracked_paths(self)
+        target_state = load_branch_data(self, target_branch)
+        target_fmt = target_state.config["data"][0]["fmt"]
+        target_tracked = {
+            row_to_path(row, target_fmt)
+            for _, row in target_state.data.iterrows()
+        }
+
+        for _, row in target_state.data.iterrows():
+            rel_path = row_to_path(row, target_fmt)
+            target_path = self.worktree / rel_path
+            if rel_path not in current_tracked and target_path.exists():
+                if self.checksum(target_path) != row["sha1"]:
+                    raise CheckoutError(
+                        f'target tracked path "{rel_path}" already exists '
+                        "as an untracked file")
+
+        # switch .hm branch
+        if new_branch:
+            self.dothm.git.checkout("-b", target_branch)
+        else:
+            self.dothm.git.checkout(target_branch)
+
+        # reload state from the new branch
+        self.state = self.dothm.load()
+
+        removed_paths = []
+        for rel_path in sorted(current_tracked - target_tracked, reverse=True):
+            path = self.worktree / rel_path
+            if path.exists():
+                path.unlink()
+                removed_paths.append(path)
+
+        # restore files from objects store via hardlinks
+        for _, row in self.state.data.iterrows():
+            self.objects.restore(row["sha1"], self.worktree / path_from_row(self, row))
+
+        for path in removed_paths:
+            parent = path.parent
+            while parent != self.worktree and parent.exists():
+                try:
+                    parent.rmdir()
+                except OSError:
+                    break
+                parent = parent.parent
+
+        return True

+
+
+

+[docs]
+    def add_worktree(self, target_branch: str) -> bool:
+        '''
+        Create or link a new worktree for a branch. Raises ValueError if branch name is invalid.
+        Rasies Runtime error if called in a bare repository or worktree creation fails.
+        
+        Args:
+            target_branch (string): Name of the branch to attach.
+        Returns:
+            boolean: True if the worktree was successfully created.
+        '''
+        from shutil import copy2
+        from git.exc import GitCommandError
+
+        if not isinstance(target_branch, str) or not target_branch.strip():
+            raise ValueError("branch name must be a non-empty string")
+
+        if self.worktree is None:
+            raise RuntimeError("cannot add a worktree in a bare " \
+            "repository without a worktree")
+
+        source = Path(self.worktree).resolve()
+        target = source.parent / target_branch
+        target_dothm = target / ".hm"
+
+        linked_dothm = None
+        if target_dothm.exists():
+            linked_dothm = Dothm(target_dothm)
+        else:
+            target.mkdir(parents=True, exist_ok=True)
+            existing_branches = {head.name for head in self.dothm.heads}
+            try:
+                if target_branch in existing_branches:
+                    linked_dothm = self.dothm.link(target_dothm, target_branch)
+                else:
+                    self.dothm.git.worktree("add","-b",target_branch,str(target_dothm))
+                    linked_dothm = Dothm(target_dothm)
+            except GitCommandError as e:
+                raise RuntimeError(f'failed to create worktree for \
+                                    branch "{target_branch}": {e}')
+
+            target_state = linked_dothm.load()
+            for _, row in target_state.data.iterrows():
+                rel_path = row_to_path(row, target_state.config["data"][0]["fmt"])
+                src = source / rel_path
+                dest = target / rel_path
+                dest.parent.mkdir(parents=True, exist_ok=True)
+                copy2(src, dest)
+        return True

+

+
+
+ +
+ +
+
+ +
+
+
+ ©2025, the Hallmark Authors. + + | + Powered by Sphinx 7.3.7 + & Alabaster 0.7.16 + +
+ + + + + + \ No newline at end of file diff --git a/doc/_build/html/_modules/hallmark/repo_state.html b/doc/_build/html/_modules/hallmark/repo_state.html new file mode 100644 index 0000000..5f1fb80 --- /dev/null +++ b/doc/_build/html/_modules/hallmark/repo_state.html @@ -0,0 +1,224 @@ + + + + + + + hallmark.repo_state — hallmark 0.2 documentation + + + + + + + + + + + + + + + + + +
+
+
+ + +
+ +

Source code for hallmark.repo_state

+from __future__ import annotations
+
+from io import StringIO
+
+import pandas as pd
+import yaml
+from git.exc import GitCommandError
+
+from .state import State
+
+
+

+[docs]
+def load_branch_config(repo, branch: str) -> dict:
+    '''
+    Load the configuration for a specific branch
+
+    Args: 
+        repo (Repo): repository object containing Git and state information/
+        branch (String): Name of the branch whose configuration should be loaded
+    Returns:
+        (dict) A dictionary contaiing the branch configuration. Returns the 
+        parsed contents of ``config.yml`` from the specified branch when
+        available, otherwise returns a copy of ``repo.state.config``.
+    '''
+    try:
+        return yaml.safe_load(repo.dothm.git.show(f"{branch}:config.yml")) or {}
+    except GitCommandError:
+        return dict(repo.state.config)

+
+
+
+

+[docs]
+def load_branch_meta(repo, branch: str) -> dict:
+    '''
+    Load the metadata for a specific branch
+    
+    Args: 
+        repo (Repo): repository object
+        branch (String): Name of the branch
+    Returns:
+        The contents of ``meta.yml`` from the specified branch. If 
+        metadata can't be loaded, returns ``repo.state.meta``.
+    '''
+    try:
+        return yaml.safe_load(repo.dothm.git.show(f"{branch}:meta.yml")) or {}
+    except GitCommandError:
+        return dict(repo.state.meta)

+
+
+
+

+[docs]
+def load_branch_data(repo, branch: str) -> State:
+    '''
+    Load the state associated with a branch
+
+    Args: 
+        repo (Repo): repository object
+        branch (String): branch name
+    Returns:
+        (State) A ``State`` consturcted from the branch's ``config.yml``,
+        ``meta.yml``, and ``data.tsv`` files. If the branch does
+        not exist, returns a copy of the current repository state.
+    '''
+    if branch in {head.name for head in repo.dothm.heads}:
+        data = repo.dothm.git.show(f"{branch}:data.tsv")
+        frame = State().data if not data.strip() else None
+        if frame is None:
+            parsed = pd.read_csv(StringIO(data), sep="\t", dtype=str)
+        else:
+            parsed = frame
+        return State(
+            load_branch_config(repo, branch),
+            load_branch_meta(repo, branch),
+            parsed,
+        )
+
+    return State(
+        dict(repo.state.config),
+        dict(repo.state.meta),
+        repo.state.data.copy(),
+    )

+
+
+
+

+[docs]
+def load_head_state(repo) -> State:
+    '''
+    Load the state stored at ``Head``.
+
+    Args:
+        repo (Repo): Repository object.
+
+    Returns:
+        (State) A ``State`` constructed from the ``config.yml``, ``meta.yml``, 
+        and``data.tsv`` files at ``HEAD``. If no state can be loaded 
+        from ``HEAD``, returns a state with the current configuration 
+        and metadata and an empty data table.
+    '''
+    try:
+        data = repo.dothm.git.show("HEAD:data.tsv")
+    except GitCommandError:
+        return State(
+            dict(repo.state.config),
+            dict(repo.state.meta),
+            State().data.copy(),
+        )
+
+    if data.strip():
+        parsed = pd.read_csv(StringIO(data), sep="\t", dtype=str)
+    else:
+        parsed = State().data.copy()
+
+    return State(
+        load_branch_config(repo, "HEAD"),
+        load_branch_meta(repo, "HEAD"),
+        parsed,
+    )

+
+
+ +
+ +
+
+ +
+
+
+ ©2025, the Hallmark Authors. + + | + Powered by Sphinx 7.3.7 + & Alabaster 0.7.16 + +
+ + + + + + \ No newline at end of file diff --git a/doc/_build/html/_modules/hallmark/state.html b/doc/_build/html/_modules/hallmark/state.html new file mode 100644 index 0000000..7d60668 --- /dev/null +++ b/doc/_build/html/_modules/hallmark/state.html @@ -0,0 +1,214 @@ + + + + + + + hallmark.state — hallmark 0.2 documentation + + + + + + + + + + + + + + + + + +
+
+
+ + +
+ +

Source code for hallmark.state

+# Copyright 2025 the Hallmark Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from dataclasses import dataclass, field
+
+import pandas as pd
+
+
+COLUMNS = ["sha1"]
+
+
+

+[docs]
+@dataclass
+class State:
+    """
+    In-memory Hallmark state database.
+
+    Attributes:
+        config: Repository configuration values.
+        meta: Repository metadata.
+        data: Tabular file index containing indexed object checksums
+        (``sha1``) and associated metadata.
+    """
+
+    config:    dict         = field(default_factory=dict)
+    meta:      dict         = field(default_factory=dict)
+    data:      pd.DataFrame = field(
+        default_factory=lambda: pd.DataFrame(columns=COLUMNS)
+    )
+
+

+[docs]
+    def update(self, pf):
+        """
+        Merge ``ParaFrame`` rows into the state database.
+
+        Existing rows with matching keys are updated, while new rows are
+        appended.
+
+        Args:
+            pf (ParaFrame): ``ParaFrame`` containing rows to add or update.
+
+        Returns:
+            None.
+        """
+        if pf.empty:
+            incoming = pd.DataFrame(columns=self.data.columns 
+                                if len(self.data.columns) else COLUMNS)
+        else:
+            incoming_columns = ["sha1"] + [
+                col for col in pf.columns
+                if col not in {"sha1", "path"}
+            ]
+            incoming = pf.loc[:, incoming_columns].copy()
+            for column in incoming.columns:
+                if column != "sha1":
+                    incoming[column] = incoming[column].astype(str)
+        # Merge the incoming rows with the existing state.
+        merged = pd.concat([self.data, incoming], ignore_index=True, sort=False)
+
+        key_columns = [column for column in merged.columns if column != "sha1"]
+        if key_columns:
+            # Remove duplicate entries while keeping the most recent row.
+            deduped = merged.drop_duplicates(subset=key_columns, keep="last")
+        else:
+            deduped = merged
+
+        self.data = deduped.loc[:, ["sha1", *key_columns]]

+
+
+

+[docs]
+    def replace(self, pf):
+        """
+        Replace the contents of the state database.
+
+        Existing rows are discarded and replaced with the rows from the
+        provided ``ParaFrame``.
+
+        Args:
+            pf (ParaFrame): ``ParaFrame`` containing the replacement rows.
+
+        Returns:
+            None.
+        """
+        if pf.empty:
+            self.data = pd.DataFrame(columns=self.data.columns 
+                                     if len(self.data.columns) else COLUMNS)
+        else:
+            columns = ["sha1"] + [
+                col for col in pf.columns
+                if col not in {"sha1", "path"}
+            ]
+            self.data = pf.loc[:, columns].copy()
+            for column in self.data.columns:
+                if column != "sha1":
+                    self.data[column] = self.data[column].astype(str)

+

+
+
+ +
+ +
+
+ +
+
+
+ ©2025, the Hallmark Authors. + + | + Powered by Sphinx 7.3.7 + & Alabaster 0.7.16 + +
+ + + + + + \ No newline at end of file diff --git a/doc/_build/html/_modules/index.html b/doc/_build/html/_modules/index.html new file mode 100644 index 0000000..21e3b42 --- /dev/null +++ b/doc/_build/html/_modules/index.html @@ -0,0 +1,105 @@ + + + + + + + Overview: module code — hallmark 0.2 documentation + + + + + + + + + + + + + + + + + +
+
+
+ + +
+ +

All modules for which code is available

+ + +
+ +
+
+ +
+
+
+ ©2025, the Hallmark Authors. + + | + Powered by Sphinx 7.3.7 + & Alabaster 0.7.16 + +
+ + + + + + \ No newline at end of file diff --git a/doc/_build/html/api.html b/doc/_build/html/api.html new file mode 100644 index 0000000..e9ba203 --- /dev/null +++ b/doc/_build/html/api.html @@ -0,0 +1,545 @@ + + + + + + + + API Reference — hallmark 0.2 documentation + + + + + + + + + + + + + + + + + + +
+
+
+ + +
+ +
+

API Reference

+
+

Repository State

+

# Example: This tells Sphinx to import hallmark.repo_state and extract the module +# and function docstrings.

+
+
+class hallmark.repo.Repo(path: Path | str)[source]
+

Hallmark repository.

+

This is the Python API boundary. +It loads the in-memory State from repository Dothm, and +potentially populate the Worktree.

+
+
+add(fstr: str, encoding: bool = False) → ParaFrame[source]
+

Stage files or updated repository indecing from the worktree.

+
+
Args:

fstr (string): Format string or “.” for full directory scan. +encoding (boolean): Whether to apply encoding rules.

+
+
Returns:

paraframe Parsed and filtered file index (without checksums).

+
+
+
+ +
+
+add_paths(paths: List[Path | str]) → ParaFrame[source]
+

Add explicit file paths to the repository index. Raises RuntimeError. +Operation not supported in Hallmark.

+
+ +
+
+add_worktree(target_branch: str) → bool[source]
+

Create or link a new worktree for a branch. Raises ValueError if branch name is invalid. +Rasies Runtime error if called in a bare repository or worktree creation fails.

+
+
Args:

target_branch (string): Name of the branch to attach.

+
+
Returns:

boolean: True if the worktree was successfully created.

+
+
+
+ +
+
+branches() → dict[str, object][source]
+

List repository branches.

+
+
Returns:
+
dictionary[string, object]: Dictionary containing:
    +

  • current (string): Active branch name

    • +

  • names: All branch names

    • +
+
+
+
+
+
+ +
+
+checkout(target_branch: str) → bool[source]
+

Switch to a different branch and update the worktree. Raises ValueError if +branch name is invalid. Raises CheckoutError if the workign directoary +is not clean or checkout can’t be completed safely.

+
+
Args:

target_branch (string): Branch to switch to.

+
+
Returns:

boolean: True if checkout succeeds.

+
+
+
+ +
+
+static checksum(path: Path, chunk_size: int = 1048576) → str[source]
+

Computes the SHA1 checksum of a file.

+
+
Args:

path (path): path to the file to hash. +chunk_size (integer): size of chunks used for streaming reads.

+
+
Returns:

String: Hexadecimal SHA1 digest of the file contents.

+
+
+
+ +
+
+classmethod clone(url: str, path: Path | str, *, fetch_data: bool = True, max_workers: int = 4, show_progress: bool = False) → Repo[source]
+

Clone a remote hallmark repository. Raises DestinationExistsError +if the destination path already exists. Raises DownloadError if +data download is enabled and files fail to download.

+
+
Args:

url(string): remote repository URL. +path(path|string): destination path for clone. +fetch_data (boolean): if true, downloads associated data files. +max_workers (integer): Number of parallel workers for downloading data. +show_progress (boolean): wether to display download progress.

+
+
Returns:

Repo: Cloned Repository instance

+
+
+
+ +
+
+commit(msg: str, allow_empty: bool = False) → bool[source]
+

Commit staged changes to the repository. Raises ValueError if commit +message is empty or invalid.

+
+
Args:

msg (string): commit message. +allow_empty (boolean): Allow comitting even if no changes exists.

+
+
Returns:

boolean: True if a commit was created, false otherwise.

+
+
+
+ +
+
+classmethod init(path: Path | str) → Repo[source]
+

Initialize a new hallmark repository.

+
+
Args:

paht(paht|string): path to initialize a worktree or .hm repository.

+
+
Returns:

Repo: newly created repository instance

+
+
+
+ +
+
+log() → str[source]
+

Return commit history log.

+
+
Returns:

string: Git log output, or an empty string if no valid HEAD exists.

+
+
+
+ +
+
+static lwpaths(path: Path | str) → Tuple[Path, Path | None][source]
+

Resolve repository and worktree paths.

+
+
Args:

path (Path | str): Path to either a worktree or a +.hm repository.

+
+
Returns:

tuple[Path, Path | None]: A (dothm_path, worktree_path) tuple. +If path refers to a .hm directory, worktree_path is None.

+
+
+
+ +
+
+set_config(*, fmt: str | None = None, remote_name: str | None = None, remote_url: str | None = None, encoding_updates: Dict[str, str] | None = None) → dict[source]
+

Update repository configuration values.

+
+
Args:

fmt (str, optional): Data format specification. +remote_name (str, optional): Name of the remote repository. +remote_url (str, optional): URL of the remote repository. +encoding_updates (dict[str, str], optional): Updates to encoding rules.

+
+
Returns:

dict: Updated configuration dictionary.

+
+
+
+ +
+
+status() → dict[str, object][source]
+

Return repository status information. Includes staged changes, +orktree modifications, deletions, and untracked files.

+
+
Args:

none?

+
+
Returns:

dict[str, object]: Status summary including: +- branch (str) +- staged changes (dict) +- worktree changes (dict) +- untracked files (list[str])

+
+
+
+ +
+ +
+
+hallmark.repo.chdir(path)[source]
+

Temporarily change the current working directory. No Yields.

+
+
Args:

Path: Directory to dwitch to while inside the context.

+
+
Returns:

None.

+
+
+
+ +
+
+hallmark.repo_state.load_branch_config(repo, branch: str) → dict[source]
+

Load the configuration for a specific branch

+
+
Args:

repo (Repo): repository object containing Git and state information/ +branch (String): Name of the branch whose configuration should be loaded

+
+
Returns:

(dict) A dictionary contaiing the branch configuration. Returns the +parsed contents of config.yml from the specified branch when +available, otherwise returns a copy of repo.state.config.

+
+
+
+ +
+
+hallmark.repo_state.load_branch_data(repo, branch: str) → State[source]
+

Load the state associated with a branch

+
+
Args:

repo (Repo): repository object +branch (String): branch name

+
+
Returns:

(State) A State consturcted from the branch’s config.yml, +meta.yml, and data.tsv files. If the branch does +not exist, returns a copy of the current repository state.

+
+
+
+ +
+
+hallmark.repo_state.load_branch_meta(repo, branch: str) → dict[source]
+

Load the metadata for a specific branch

+
+
Args:

repo (Repo): repository object +branch (String): Name of the branch

+
+
Returns:

The contents of meta.yml from the specified branch. If +metadata can’t be loaded, returns repo.state.meta.

+
+
+
+ +
+
+hallmark.repo_state.load_head_state(repo) → State[source]
+

Load the state stored at Head.

+
+
Args:

repo (Repo): Repository object.

+
+
Returns:

(State) A State constructed from the config.yml, meta.yml, +and``data.tsv`` files at HEAD. If no state can be loaded +from HEAD, returns a state with the current configuration +and metadata and an empty data table.

+
+
+
+ +
+
+class hallmark.state.State(config: dict = <factory>, meta: dict = <factory>, data: ~pandas.core.frame.DataFrame = <factory>)[source]
+

In-memory Hallmark state database.

+
+
Attributes:

config: Repository configuration values. +meta: Repository metadata. +data: Tabular file index containing indexed object checksums +(sha1) and associated metadata.

+
+
+
+
+replace(pf)[source]
+

Replace the contents of the state database.

+

Existing rows are discarded and replaced with the rows from the +provided ParaFrame.

+
+
Args:

pf (ParaFrame): ParaFrame containing the replacement rows.

+
+
Returns:

None.

+
+
+
+ +
+
+update(pf)[source]
+

Merge ParaFrame rows into the state database.

+

Existing rows with matching keys are updated, while new rows are +appended.

+
+
Args:

pf (ParaFrame): ParaFrame containing rows to add or update.

+
+
Returns:

None.

+
+
+
+ +
+ +

Remote data file downloader for hallmark repositories.

+
+
+exception hallmark.downloader.DownloadError[source]
+

Raised when remote data download fails.

+
+ +
+
+class hallmark.downloader.DownloadProgress(filename: str, total_bytes: int, downloaded_bytes: int = 0)[source]
+

Compatibility wrapper for older callers.

+
+ +
+
+hallmark.downloader.download_remote_data(repo, worktree_path: Path, max_workers: int = 4, show_progress: bool = False) → dict[source]
+

Download remote data files for a cloned repository.

+
+ +

Utilities for discovering and parsing parameterized file collections.

+

This module defines ParaFrame, a subclass of +pandas.DataFrame that provides convenient methods for locating, +parsing, and filtering files whose names follow parameterized naming +conventions.

+
+
+class hallmark.paraframe.ParaFrame(data=None, encodings=None, base_path=None, **kwargs)[source]
+

A subclass of pandas.DataFrame with additional methods for +parameterized file discovery and filtering.

+

ParaFrame behaves like a standard pandas.DataFrame while +providing the following additional functionality:

+
    +

  • __init__: Initializes the class and stores encodings and +base_path as metadata.

    • +

  • _constructor: Returns a new ParaFrame while preserving +encodings and base_path.

    • +

  • glob_search: Finds files matching a parameterized format string.

    • +

  • parse: Builds a ParaFrame by parsing file paths that match +a format pattern.

    • +

  • __call__ and filter: Convenience methods for filtering rows +by column values.

    • +
+
+
+filter(**kwargs)[source]
+

Filter a ParaFrame by matching column values.

+

This method filters the current ParaFrame by applying one or more +conditions on its columns. Rows satisfying any of the specified +conditions are returned.

+
+
Args:

**kwargs: Keyword arguments specifying column names and values to +filter by. Values may be scalars or sequences (lists or tuples). +Sequence values match any element in the sequence.

+
+
Returns:

pandas.DataFrame: A filtered DataFrame containing only the rows +that satisfy the requested conditions.

+
+
+
+ +
+ +

Find files matching a parameterized format string.

+

This method searches for files using the supplied format string. +When encoding=True, regular-expression encodings defined in the +YAML configuration are also applied.

+
+
Args:
+
fmt (str):

Format string describing the expected filename pattern.

+
+
*args:

Positional arguments used to fill the format string.

+
+
encodings (dict):

Encoding specifications from config.yml.

+
+
base_path (Path):

Root directory to search.

+
+
debug (bool):

If True, prints debugging information.

+
+
return_pattern (bool):

If True, returns both the glob pattern and the matched +files.

+
+
encoding (bool):

If True, applies regex encodings defined in the YAML +configuration.

+
+
**kwargs:

Keyword arguments used to fill the format string. +Missing values are replaced with the wildcard *.

+
+
+
+
Returns:
+
tuple:

If return_pattern is True, returns +(globbed_files, pattern).

+

Otherwise returns +(yaml_encodings, fmt_g, globbed_files).

+
+
+
+
+
+ +
+
+classmethod parse(fmt, *args, encodings=None, base_path=None, debug=False, encoding=False, **kwargs)[source]
+

Build a ParaFrame by parsing file paths that match a format +string.

+
+
Args:
+
fmt (str):

Format string containing {parameter} fields.

+
+
encodings (dict):

Encoding specifications from config.yml. +Defaults to {}.

+
+
base_path (Path):

Root directory to search. +Defaults to Path.cwd().

+
+
debug (bool):

If True, prints debugging information. +Defaults to False.

+
+
encoding (bool):

If True, applies regex encoding. +Defaults to False.

+
+
+
+
Returns:

ParaFrame: A ParaFrame whose rows correspond to matched files. +Parsed parameters are stored as columns together with a path column.

+
+
+
+ +
+ +
+
+ + +
+ +
+
+ +
+
+
+ ©2025, the Hallmark Authors. + + | + Powered by Sphinx 7.3.7 + & Alabaster 0.7.16 + + | + Page source +
+ + + + + + \ No newline at end of file diff --git a/doc/_build/html/design.html b/doc/_build/html/design.html new file mode 100644 index 0000000..d36da67 --- /dev/null +++ b/doc/_build/html/design.html @@ -0,0 +1,231 @@ + + + + + + + + Design — hallmark 0.2 documentation + + + + + + + + + + + + + + + + + + + +
+
+
+ + +
+ +
+

Design

+

hallmark follows the +Unix philosophy:

+
+

Write programs that do one thing and do it well. +Write programs to work together. +Write programs to handle text streams, because that is a universal +interface.

+

—Douglas McIlroy

+
+

The “one thing” for hallmark is maintaining a reproducible data +index. +With a well-designed indexing mechanism, it becomes natural to expose +a small set of core functions:

+
    +

  1. add/remove: +find data from any source and bring them into the index;

    2. +

  2. index: +compute checksums of data objects and index their relationships;

    3. +

  3. version control: +append immutable records; and

    4. +

  4. view: +emit manifests of subsets for other tools to consume.

    5. +
+
+

Architecture

+

A hallmark repository is the entry point for a version-controlled +dataset index. +It has three architecture components with different responsibilities:

+
    +

  1. State: +the canonical in-memory data container, where all index mutations +happen (add/remove/index);

    2. +

  2. Dothm: +an on-disk version-controlled repository, persisting State and +providing immutable history (using git); and

    3. +

  3. Worktree: +an on-disk working tree/directory, where data files are discovered +and consumed.

    4. +
+

The data flow can be summarized as:

+
            Repo
+ ____________/\____________
+/                          \
+
+State ---persist-------+
+  ^                    |
+  |                    |
+  |                    v
+  +----instantiate-- Dothm (".hm" git repository)
+  |                    ^
+  |                    |link
+  |                    v
+  +-----discover---- Worktree
+                       |
+                       |access
+                       v
+                     Other tools
+
+
+
+
+

API

+

Currently, hallmark has two built-in APIs:

+
    +

  1. Python API: +the native API that all hallmark features are implemented in. +State is the active object during the process lifetime. +Dothm and Worktree are optional depending on workflow (for +example, in-memory workflows may omit both).

    2. +

  2. CLI: +python features wrapped by click. +Each hallmark ... command loads State from a discovered +Dothm repository, executes the requested operation, then +writes staged state updates back to Dothm before exit. +In this mode, State is short-lived and Dothm is required.

    3. +
+

Worktree follows the same idea as +git worktree: +multiple repo/.hm directories can be attached to one underlying +repository to support parallel data transformations and +branch-isolated workflows.

+
+
+

Repository Repo

+

hallmark supports three repository forms with the same internal +Dothm data model:

+
    +

  1. standard repository::

    +
     +--- Worktree
+ v
+"repo/.hm/" <--- Dothm, a standard git repo
+
    +
    +
    2. +

  2. bare repository::

    +
    "repo.hm/" <--- Dothm, a standard or bare git repo
+
    +
    +
    3. +

  3. shared repository (multiple worktrees)::

    +
     +--- Worktree                     git repo
+ v                                        ^
+"repo1/.hm/" <--- Dothm, a git worktree --+
+                                          |
+ +--- Worktree                            |
+ v                                        |
+"repo2/.hm/" <--- Dothm, a git worktree --+
+                                          |
+ +--- Worktree                            |
+ v                                        |
+"repo3/.hm/" <--- Dothm, a git worktree --+
+
    +
    +
    4. +
+
+
+ + +
+ +
+
+ +
+
+
+ ©2025, the Hallmark Authors. + + | + Powered by Sphinx 7.3.7 + & Alabaster 0.7.16 + + | + Page source +
+ + + + + + \ No newline at end of file diff --git a/doc/_build/html/genindex.html b/doc/_build/html/genindex.html new file mode 100644 index 0000000..82a0f90 --- /dev/null +++ b/doc/_build/html/genindex.html @@ -0,0 +1,327 @@ + + + + + + + Index — hallmark 0.2 documentation + + + + + + + + + + + + + + + + + +
+
+
+ + +
+ + +

Index

+ +
+ A + | B + | C + | D + | F + | G + | H + | I + | L + | M + | P + | R + | S + | U + +
+

A

+ + + +
+ +

B

+ + +
+ +

C

+ + + +
+ +

D

+ + + +
+ +

F

+ + +
+ +

G

+ + +
+ +

H

+ + + +
    +
  • + hallmark.downloader + +
    • +
  • + hallmark.paraframe + +
    • +
  • + hallmark.repo + +
    • +
    +
  • + hallmark.repo_state + +
    • +
  • + hallmark.state + +
    • +
+ +

I

+ + +
+ +

L

+ + + +
+ +

M

+ + +
+ +

P

+ + + +
+ +

R

+ + + +
+ +

S

+ + + +
+ +

U

+ + +
+ + + +
+ +
+
+ +
+
+
+ ©2025, the Hallmark Authors. + + | + Powered by Sphinx 7.3.7 + & Alabaster 0.7.16 + +
+ + + + + + \ No newline at end of file diff --git a/doc/_build/html/index.html b/doc/_build/html/index.html new file mode 100644 index 0000000..c006953 --- /dev/null +++ b/doc/_build/html/index.html @@ -0,0 +1,196 @@ + + + + + + + + hallmark — hallmark 0.2 documentation + + + + + + + + + + + + + + + + + + +
+
+
+ + +
+ +
+

hallmark

+
+

Reproducibility is the hallmark of the scientific method.

+
+

Modern science has become so complex that many science projects rely +on multiple software packages to work in unison, resulting in networks of +data products along the analyses. +Versioning and managing these data products are essential in making +modern data- and computation-intensive science reproducible.

+

Motivated by the Event Horizon Telescope (EHT)’s +observational data calibration pipelines and theory data analyses +tools, hallmark is a lightweight package designed to version +control and manage data products in a complex workflow. +It provides a simple abstraction and a uniform Application Programming +Interface (API) on top of different backend technologies such as +POSIX file system, +object storage, +globus, +iRODS, +stream, +etc. +By using hallmark with other packages such as yukon and +banyan in Project Laniakea, researchers can utilize +computing infrastructures in a global scale to accelerate their +science.

+
+

ParaFrame

+

ParaFrame is a specialized subclass of pandas.DataFrame that +automatically extracts parameters encoded in file paths. When performing +large-scale parameter surveys or building simulation libraries, parameters +are often encoded directly in file naming schemes (e.g., +Ma+0.94_i70/sed_Rh160.h5).

+

Features:

+
    +

  • Decodes file paths back to structured parameters using Python format strings

    • +

  • Builds DataFrames with parsed parameters as columns

    • +

  • Supports custom encodings via YAML configuration for complex parsing rules

    • +

  • Provides intuitive filtering for parameter selection—easier than pure pandas

    • +
+
+
+

Tutorial

+

Examples of using ParaFrame with Python API or Command Line Interface (CLI) +can be found in the Jupyter Notebook tutorials in the demos folder.

+
+
+

Installation

+

Install hallmark from PyPI:

+
pip install hallmark
+
+
+

Or install from source for development purposes:

+
git clone https://github.com/l6a/hallmark.git
+cd hallmark
+pip install -e .
+
+
+
+

Contents:

+ +
+
+
+
+

Indices and tables

+ +
+ + +
+ +
+
+ +
+
+
+ ©2025, the Hallmark Authors. + + | + Powered by Sphinx 7.3.7 + & Alabaster 0.7.16 + + | + Page source +
+ + + + + + \ No newline at end of file diff --git a/doc/_build/html/py-modindex.html b/doc/_build/html/py-modindex.html new file mode 100644 index 0000000..50d1e90 --- /dev/null +++ b/doc/_build/html/py-modindex.html @@ -0,0 +1,145 @@ + + + + + + + Python Module Index — hallmark 0.2 documentation + + + + + + + + + + + + + + + + + + + + +
+
+
+ + +
+ + +

Python Module Index

+ +
+ h +
+ + + + + + + + + + + + + + + + + + + + + + +
 
+ h
+ hallmark +
    + hallmark.downloader +
    + hallmark.paraframe +
    + hallmark.repo +
    + hallmark.repo_state +
    + hallmark.state +
+ + +
+ +
+
+ +
+
+
+ ©2025, the Hallmark Authors. + + | + Powered by Sphinx 7.3.7 + & Alabaster 0.7.16 + +
+ + + + + + \ No newline at end of file diff --git a/doc/_build/html/search.html b/doc/_build/html/search.html new file mode 100644 index 0000000..56ab537 --- /dev/null +++ b/doc/_build/html/search.html @@ -0,0 +1,122 @@ + + + + + + + Search — hallmark 0.2 documentation + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+ + +
+ +

Search

+ + + + +

+ Searching for multiple words only shows matches that contain + all words. +

+ + +
+ + + +
+ + +
+ + +
+ +
+
+ +
+
+
+ ©2025, the Hallmark Authors. + + | + Powered by Sphinx 7.3.7 + & Alabaster 0.7.16 + +
+ + + + + + \ No newline at end of file diff --git a/doc/_build/html/usecase.html b/doc/_build/html/usecase.html new file mode 100644 index 0000000..8433015 --- /dev/null +++ b/doc/_build/html/usecase.html @@ -0,0 +1,221 @@ + + + + + + + + Use Cases — hallmark 0.2 documentation + + + + + + + + + + + + + + + + + + + +
+
+
+ + +
+ +
+

Use Cases

+

This section describes representative workflows supported by the +current hallmark CLI and Python API.

+
+

1. CLI: Standard Repository Ingest

+

Alice is organizing telescope data in a normal directory. +She initializes that directory as a hallmark worktree:

+
hallmark init obs
+cd obs
+
+
+

She adds the files using a python format-string pattern:

+
hallmark add "{site}/{year:d}/{day:d}.fits"
+
+
+

She then commits the updated hallmark index:

+
hallmark commit -m "Initial observation ingest"
+
+
+

She can inspect current repository paths at any time:

+
hallmark info
+
+
+

She can also inspect current changes and commit history:

+
hallmark status
+hallmark log
+
+
+

These git-like commands report or modify hallmark tracked state files +in .hm. +Dataset files are represented through staged sha1 column in +data.tsv, with other useful parameters (e.g., site, year, day), +associated with each file in different rows.

+
+
+

2. CLI: Bare Repository with Worktree Ingest

+

Bob prefers managing a bare hallmark repository for storage and +staging data from a linked worktree. +He initialized the bare repository and verify its mode:

+
hallmark init --bare sim.hm
+cd sim.hm
+hallmark info
+
+
+

He then attaches a worktree, stages discovered files, and commits:

+
hallmark worktree add /data/outputs
+cd /data/outputs
+hallmark add "run{run:d}/frame{frame:d}.h5"
+hallmark status
+hallmark diff
+hallmark commit -m "Simulation snapshots"
+
+
+

The commit updates the same bare repository that owns the linked +worktree.

+
+
+

3. CLI: Branch-Isolated Analysis with Multiple Worktrees

+

Carol wants to manage data from multiple simultaneous observations +without mixing data. +She creates a new branch and attach a second worktree to it:

+
hallmark branch obs2
+hallmark worktree add remote:/data/obs obs2
+
+
+

She lists linked worktrees and continue on the new branch:

+
hallmark worktree list
+hallmark status
+hallmark add "{site}/{year:d}/{day:d}.fits"
+hallmark commit -m "Observation ingest on branch obs2"
+
+
+

Each worktree stays isolated by branch, so staged state and commits do +not interfere.

+
+
+

4. Python: Programmatic Repository Updates

+

David uses the Python API for scripted ingest against an on-disk +repository:

+
from hallmark import Repo
+
+repo = Repo.open("obs")
+info = repo.info()
+print(info.local_path, info.worktree_path)
+
+for y in range(2000,2026,5):
+    repo.add(f"{{site}}/{y}/{{day:d}}.fits")  # escape {{ and }}
+repo.commit("Nightly ingest")
+
+
+

This workflow is suitable for finer control of data ingest.

+
+
+

5. Python: In-Memory State Workflows

+

Emma can use an memory-backed facade for data transformations that do +not require git operations:

+
from hallmark import Repo
+
+repo = Repo()
+repo.add("data/{site}/{year:d}/{day:d}.fits")
+repo.worktree("data_transformed/{year:d}/{day:d}/{site}.fits")
+
+
+

This is especially useful for data (re-)organization.

+
+
+ + +
+ +
+
+ +
+
+
+ ©2025, the Hallmark Authors. + + | + Powered by Sphinx 7.3.7 + & Alabaster 0.7.16 + + | + Page source +
+ + + + + + \ No newline at end of file From 572a353d1feaab6e052f54063727733db1dc4c52 Mon Sep 17 00:00:00 2001 From: YaqingSu Date: Wed, 15 Jul 2026 23:14:07 +0800 Subject: [PATCH 2/5] Documentation Trial --- doc/api.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/api.rst b/doc/api.rst index 1e09ba6..726fc3c 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -19,4 +19,7 @@ Repository State :members: .. automodule:: hallmark.paraframe + :members: + +.. automodule:: apple :members: \ No newline at end of file From f10d984b57ce3bbdf2bfab8e943467d7e4af6057 Mon Sep 17 00:00:00 2001 From: YaqingSu Date: Mon, 20 Jul 2026 15:37:36 +0800 Subject: [PATCH 3/5] updated documentation --- doc/api.rst | 88 ++++++++++++++++++++++++++++++++++++++++++++--------- doc/conf.py | 5 +++ 2 files changed, 79 insertions(+), 14 deletions(-) diff --git a/doc/api.rst b/doc/api.rst index 726fc3c..8fb2579 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -1,25 +1,85 @@ -API Reference -============= +# API Reference -Repository State ----------------- -# This tells Sphinx to import hallmark.filename and extract the module -# and function docstrings. +This section contains the automatically generated API documentation for the +`hallmark` package. + +## Core Repository .. automodule:: hallmark.repo - :members: +:members: +:show-inheritance: .. automodule:: hallmark.repo_state - :members: +:members: +:show-inheritance: + +.. automodule:: hallmark.repo_config +:members: +:show-inheritance: + +.. automodule:: hallmark.repo_manifest +:members: +:show-inheritance: + +## Repository Worktrees + +.. automodule:: hallmark.worktree +:members: +:show-inheritance: + +.. automodule:: hallmark.repo_worktree +:members: +:show-inheritance: + +## State Management .. automodule:: hallmark.state - :members: +:members: +:show-inheritance: -.. automodule:: hallmark.downloader - :members: +## Data Handling .. automodule:: hallmark.paraframe - :members: +:members: +:show-inheritance: + +.. automodule:: hallmark.objects +:members: +:show-inheritance: + +.. automodule:: hallmark.eht_datatree +:members: +:show-inheritance: + +## Downloading + +.. automodule:: hallmark.downloader +:members: +:show-inheritance: + +## Utilities + +.. automodule:: hallmark.helper_functions +:members: +:show-inheritance: + +.. automodule:: hallmark.fmt_detection +:members: +:show-inheritance: + +.. automodule:: hallmark.dothm +:members: +:show-inheritance: + +.. automodule:: hallmark.error +:members: +:show-inheritance: + +## Command Line Interface + +The Hallmark command-line interface provides commands for creating, managing, +and interacting with Hallmark repositories. -.. automodule:: apple - :members: \ No newline at end of file +.. automodule:: hallmark.cli +:members: +:show-inheritance: diff --git a/doc/conf.py b/doc/conf.py index b15b3b1..2becfbb 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -20,10 +20,15 @@ extensions = [ 'sphinx.ext.autodoc', + 'sphinx.ext.autosummary', + 'sphinx.ext.napoleon', 'sphinx.ext.mathjax', 'sphinx.ext.viewcode', ] +# Automatically generate summary tables +autosummary_generate = True + templates_path = ['_templates'] exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] From a64b809a08c1216e6080e12944a5be37c6954725 Mon Sep 17 00:00:00 2001 From: YaqingSu Date: Mon, 20 Jul 2026 15:44:47 +0800 Subject: [PATCH 4/5] Updated api file formatting for automatic documentation generation. --- doc/_build/html/_modules/hallmark/dothm.html | 259 ++++ .../html/_modules/hallmark/downloader.html | 3 +- .../html/_modules/hallmark/eht_datatree.html | 320 +++++ doc/_build/html/_modules/hallmark/error.html | 178 +++ .../html/_modules/hallmark/fmt_detection.html | 397 ++++++ .../_modules/hallmark/helper_functions.html | 204 ++++ .../html/_modules/hallmark/objects.html | 200 +++ doc/_build/html/_modules/hallmark/repo.html | 3 +- .../html/_modules/hallmark/repo_config.html | 370 ++++++ .../html/_modules/hallmark/repo_manifest.html | 197 +++ .../html/_modules/hallmark/repo_worktree.html | 215 ++++ .../html/_modules/hallmark/worktree.html | 150 +++ doc/_build/html/_modules/index.html | 12 +- doc/_build/html/api.html | 1086 ++++++++++++++--- doc/_build/html/design.html | 1 - doc/_build/html/genindex.html | 229 +++- doc/_build/html/index.html | 8 +- doc/_build/html/py-modindex.html | 55 + doc/_build/html/usecase.html | 5 +- doc/api.rst | 90 +- 20 files changed, 3777 insertions(+), 205 deletions(-) create mode 100644 doc/_build/html/_modules/hallmark/dothm.html create mode 100644 doc/_build/html/_modules/hallmark/eht_datatree.html create mode 100644 doc/_build/html/_modules/hallmark/error.html create mode 100644 doc/_build/html/_modules/hallmark/fmt_detection.html create mode 100644 doc/_build/html/_modules/hallmark/helper_functions.html create mode 100644 doc/_build/html/_modules/hallmark/objects.html create mode 100644 doc/_build/html/_modules/hallmark/repo_config.html create mode 100644 doc/_build/html/_modules/hallmark/repo_manifest.html create mode 100644 doc/_build/html/_modules/hallmark/repo_worktree.html create mode 100644 doc/_build/html/_modules/hallmark/worktree.html diff --git a/doc/_build/html/_modules/hallmark/dothm.html b/doc/_build/html/_modules/hallmark/dothm.html new file mode 100644 index 0000000..c967666 --- /dev/null +++ b/doc/_build/html/_modules/hallmark/dothm.html @@ -0,0 +1,259 @@ + + + + + + + hallmark.dothm — hallmark 0.2 documentation + + + + + + + + + + + + + + + + + +
+
+
+ + +
+ +

Source code for hallmark.dothm

+# Copyright 2025 the Hallmark Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from __future__ import annotations
+
+from pathlib   import Path
+from functools import cached_property
+from typing import Optional, Union
+
+from git import GitCommandError, Repo
+import pandas as pd
+import yaml
+
+from .state import State
+from .error import CloneError, DothmError
+
+
+

+[docs]
+class Dothm(Repo):
+    """Local ``.hm`` storage backend.
+
+    The backend version controls the hallmark ``State`` database files
+    (``config.yml``, ``meta.yml``, ``data.tsv``) on-disk.
+    It is itself a git worktree.
+    """
+
+    @cached_property
+    def path(self) -> Path:
+        return Path(self.working_tree_dir)
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        if self.working_tree_dir is None:
+            raise DothmError('The ".hm" directory must be a valid git ' \
+            'worktree.')
+
+

+[docs]
+    @classmethod
+    def init(cls, *args, **kwargs) -> "Dothm":
+        if kwargs.get('bare', False):
+            raise DothmError('A ".hm" directory must not be a bare git ' \
+            'repository')
+        kwargs.setdefault("initial_branch", "main")
+
+        dothm = super().init(*args, **kwargs)
+        readme_path = dothm.path / "README.md"
+        with open(readme_path, "w", encoding="utf-8") as f:
+            f.write("""# Local `.hm` Repository
+
+This is a dot-hallmark repository.
+It is a git-version-controlled dataset index used by `hallmark`.
+See https://l6a.github.io/hallmark/ for `hallmark` usage.
+""")
+        if not dothm.heads:
+            dothm.index.add([readme_path])
+            dothm.index.commit("Initial commit: local `.hm` repository")
+        return dothm

+
+
+    @staticmethod
+    def config_template() -> str:
+        return """# Edit this file only if your branch needs regex substitutions.
+# For simple names, you can just run: hallmark add "a{a}_i{i}.h5"
+data:
+  -
+    # fmt: "{release}_{source}_{year}_{doy:03d}_{band}.uvfits"
+    encoding:
+      # aspin: m([0-9]+(\\.[0-9]+)?|\\.[0-9]+)
+remote:
+  # name: origin
+  # url: https://example.com/path/to/data/
+"""
+
+

+[docs]
+    @classmethod
+    def clone(
+        cls,
+        url: str,
+        to_path: Union[Path, str],
+        display_path: Optional[Union[Path, str]] = None,
+    ) -> "Dothm":
+        to_path = Path(to_path)
+
+        try:
+            super().clone_from(url, str(to_path))
+            dothm = cls(str(to_path))
+
+            required_files = ["config.yml", "meta.yml", "data.tsv"]
+            for file in required_files:
+                if not (dothm.path / file).exists():
+                    raise CloneError(
+                        f'Cloned repository missing required file: {file}'
+                    )
+            return dothm
+        except GitCommandError as e:
+            raise CloneError.from_git_command(
+                e,
+                fallback=f'Failed to clone from "{url}"',
+                clone_path=to_path,
+                display_path=display_path,
+            ) from e
+        except CloneError:
+            raise

+
+
+    def link(self, path: Union[Path, str], branch: Optional[str] = None):
+        cmd = self.git  # has its own working directory
+        path = Path(path).resolve()  # use absolute path
+        try:
+            cmd.worktree("add", path, branch)
+        except GitCommandError as e:
+            raise DothmError(f'Failed to link "{path}": {e}')
+        return Dothm(path)
+
+    def load(self) -> State:
+        return State(
+            self.load_yml("config"),
+            self.load_yml("meta"),
+            self.load_tsv("data"),
+        )
+
+    def dump(self, state: State) -> None:
+        self.dump_yml(state.config, "config")
+        self.dump_yml(state.meta,   "meta")
+        self.dump_tsv(state.data,   "data")
+        self.index.add(["config.yml", "meta.yml", "data.tsv"])
+
+    def load_yml(self, stem: Union[Path, str]) -> dict:
+        with open((self.path/stem).with_suffix(".yml"), "r") as f:
+            return yaml.safe_load(f)
+
+    def dump_yml(self, data: dict, stem: Union[Path, str]) -> None:
+        with open((self.path/stem).with_suffix(".yml"), "w") as f:
+            yaml.dump(data, f, sort_keys=False)
+
+    def load_tsv(self, stem: Union[Path, str]) -> pd.DataFrame:
+        return pd.read_csv((self.path/stem).with_suffix(".tsv"), sep="\t", 
+                           dtype=str)
+
+    def dump_tsv(self, data: pd.DataFrame, stem: Union[Path, str]) -> None:
+        data.to_csv((self.path/stem).with_suffix(".tsv"), sep="\t", index=False)

+
+
+ +
+ +
+
+ +
+
+
+ ©2025, the Hallmark Authors. + + | + Powered by Sphinx 7.3.7 + & Alabaster 0.7.16 + +
+ + + + + + \ No newline at end of file diff --git a/doc/_build/html/_modules/hallmark/downloader.html b/doc/_build/html/_modules/hallmark/downloader.html index 358476f..45fec27 100644 --- a/doc/_build/html/_modules/hallmark/downloader.html +++ b/doc/_build/html/_modules/hallmark/downloader.html @@ -186,7 +186,8 @@

Source code for hallmark.downloader

     try:
         with ThreadPoolExecutor(max_workers=max_workers) as executor:
             futures = {
-                executor.submit(_download_file, url, destination, sha1): destination
+                executor.submit(_download_file, 
+                                url, destination, sha1): destination
                 for url, destination, sha1 in files_to_download
             }
             for future in as_completed(futures):
diff --git a/doc/_build/html/_modules/hallmark/eht_datatree.html b/doc/_build/html/_modules/hallmark/eht_datatree.html
new file mode 100644
index 0000000..d239506
--- /dev/null
+++ b/doc/_build/html/_modules/hallmark/eht_datatree.html
@@ -0,0 +1,320 @@
+
+
+
+  
+    
+    
+    hallmark.eht_datatree — hallmark 0.2 documentation
+    
+    
+    
+    
+    
+    
+    
+   
+  
+  
+
+  
+  
+
+  
+  
+
+    

+      

+        

+          
+
+          

+            
+  
Source code for hallmark.eht_datatree
+from __future__ import annotations
+from pathlib import Path
+import shutil
+from hallmark import ParaFrame
+from .fmt_detection import detect_fmt, DRIVE_EXTENSIONS
+import parse
+import re
+
+def _extract_drive(drive_path: Path) -> Path:
+    # remove extension from drive name to create the extraction directory
+    extract_dir = drive_path.parent / drive_path.stem
+    # check that the drive has not already been extracted
+    if not extract_dir.exists():
+        shutil.unpack_archive(str(drive_path), str(extract_dir))
+    return extract_dir
+
+# private function used by build_tree to create nested data branch
+def _build_data_branches(root: Path, fmt: str | list[str] | None, tracked: set[str],) \
+                            -> dict:
+    """
+    Build the fmt/stem data structure for files matching the given fmt(s).
+
+    Args:
+        root:    Path to search for matching files.
+        fmt:     Format string(s) for parsing data files. If None,
+                 fmts are auto-detected from files list
+        tracked: Set of relative file paths already accounted for.
+
+    Returns:
+        Dict of {fmt_str: {stem_key: ParaFrame, ..., "all": ParaFrame}}.
+    """
+    ### FMT STEMS OUTER DICT ###
+    # if no fmt was entered, parse the files to find them
+    if fmt is None:
+        fmts = detect_fmt(root)
+    else:
+        # check if there is only one fmt
+        fmts = [fmt] if isinstance(fmt, str) else fmt
+    # make a parser for each fmt with normalized deliminators
+    parsers = []
+    # normalize the deliminators for parsing
+    for f in fmts:
+        # normalize the deliminators for parsing
+        stem = re.sub(r"[\-.]", "_", f)
+        tokens = stem.split("_")
+        # keep the full stem in the variants list
+        variants = [stem]
+        for i, token in enumerate(tokens):
+            if re.fullmatch(r"\{p\d+\}", token):
+                # remove one parameter token at a time to create a variant
+                dropped = tokens[:i] + tokens[i + 1:]
+                variants.append("_".join(dropped))
+
+        for v in variants:
+            # create a parser for each variant
+            parsers.append((f, len(v.split("_")), parse.compile(v)))
+
+    # dict of the different fmt stems initialization
+    fmt_stems = {}
+    # search all subdirectories from root
+    for file in root.rglob("*"):
+        # if the file isn't a dir or a drive
+        relative_path = str(file.relative_to(root))
+        if relative_path in tracked or not file.is_file():
+            continue
+        # reset parse and fmt for each file
+        parsed = None
+        matched_fmt = None
+
+        stem_only = re.sub(r"[\-.]", "_", file.stem)
+        stem_with_ext = stem_only + re.sub(r"[\-.]", "_", file.suffix)
+        # counts to see if ext was included in fmt
+        stem_only_token_count = len(stem_only.split("_"))
+        stem_with_ext_token_count = len(stem_with_ext.split("_")) 
+
+        # parse the file name to extract fields, skip if it doesn't match the format
+        for fmt_str, variant_token_count, parser in parsers:
+            # if these are equal, there was no ext in the fmt
+            if variant_token_count == stem_only_token_count:
+                candidate = stem_only
+            # if these are equal, there was an ext in the fmt
+            elif variant_token_count == stem_with_ext_token_count:
+                candidate = stem_with_ext
+            # if neither are equal, this fmt is not compatible with this file
+            else:
+                continue
+            parsed = parser.parse(candidate)
+            # break out once correct parser is found
+            if parsed:
+                matched_fmt = fmt_str
+                break
+
+        if parsed:
+            # get path to file from data root
+            relative_path = str(file.relative_to(root))
+            # create unique stem name based on fmt parameters excluding extension
+            stem_key = "_".join(str(value) for key, value in parsed.named.items() 
+                                if key != "ext")
+            # create a row for this file to add to relevant Paraframe
+            row = {
+                "path": relative_path,
+                # normalize ext to not have period
+                "ext": Path(relative_path).suffix.lstrip("."),
+                # one column for each different parsed field
+                **{key: value for key, value in parsed.named.items() if key != "ext"},}
+            # add row to dict, and create the dict if it doesn't exist yet
+            fmt_stems.setdefault(matched_fmt, {}).setdefault(stem_key, []).append(row)
+            tracked.add(relative_path)
+
+    ### FMT STEMS INNER DICTS ###
+    # data structure initialization
+    data_branches = {}
+    # for every fmt and its stem dict
+    for fmt_str, stems in fmt_stems.items():
+        fmt_dict = {}
+        all_rows = []
+        # loop thru every stem for this fmt
+        for stem_key, rows in stems.items():
+            # create a Paraframe for each different stem
+            stem_pf = ParaFrame(rows, base_path=root)
+            # stem Paraframes are nested inside the fmt dict
+            fmt_dict[stem_key] = stem_pf
+            all_rows.extend(rows)
+        # create a Paraframe with all the stems for this dict merged
+        fmt_dict["all"] = ParaFrame(all_rows, base_path=root)
+        # each fmt broken into different data branches
+        data_branches[fmt_str] = fmt_dict
+
+    return data_branches
+
+
+

+[docs]
+def build_tree(root: Path, fmt: str | list[str] | None = None, data_type: str = "L2")\
+                 -> dict:
+    """
+    Build an in-memory pytree for an EHT dataset directory.
+
+    Args:
+        root: Path to the EHT dataset root directory.
+        fmt: Format string for parsing data files.
+        data_type: Type of data to build ("L1" or "L2")
+
+    Returns:
+        A dictionary with keys:
+        - "meta"   : ParaFrame of housekeeping files
+        - "drives" : ParaFrame of compressed archives
+        - "data"   : dict of {stem -> ParaFrame}
+    """
+    # create clean root path
+    root = Path(root).expanduser().resolve()
+    # track files that are included in the tree to avoid double counting
+    tracked = set()
+
+    # L1 data makes recursive calls for each directory, so use glob instead of rglob
+    glob_fn = root.glob if data_type == "L1" else root.rglob
+
+    ### DRIVES ###
+    # collect all drive paths matching any supported extension
+    drive_paths = []
+    extracted_dir_names = set()
+    for ext in DRIVE_EXTENSIONS:
+        # add any file that has this ext to the drive paths list and tracked set
+        for path in glob_fn(f"*{ext}"):
+            extract_dir = _extract_drive(path)
+            # only L1 data needs to track the extracted directory names for recursion
+            if data_type == "L1":
+                extracted_dir_names.add(extract_dir.name)
+            drive_paths.append(str(path.relative_to(root)))
+            tracked.add(str(path.relative_to(root)))
+    # build a single ParaFrame from all drive paths with extensions column
+    drives_pf = ParaFrame(
+    [{"path": path, "ext": Path(path).suffix.lstrip(".")} 
+     for path in sorted(drive_paths)],
+    base_path=root,)
+
+    ### DATA ###
+    data_branches = _build_data_branches(root, fmt, tracked) \
+                    if data_type == "L2" else {}
+
+    ### META ###
+    # create a list of all files under root that aren't tracked
+    meta_files = {
+        str(file.relative_to(root))
+        for file in glob_fn("*")
+        # add inventory item if its not a dir, not the .hm, and not in tracked
+        if file.is_file() 
+           and ".hm" not in file.parts
+           and str(file.relative_to(root)) not in tracked
+    }
+    # create a paraframe for the meta files with extensions column
+    meta_pf = ParaFrame(
+    [{"path": file, "ext": Path(file).suffix.lstrip(".")} 
+     for file in sorted(meta_files)],
+    base_path=root,)
+    for _, row in meta_pf.iterrows():
+        tracked.add(row["path"])
+        
+    # create dict with three keys, only data has subbranches
+    tree = {}
+    if not meta_pf.empty:
+        tree["meta"] = meta_pf
+    if not drives_pf.empty:
+        tree["drives"] = drives_pf
+    if data_branches:
+        tree["data"] = data_branches
+ 
+    # recursive L1 tree construction
+    if data_type == "L1":
+        # call build_tree for each subdirectory
+        for subdir in sorted(p for p in root.iterdir() if p.is_dir()):
+            # calls with drives will end recursion
+            next_level = "L2" if subdir.name in extracted_dir_names else "L1"
+            # append each subdirectory to the tree
+            tree[subdir.name] = build_tree(subdir, fmt, data_type=next_level)
+ 
+    return tree

+
+

+
+          

+          
+        

+      

+      
+      

+    

+    

+      ©2025, the Hallmark Authors.
+      
+      |
+      Powered by Sphinx 7.3.7
+      & Alabaster 0.7.16
+      
+    

+
+    
+
+    
+  
+
\ No newline at end of file
diff --git a/doc/_build/html/_modules/hallmark/error.html b/doc/_build/html/_modules/hallmark/error.html
new file mode 100644
index 0000000..57eaa41
--- /dev/null
+++ b/doc/_build/html/_modules/hallmark/error.html
@@ -0,0 +1,178 @@
+
+
+
+  
+    
+    
+    hallmark.error — hallmark 0.2 documentation
+    
+    
+    
+    
+    
+    
+    
+   
+  
+  
+
+  
+  
+
+  
+  
+
+    

+      

+        

+          
+
+          

+            
+  
Source code for hallmark.error
+# Copyright 2026 the Hallmark Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+"""Error hierarchy for Hallmark."""
+
+
+from pathlib import Path
+from typing import Optional, Union
+
+from git.exc import GitError, GitCommandError
+
+
+

+[docs]
+class HallmarkError(RuntimeError):
+    """Base exception for Hallmark-specific failures."""

+
+
+
+

+[docs]
+class DothmError(HallmarkError, GitError):
+    """Raised for `.hm` repository validation and access failures."""

+
+
+
+

+[docs]
+class CloneError(HallmarkError, GitError):
+    """Raised for hallmark clone failures."""
+
+    @classmethod
+    def from_git_command(
+        cls,
+        error: GitCommandError,
+        fallback: Optional[str] = None,
+        clone_path: Optional[Union[Path, str]] = None,
+        display_path: Optional[Union[Path, str]] = None,
+    ) -> "CloneError":
+        text = str(error.stderr or fallback or error).strip()
+        if "fatal:" in text:
+            text = text[text.index("fatal:"):].strip(" '")
+
+        if clone_path is not None and display_path is not None:
+            clone_path = Path(clone_path)
+            display_path = Path(display_path)
+            for candidate in (str(clone_path), str(clone_path.resolve())):
+                text = text.replace(candidate, str(display_path))
+
+        return cls(text)

+
+
+
+

+[docs]
+class DestinationExistsError(CloneError):
+    """Raised when clone destination already exists."""

+
+
+

+[docs]
+class CheckoutError(HallmarkError):
+    """Raised when a hallmark checkout cannot proceed safely."""

+
+

+
+          

+          
+        

+      

+      
+      

+    

+    

+      ©2025, the Hallmark Authors.
+      
+      |
+      Powered by Sphinx 7.3.7
+      & Alabaster 0.7.16
+      
+    

+
+    
+
+    
+  
+
\ No newline at end of file
diff --git a/doc/_build/html/_modules/hallmark/fmt_detection.html b/doc/_build/html/_modules/hallmark/fmt_detection.html
new file mode 100644
index 0000000..4d88bab
--- /dev/null
+++ b/doc/_build/html/_modules/hallmark/fmt_detection.html
@@ -0,0 +1,397 @@
+
+
+
+  
+    
+    
+    hallmark.fmt_detection — hallmark 0.2 documentation
+    
+    
+    
+    
+    
+    
+    
+   
+  
+  
+
+  
+  
+
+  
+  
+
+    

+      

+        

+          
+
+          

+            
+  
Source code for hallmark.fmt_detection
+import re
+from pathlib import Path
+
+# common meta extensions to look for when building the fmts
+META_EXTENSIONS = [".py", ".sh", ".md", ".pdf", ".rst", ".cfg", ".ini", ".yml", ".yaml"\
+                   , ".sl", ".par", ".xcm"]
+# common meta files to look for when building fmts
+KNOWN_META_FILES = {"README", "LICENSE", "LICENCE", "INVENTORY", "run"}
+# common drive extensions to look for when building fmts
+DRIVE_EXTENSIONS = [".tgz", ".tar", ".gz", ".zip", ".bz2", ".xz", ".zst", ".7z", ".rar"]
+# delimiter characters fmt detection splits on
+_DELIM_PATTERN = r"[_\-.]"
+
+
+def _stems_to_fmts(stems: list[str]) -> list[str]:
+    """Cluster same-token-count stems into one fmt per shared literal pattern.
+       
+        Recursively call until all fmts are found
+
+    Args:
+        stems: Stems (with extension reattached if present) that all have
+               the same number of tokens.
+
+    Returns:
+        List of fmt strings, one per distinct cluster found.
+    """
+    tokenized = {s: re.split(_DELIM_PATTERN, s) for s in stems}
+    # keep track of delimitors for fmt reconstruction
+    delimiters = {s: re.findall(_DELIM_PATTERN, s) for s in stems}
+    remaining_stems = list(stems)
+    fmts = []
+
+    while remaining_stems:
+        # check that every stem shares at least one token
+        reference_tokens = tokenized[remaining_stems[0]]
+        token_count = len(reference_tokens)
+        global_has_fixed = any(
+            # set will be length 1 if all stems have the same token at this index
+            len({tokenized[s][index] for s in remaining_stems}) == 1
+            for index in range(token_count))
+        # cluster found
+        if global_has_fixed:
+            cluster = list(remaining_stems)
+
+        else:
+            # base the anchor around the first non-assigned stem
+            anchor = remaining_stems[0]
+            anchor_tokens = tokenized[anchor]
+
+            # group remaining stems by which token positions they share with the anchor
+            stems_by_shared_positions : dict[tuple, list[str]] = {}
+            for stem in remaining_stems[1:]:
+                tokens = tokenized[stem]
+                shared_positions = tuple(
+                    i for i in range(len(anchor_tokens))
+                    if tokens[i] == anchor_tokens[i])
+                # if there are any tokens that match the anchor
+                if shared_positions:
+                    stems_by_shared_positions.setdefault(shared_positions, \
+                                                         []).append(stem)
+
+            if not stems_by_shared_positions:
+                # anchor matches nobody on any position and stem doesn't contain fmt
+                remaining_stems.remove(anchor)
+                continue
+
+            # find the cluster with most members that share positions with the anchor
+            _, best_members = max(stems_by_shared_positions.items(), \
+                                  key=lambda kv: len(kv[1]))
+            cluster = [anchor] + best_members
+
+        # build the fmt directly from the cluster
+        cluster_tokenized = [tokenized[stem] for stem in cluster]
+        fmt_tokens = []
+        field_count = 0
+        has_fixed = False
+        for index, token in enumerate(cluster_tokenized[0]):
+            values = [tokens[index] for tokens in cluster_tokenized]
+            # if every stem has the same token at this position
+            if len(set(values)) == 1:
+                fmt_tokens.append(token)
+                has_fixed = True
+            else:
+                # if the token is different for any, its a parameter
+                fmt_tokens.append(f"{{p{field_count}}}")
+                field_count += 1
+
+        # concatenate the fmt from the tokens and delimiters
+        if has_fixed and field_count > 0:
+            cluster_delims = delimiters[cluster[0]]
+            fmt_parts = [fmt_tokens[0]]
+            for token, delim in zip(fmt_tokens[1:], cluster_delims):
+                fmt_parts.append(delim)
+                fmt_parts.append(token)
+            fmts.append("".join(fmt_parts))
+        # remove all the stems that fit this fmt
+        for stem in cluster:
+            remaining_stems.remove(stem)
+
+    return fmts
+
+def _is_extendable_variant(longer: str, shorter: str) -> str | None:
+    """
+    Check if `shorter` and `longer` are compatible fmt strings, where `longer` 
+    has one additional token that can be removed to match `shorter`.
+
+    Args:
+        longer: A fmt string with one more token than `shorter`.
+        shorter: A fmt string with one less token than `longer`.
+
+        Returns:
+            A new fmt string with the additional token in `longer` replaced by a
+            parameter placeholder, if the two fmt strings are compatible. Otherwise,
+            return None.
+    """
+    longer_tokens = re.split(_DELIM_PATTERN, longer)
+    shorter_tokens = re.split(_DELIM_PATTERN, shorter)
+
+    # if longer doesn't have a single extra token, they can't be compatible
+    if len(longer_tokens) != len(shorter_tokens) + 1:
+        return None
+    
+    # collect every compatible dropped position
+    candidates = []
+    # check each token position in the longer fmt for compatibility with the shorter fmt
+    for i in range(len(longer_tokens)):
+        # remove the token at index i from longer and compare to shorter
+        candidate = longer_tokens[:i] + longer_tokens[i + 1:]
+        compatible = True
+        param_positions = {i}
+        # check that every token in candidate matches the corresponding token in shorter
+        for k, (candidate_token, shorter_token) in enumerate(
+                zip(candidate, shorter_tokens)):
+            # correcting for the removed token index
+            longer_idx = k if k < i else k + 1
+ 
+            # check if either token is already a parameter placeholder
+            if re.fullmatch(r"\{p\d+\}", candidate_token) or \
+                re.fullmatch(r"\{p\d+\}", shorter_token):
+                # track the index of the parameter placeholder
+                param_positions.add(longer_idx)
+                continue
+            # if the tokens are not equal, they are incompatible
+            if candidate_token != shorter_token:
+                compatible = False
+                break
+        if compatible:
+            # record the num of params, the index, and the positions
+            candidates.append((len(param_positions), i, param_positions))
+        # don't merge if it will result in a fmt with no fixed tokens
+    if not candidates:
+        # the shorter fmt is not a compatible variant of the longer fmt
+        return None
+
+    # find the candidate with the fewest parameter placeholders
+    min_params = min(count for count, _, _ in candidates)
+    # only keep candidates with the minimum number of parameter placeholders
+    best = [c for c in candidates if c[0] == min_params]
+    # sort by the index of the dropped token in the longer fmt, descending
+    best.sort(key=lambda c: c[1], reverse=True)
+    # select the candidate with the highest dropped token index to keep leftmost tokens
+    _, _, param_positions = best[0]
+ 
+    # safety check that they aren't the same length, meaning there is no variance
+    if len(param_positions) == len(longer_tokens):
+        return None    
+    
+    # rebuild the fmt string with the parameter placeholders renumbered
+    new_tokens = []
+    field_count = 0
+    for index, token in enumerate(longer_tokens):
+        # if we are at the removed token index or a parameter placeholder
+        if index in param_positions:
+            new_tokens.append(f"{{p{field_count}}}")
+            field_count += 1
+        else:
+            new_tokens.append(token)
+ 
+    # get the delimiters from the longer fmt string to reconstruct the final fmt
+    delims = re.findall(_DELIM_PATTERN, longer)
+    tokens = [new_tokens[0]]
+    # reconstruct the fmt string with the delimiters in between the tokens
+    for token, delim in zip(new_tokens[1:], delims):
+        tokens.append(delim)
+        tokens.append(token)
+    return "".join(tokens)
+
+
+

+[docs]
+def scan_inventory(root: Path) -> list[str]:
+    """
+    Scan a directory and return all file paths found, recursively.
+
+    Args:
+        root: Path to the dataset root directory to scan.
+
+    Returns:
+        List of relative file path strings found under root.
+
+    Raises:
+        FileNotFoundError: If root does not exist.
+    """
+    root = Path(root).expanduser().resolve()
+    if not root.is_dir():
+        raise FileNotFoundError(f"{root} is not a directory")
+
+    return sorted(
+        # convert to relative path strings for consistency with fmt detection
+        str(file.relative_to(root))
+        for file in root.rglob("*")
+        # only include files, and ignore any hallmark meta files
+        if file.is_file() and ".hm" not in file.parts
+    )

+
+
+
+

+[docs]
+def detect_fmt(root: Path) -> list[str]:
+    """
+    Auto-detect format strings from the files in the directory.
+
+    Args:
+        root: Path to the dataset root directory containing the files.
+
+    Returns:
+        List of fmt strings without extensions, one per distinct file
+        structure found in the files list.
+    """
+    root = Path(root).expanduser().resolve()
+    inventory_files = scan_inventory(root)
+
+    drive_exts = {ext.lstrip(".").lower() for ext in DRIVE_EXTENSIONS}
+    meta_exts = {ext.lstrip(".").lower() for ext in META_EXTENSIONS}
+
+    data_stems = []
+    seen = set()
+    for file in inventory_files:
+        path = Path(file)
+        # separate the extension from the stem
+        extension = path.suffix.lstrip(".").lower()
+        stem = path.stem
+
+        # skip any files that are known meta files or have known meta/drive extensions
+        if extension in drive_exts or extension in meta_exts:
+            continue
+        if stem.split(".")[0] in KNOWN_META_FILES:
+            continue
+        # check this stem hasn't already been added to the list
+        if stem not in seen:
+            seen.add(stem)
+            data_stems.append(stem)
+
+    # filter stems by token count to group them for fmt detection
+    by_token_count: dict[int, list[str]] = {}
+    for stem in data_stems:
+        # split the stem up by delimiters
+        token_count = len(re.split(_DELIM_PATTERN, stem))
+        # add the stem to the list of stems with the same token count
+        by_token_count.setdefault(token_count, []).append(stem)
+
+    group_fmts = []
+    for token_count, same_count_stems in sorted(by_token_count.items()):
+        # create the fmt strings for each group of stems with the same token count
+        group_fmts.extend(_stems_to_fmts(same_count_stems))
+        
+    # merge fmts that are the same except for an optional param
+    group_fmts.sort(key=lambda f: len(re.split(_DELIM_PATTERN, f)), reverse=True)
+    merged = []
+    # check if the fmt has already been merged to avoid duplicates
+    used = [False] * len(group_fmts)
+    for i, longer in enumerate(group_fmts):
+        if used[i]:
+            continue
+        current = longer
+        for j, shorter in enumerate(group_fmts):
+            # skip if the fmt is the same or has already been merged
+            if i == j or used[j]:
+                continue
+            # check if the shorter fmt is an extendable variant of the current fmt
+            extended = _is_extendable_variant(current, shorter)
+            if extended is not None:
+                current = extended
+                used[j] = True
+        merged.append(current)
+        used[i] = True
+
+    fmts = []
+    for fmt in merged:
+        # avoid adding duplicate fmts to the final list
+        if fmt not in fmts:
+            fmts.append(fmt)
+    return fmts

+
+

+
+          

+          
+        

+      

+      
+      

+    

+    

+      ©2025, the Hallmark Authors.
+      
+      |
+      Powered by Sphinx 7.3.7
+      & Alabaster 0.7.16
+      
+    

+
+    
+
+    
+  
+
\ No newline at end of file
diff --git a/doc/_build/html/_modules/hallmark/helper_functions.html b/doc/_build/html/_modules/hallmark/helper_functions.html
new file mode 100644
index 0000000..1955467
--- /dev/null
+++ b/doc/_build/html/_modules/hallmark/helper_functions.html
@@ -0,0 +1,204 @@
+
+
+
+  
+    
+    
+    hallmark.helper_functions — hallmark 0.2 documentation
+    
+    
+    
+    
+    
+    
+    
+   
+  
+  
+
+  
+  
+
+  
+  
+
+    

+      

+        

+          
+
+          

+            
+  
Source code for hallmark.helper_functions
+# Copyright 2025 the Hallmark Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+import pandas as pd
+
+
+

+[docs]
+def find_spec_by_fmt(fmt, encodings):
+    """Find the encoding spec for a given format string.
+
+    Args:
+        fmt:       Format string to look up.
+        encodings: The ``encodings`` list from ``State`` (i.e., the contents
+                   of ``config.yml``).
+
+    Returns:
+        The matching spec dict, or ``None`` if not found.
+    """
+    for spec in encodings:
+        if spec.get("fmt") == fmt:
+            return spec
+    return None

+
+
+
+

+[docs]
+def regex_sub(value, yaml_encodings):
+    """Apply regex substitution defined in an encoding spec.
+
+    Args:
+        value:          Format string / file path to transform.
+        yaml_encodings: A single encoding spec dict (one entry from the
+                        ``data`` list in ``hallmark.yml``), or ``None``.
+
+    Returns:
+        The (possibly transformed) format string.
+    """
+    if yaml_encodings is None:
+        return value
+
+    enc = yaml_encodings.get("encoding")
+    if not enc:
+        return value
+
+    regex = enc.get("aspin", "")
+    if not regex:
+        return value
+
+    result = value
+    for match in re.finditer(regex, value):
+        result = re.sub(match.group(0), "-" + str(match.group(1)), result)
+
+    return result

+
+
+

+[docs]
+def try_numeric_conversion(series):
+    """
+    Attempt to convert a pandas Series to numeric.
+
+    Converts the series to numeric iff:
+      1. All values are numeric
+      2. Converting back to string matches the original values to avoid
+         unintended conversions (e.g., "001" -> 1)
+
+    Args:
+        series: A pandas Series of strings to attempt conversion on.
+
+    Returns:
+        The converted numeric Series if both conditions are met,
+        otherwise returns original series.
+    """
+    # replace unconvertible values with NaN
+    converted = pd.to_numeric(series, errors="coerce")
+    # if any values were unconvertible, return original series
+    if converted.isna().any():
+        return series
+    # if converting back to str doesn't match original, return original series
+    # prevents unintended conversions like "001" -> 1
+    if not all(str(int(numeric_val)) == str(original_val) 
+                 or str(numeric_val) == str(original_val)
+               # check each pair of converted and original values
+               for numeric_val, original_val in zip(converted, series)):
+        return series
+    return converted

+
+
+

+
+          

+          
+        

+      

+      
+      

+    

+    

+      ©2025, the Hallmark Authors.
+      
+      |
+      Powered by Sphinx 7.3.7
+      & Alabaster 0.7.16
+      
+    

+
+    
+
+    
+  
+
\ No newline at end of file
diff --git a/doc/_build/html/_modules/hallmark/objects.html b/doc/_build/html/_modules/hallmark/objects.html
new file mode 100644
index 0000000..bc42041
--- /dev/null
+++ b/doc/_build/html/_modules/hallmark/objects.html
@@ -0,0 +1,200 @@
+
+
+
+  
+    
+    
+    hallmark.objects — hallmark 0.2 documentation
+    
+    
+    
+    
+    
+    
+    
+   
+  
+  
+
+  
+  
+
+  
+  
+
+    

+      

+        

+          
+
+          

+            
+  
Source code for hallmark.objects
+"""
+Utilities for storing and restoring content-addressed objects.
+
+This module defines the ``Objects`` class, which manages the Hallmark
+object store. Files are stored using their SHA-1 checksum and can later
+be restored to a specified location.
+"""
+
+import os
+import shutil
+from pathlib import Path
+from typing import Union
+
+
+

+[docs]
+class Objects:
+    """
+    Manage the Hallmark object store.
+
+    Files are stored in a content-addressed directory structure based on
+    their SHA-1 checksum, allowing them to be efficiently retrieved and
+    restored.
+    """
+    def __init__(self, path: Union[Path, str]):
+        """
+        Initialize the object store.
+
+        Args:
+            path (Path or str): Path to the Hallmark repository. The object
+                store is located in the ``objects`` subdirectory.
+        """
+        self.root = Path(path) / "objects"
+
+    def _split_checksum(self, sha1: str) -> Path:
+        """
+        Convert a SHA-1 checksum into its storage path.
+
+        The first two characters of the checksum form the directory name,
+        while the remaining characters form the filename.
+
+        Args:
+            sha1 (str): SHA-1 checksum.
+
+        Returns:
+            Path: Path where the object is stored.
+        """
+        return self.root / sha1[:2] / sha1[2:]
+
+

+[docs]
+    def store(self, src: Path, sha1: str) -> Path:
+        """
+        Store a file in the object store.
+
+        The file is copied only if an object with the same checksum does not
+        already exist.
+
+        Args:
+            src (Path): Source file to store.
+            sha1 (str): SHA-1 checksum of the file.
+
+        Returns:
+            Path: Path to the stored object.
+        """
+        stored_checksum = self._split_checksum(sha1)
+        if not stored_checksum.exists():
+            stored_checksum.parent.mkdir(parents=True, exist_ok=True)
+            shutil.copy2(src, stored_checksum)
+        return stored_checksum

+
+
+

+[docs]
+    def restore(self, sha1: str, dest: Path) -> Path:
+        """
+        Restore an object from the object store. Raises FileNotFoundError if 
+        the requested object does not exist in the object store.
+
+        The stored object is hard-linked to the destination path.
+
+        Args:
+            sha1 (str): SHA-1 checksum of the stored object.
+            dest (Path): Destination path for the restored file.
+
+        Returns:
+            Path: Path to the restored file.
+        """
+        stored = self._split_checksum(sha1)
+        if not stored.exists():
+            raise FileNotFoundError(f"object {sha1} not found in objects store")
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        if dest.exists():
+            dest.unlink()
+        os.link(stored, dest)
+        return dest

+

+
+

+
+          

+          
+        

+      

+      
+      

+    

+    

+      ©2025, the Hallmark Authors.
+      
+      |
+      Powered by Sphinx 7.3.7
+      & Alabaster 0.7.16
+      
+    

+
+    
+
+    
+  
+
\ No newline at end of file
diff --git a/doc/_build/html/_modules/hallmark/repo.html b/doc/_build/html/_modules/hallmark/repo.html
index 7149606..4502bab 100644
--- a/doc/_build/html/_modules/hallmark/repo.html
+++ b/doc/_build/html/_modules/hallmark/repo.html
@@ -575,7 +575,8 @@ 
Source code for hallmark.repo
 [docs]
     def add_worktree(self, target_branch: str) -> bool:
         '''
-        Create or link a new worktree for a branch. Raises ValueError if branch name is invalid.
+        Create or link a new worktree for a branch. Raises ValueError if branch name 
+        is invalid.
         Rasies Runtime error if called in a bare repository or worktree creation fails.
         
         Args:
diff --git a/doc/_build/html/_modules/hallmark/repo_config.html b/doc/_build/html/_modules/hallmark/repo_config.html
new file mode 100644
index 0000000..93184b6
--- /dev/null
+++ b/doc/_build/html/_modules/hallmark/repo_config.html
@@ -0,0 +1,370 @@
+
+
+
+  
+    
+    
+    hallmark.repo_config — hallmark 0.2 documentation
+    
+    
+    
+    
+    
+    
+    
+   
+  
+  
+
+  
+  
+
+  
+  
+
+    

+      

+        

+          
+
+          

+            
+  
Source code for hallmark.repo_config
+"""
+Utilities for managing Hallmark repository configuration.
+
+This module provides helper functions for reading, validating, and
+updating repository configuration values stored in ``config.yml``.
+"""
+
+from __future__ import annotations
+
+from string import Formatter
+from pathlib import Path
+from typing import Dict, Optional
+
+#test
+

+[docs]
+def ensure_branch_data_spec(config: dict) -> dict:
+    """
+    Ensure the configuration contains a valid data specification.
+
+    If the ``data`` entry is missing or malformed, it is initialized with
+    a single empty dictionary.
+
+    Args:
+        config (dict): Repository configuration.
+
+    Returns:
+        dict: The branch data specification.
+    """
+    data = config.get("data")
+    if not isinstance(data, list) or len(data) != 1 or not isinstance(data[0], dict):
+        config["data"] = [{}]
+    return config["data"][0]

+
+
+
+

+[docs]
+def branch_data_spec(repo) -> dict:
+    """
+    Return the branch data specification. Raises RuntimeError if 
+    the configuration does not define exactly one entry under ``data``.
+
+    Args:
+        repo: Repository object.
+
+    Returns:
+        dict: The branch data specification.
+    """
+    data = repo.state.config.get("data")
+    if not isinstance(data, list) or len(data) != 1 or not isinstance(data[0], dict):
+        raise RuntimeError('branch config must define exactly ' \
+        'one entry under "data" in config.yml')
+    return data[0]

+
+
+
+

+[docs]
+def branch_fmt(repo) -> str:
+    """
+    Return the configured filename format. Raises RuntimeError if 
+    no valid format string is defined.
+
+    Args:
+        repo: Repository object.
+
+    Returns:
+        str: The format string stored in ``data[0].fmt``.
+    """
+    fmt = branch_data_spec(repo).get("fmt")
+    if not isinstance(fmt, str) or not fmt.strip():
+        raise RuntimeError('branch config must define one ' \
+        'non-empty data[0].fmt in config.yml')
+    return fmt

+
+
+
+

+[docs]
+def set_branch_fmt(repo, fmt: str) -> None:
+    """
+    Set the branch filename format.
+
+    Args:
+        repo: Repository object.
+        fmt (str): New filename format.
+    """
+    set_config(repo, fmt=fmt)

+
+
+
+

+[docs]
+def set_config(
+    repo,
+    *,
+    fmt: Optional[str] = None,
+    remote_name: Optional[str] = None,
+    remote_url: Optional[str] = None,
+    encoding_updates: Optional[Dict[str, str]] = None,
+) -> dict:
+    """
+    Update the repository configuration.
+
+    Existing configuration values are preserved unless explicitly
+    replaced.
+
+    Args:
+        repo: Repository object.
+        fmt (str, optional): Filename format.
+        remote_name (str, optional): Remote repository name.
+        remote_url (str, optional): Remote repository URL.
+        encoding_updates (dict, optional): Encoding values to merge into
+            the existing configuration.
+
+    Returns:
+        dict: The updated configuration.
+    """
+    config = repo.state.config
+
+    spec = ensure_branch_data_spec(config)
+    updated_spec = {}
+
+    if fmt is not None:
+        updated_spec["fmt"] = fmt
+    elif "fmt" in spec:
+        updated_spec["fmt"] = spec["fmt"]
+
+    encoding_value = spec.get("encoding")
+    if encoding_updates:
+        if not isinstance(encoding_value, dict):
+            encoding_value = {}
+        encoding_value = {**encoding_value, **encoding_updates}
+    if "encoding" in spec or encoding_updates is not None:
+        updated_spec["encoding"] = encoding_value
+
+    for key, value in spec.items():
+        if key not in {"fmt", "encoding"}:
+            updated_spec[key] = value
+    config["data"][0] = updated_spec
+
+    if remote_name is not None or remote_url is not None:
+        remote = config.get("remote")
+        if not isinstance(remote, dict):
+            remote = {}
+
+        updated_remote = {}
+        if remote_name is not None:
+            updated_remote["name"] = remote_name
+        elif "name" in remote:
+            updated_remote["name"] = remote["name"]
+
+        if remote_url is not None:
+            updated_remote["url"] = remote_url
+        elif "url" in remote:
+            updated_remote["url"] = remote["url"]
+
+        for key, value in remote.items():
+            if key not in {"name", "url"}:
+                updated_remote[key] = value
+        config["remote"] = updated_remote
+
+    return config

+
+
+
+

+[docs]
+def branch_encodings(repo) -> list[dict]:
+    """
+    Return the configured filename encodings.
+
+    Args:
+        repo: Repository object.
+
+    Returns:
+        list[dict]: A list containing the encoding specification, or an
+        empty list if no encodings are defined.
+    """
+    spec = branch_data_spec(repo)
+    return [spec] if isinstance(spec.get("encoding"), dict) else []

+
+
+
+

+[docs]
+def fmt_fields(fmt: str) -> list[str]:
+    """
+    Extract field names from a format string.
+
+    Args:
+        fmt (str): Format string containing replacement fields.
+
+    Returns:
+        list[str]: Unique field names in the order they appear.
+    """
+    fields: list[str] = []
+    for _, field_name, _, _ in Formatter().parse(fmt):
+        if field_name and field_name not in fields:
+            fields.append(field_name)
+    return fields

+
+
+
+

+[docs]
+def coerce_fmt_value(value: str, spec: str):
+    """
+    Convert a value according to a format specification.
+
+    Args:
+        value (str): Value to convert.
+        spec (str): Format specification.
+
+    Returns:
+        The converted value.
+    """
+    if not spec:
+        return value
+    if spec.endswith("d"):
+        return int(float(value))
+    if spec[-1] in {"f", "F", "g", "G", "e", "E"}:
+        return float(value)
+    return value

+
+
+
+

+[docs]
+def row_to_path(row, fmt: str) -> Path:
+    """
+    Construct a file path from a table row.
+
+    Args:
+        row: Table row containing field values.
+        fmt (str): Format string used to build the path.
+
+    Returns:
+        Path: Path generated from the row values.
+    """
+    values = {}
+    for _, field_name, format_spec, _ in Formatter().parse(fmt):
+        if field_name:
+            values[field_name] = coerce_fmt_value(str(row[field_name]), format_spec)
+    return Path(fmt.format(**values))

+
+
+
+

+[docs]
+def path_from_row(repo, row, fmt: Optional[str] = None) -> Path:
+    """
+    Construct a file path from a table row.
+
+    If no format string is provided, the repository's configured format
+    is used.
+
+    Args:
+        repo: Repository object.
+        row: Table row containing field values.
+        fmt (str, optional): Format string to use.
+
+    Returns:
+        Path: Path generated from the row values.
+    """
+    return row_to_path(row, fmt or branch_fmt(repo))

+
+

+
+          

+          
+        

+      

+      
+      

+    

+    

+      ©2025, the Hallmark Authors.
+      
+      |
+      Powered by Sphinx 7.3.7
+      & Alabaster 0.7.16
+      
+    

+
+    
+
+    
+  
+
\ No newline at end of file
diff --git a/doc/_build/html/_modules/hallmark/repo_manifest.html b/doc/_build/html/_modules/hallmark/repo_manifest.html
new file mode 100644
index 0000000..1cef560
--- /dev/null
+++ b/doc/_build/html/_modules/hallmark/repo_manifest.html
@@ -0,0 +1,197 @@
+
+
+
+  
+    
+    
+    hallmark.repo_manifest — hallmark 0.2 documentation
+    
+    
+    
+    
+    
+    
+    
+   
+  
+  
+
+  
+  
+
+  
+  
+
+    

+      

+        

+          
+
+          

+            
+  
Source code for hallmark.repo_manifest
+from __future__ import annotations
+
+from pathlib import Path
+
+import parse
+import pandas as pd
+
+from .repo_config import fmt_fields, row_to_path
+
+
+

+[docs]
+def manifest_frame_from_pf(pf, fmt: str) -> pd.DataFrame:
+    """
+    Build a manifest table from a ``ParaFrame``. Raises RuntimeError
+    if a file path cannot be parsed using ``fmt``.
+
+    The returned table contains a ``sha1`` column together with the
+    fields extracted from the configured filename format.
+
+    Args:
+        pf: ``ParaFrame`` containing indexed file paths.
+        fmt (str): Filename format used to parse the paths.
+
+    Returns:
+        pandas.DataFrame: Manifest table containing ``sha1`` values and
+        parsed filename fields.
+    """
+    if pf.empty:
+        return pd.DataFrame(columns=["sha1", *fmt_fields(fmt)])
+
+    parser = parse.compile(fmt)
+    rows = []
+    for _, row in pf.iterrows():
+        parsed = parser.parse(str(row["path"]))
+        if parsed is None:
+            raise RuntimeError(f'failed to parse "{row["path"]}" \
+                               using branch fmt "{fmt}"')
+        rows.append({"sha1": row["sha1"], **{k: str(v) 
+                                for k, v in parsed.named.items()}})
+    return pd.DataFrame(rows, columns=["sha1", *fmt_fields(fmt)])

+
+
+
+

+[docs]
+def manifest_map(state) -> dict[str, str]:
+    """
+    Create a mapping from file paths to SHA-1 checksums.
+
+    Args:
+        state: Repository state.
+
+    Returns:
+        dict[str, str]: Dictionary mapping relative file paths to their
+        corresponding SHA-1 checksums.
+    """
+    if state.data.empty:
+        return {}
+    data = state.config.get("data")
+    if not isinstance(data, list) or len(data) != 1 or not isinstance(data[0], dict):
+        return {}
+    fmt = data[0].get("fmt")
+    if not isinstance(fmt, str) or not fmt.strip():
+        return {}
+    return {
+        str(row_to_path(row, fmt)): str(row["sha1"])
+        for _, row in state.data.iterrows()
+    }

+
+
+
+

+[docs]
+def frame_from_paths(repo, rel_paths: list[Path]):
+    """
+    Build a ``ParaFrame`` from repository paths.
+
+    Each path is converted into a row containing the relative path and
+    its SHA-1 checksum.
+
+    Args:
+        repo: Repository object.
+        rel_paths (list[Path]): Relative paths within the repository.
+
+    Returns:
+        ParaFrame: ``ParaFrame`` containing the indexed paths and their
+        checksums.
+    """
+    records = [{"path": str(path)} for path in rel_paths]
+    pf = repo.paraframe_cls(records, base_path=repo.worktree)
+    if not pf.empty:
+        pf["sha1"] = [repo.checksum(repo.worktree / path) for path in rel_paths]
+    return pf

+
+

+
+          

+          
+        

+      

+      
+      

+    

+    

+      ©2025, the Hallmark Authors.
+      
+      |
+      Powered by Sphinx 7.3.7
+      & Alabaster 0.7.16
+      
+    

+
+    
+
+    
+  
+
\ No newline at end of file
diff --git a/doc/_build/html/_modules/hallmark/repo_worktree.html b/doc/_build/html/_modules/hallmark/repo_worktree.html
new file mode 100644
index 0000000..d81b3c9
--- /dev/null
+++ b/doc/_build/html/_modules/hallmark/repo_worktree.html
@@ -0,0 +1,215 @@
+
+
+
+  
+    
+    
+    hallmark.repo_worktree — hallmark 0.2 documentation
+    
+    
+    
+    
+    
+    
+    
+   
+  
+  
+
+  
+  
+
+  
+  
+
+    

+      

+        

+          
+
+          

+            
+  
Source code for hallmark.repo_worktree
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+from .repo_config import branch_fmt, path_from_row
+from .error import CheckoutError
+
+
+

+[docs]
+def effective_cwd(repo) -> Path:
+    '''
+    Determine the effective working directory for repository operations.
+    Raises RuntimeError if the repository has no worktree
+
+    Args:
+        repo (repo): repository object
+    Returns:
+        path: The current working directory if it is inside the repository
+        worktree; otherwise, the worktree root.
+    '''
+    if repo.worktree is None:
+        raise RuntimeError("cannot inspect files in a bare repository " \
+        "without a worktree")
+
+    cwd = Path.cwd().resolve()
+    worktree = Path(repo.worktree).resolve()
+    try:
+        cwd.relative_to(worktree)
+    except ValueError:
+        return worktree
+    return cwd

+
+
+
+

+[docs]
+def filtered_paraframe(repo, pf):
+    '''
+    Filter a paraframe to the effective working directory.
+
+    Args:
+        repo (Repo): Repository object.
+        pf (ParaFrame): paraframe to filter.
+    Returns:
+        Paraframe: The original paraframe if operating at the worktree root.
+        Otherwise, only rows whose paths lie within the effective working 
+        directory.
+    '''
+    root = effective_cwd(repo)
+    worktree = Path(repo.worktree)
+    if root == worktree:
+        return pf
+
+    prefix = str(root.relative_to(worktree))
+    mask = pf["path"].astype(str).str.startswith(prefix + os.sep)
+    return pf[mask]

+
+
+
+

+[docs]
+def tracked_paths(repo) -> set[Path]:
+    '''
+    Return the set of tracked file paths.
+
+    Args:
+        repo (Repo): Repository object.
+
+    Returns:
+        set[Path]: Paths of all files tracked in the current 
+        repository state.
+    '''
+    fmt = branch_fmt(repo)
+    return {path_from_row(repo, row, fmt) for _, row in repo.state.data.iterrows()}

+
+
+
+

+[docs]
+def ensure_clean_tracked_files(repo) -> None:
+    '''
+    Verify that tracked files and repository state are clean. No returns.
+    Raises CheckoutError if the reposiotry has no worktree, a tracked 
+    file is missing, a tracked file has uncommited changes, or the
+    hallmark state contains uncommitted changes.
+
+    Args:
+        repo (Repo): Repository object
+    Returns:
+        None.
+    '''
+    if repo.worktree is None:
+        raise CheckoutError("cannot checkout without a worktree")
+
+    for _, row in repo.state.data.iterrows():
+        rel_path = path_from_row(repo, row)
+        path = repo.worktree / rel_path
+        if not path.exists():
+            raise CheckoutError(
+                f'tracked file "{rel_path}" is missing; commit or \
+                restore it before checkout')
+        if repo.checksum(path) != row["sha1"]:
+            raise CheckoutError(
+                f'tracked file "{rel_path}" has uncommitted changes; \
+                commit them before checkout')
+
+    if repo.dothm.index.diff("HEAD"):
+        raise CheckoutError(
+            "you have uncommitted hallmark state changes — " \
+            "commit them before checkout")

+
+

+
+          

+          
+        

+      

+      
+      

+    

+    

+      ©2025, the Hallmark Authors.
+      
+      |
+      Powered by Sphinx 7.3.7
+      & Alabaster 0.7.16
+      
+    

+
+    
+
+    
+  
+
\ No newline at end of file
diff --git a/doc/_build/html/_modules/hallmark/worktree.html b/doc/_build/html/_modules/hallmark/worktree.html
new file mode 100644
index 0000000..0441a2e
--- /dev/null
+++ b/doc/_build/html/_modules/hallmark/worktree.html
@@ -0,0 +1,150 @@
+
+
+
+  
+    
+    
+    hallmark.worktree — hallmark 0.2 documentation
+    
+    
+    
+    
+    
+    
+    
+   
+  
+  
+
+  
+  
+
+  
+  
+
+    

+      

+        

+          
+
+          

+            
+  
Source code for hallmark.worktree
+# Copyright 2025 the Hallmark Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Union
+
+
+

+[docs]
+class Worktree(type(Path())):
+    """Materialized data root used by indexing and consumer tools.
+
+    ``Worktree`` is where file objects are discovered by format string
+    and later consumed by downstream software.
+    """
+
+    def __new__(cls, path: Union[Path, str]) -> "Worktree":
+        path = Path(path).resolve()
+        if path.is_dir():
+            return super().__new__(cls, path)
+        elif path.exists():
+            raise FileNotFoundError(f'Worktree "{path}" is not a directory')
+        else:
+            raise FileNotFoundError(f'Worktree "{path}" not found')
+
+    @classmethod
+    def init(cls, path: Union[Path, str]) -> "Worktree":
+        path = Path(path)
+        path.mkdir(parents=True, exist_ok=True)
+        return cls(path)
+
+    def __truediv__(self, key):
+        return Path(self) / key

+
+

+
+          

+          
+        

+      

+      
+      

+    

+    

+      ©2025, the Hallmark Authors.
+      
+      |
+      Powered by Sphinx 7.3.7
+      & Alabaster 0.7.16
+      
+    

+
+    
+
+    
+  
+
\ No newline at end of file
diff --git a/doc/_build/html/_modules/index.html b/doc/_build/html/_modules/index.html
index 21e3b42..7981428 100644
--- a/doc/_build/html/_modules/index.html
+++ b/doc/_build/html/_modules/index.html
@@ -30,11 +30,21 @@
           

             
   
All modules for which code is available

-

diff --git a/doc/_build/html/api.html b/doc/_build/html/api.html
index e9ba203..1411aa3 100644
--- a/doc/_build/html/api.html
+++ b/doc/_build/html/api.html
@@ -33,14 +33,15 @@
             
   

 
API Reference

-

-
Repository State

-
# Example: This tells Sphinx to import hallmark.repo_state and extract the module
-# and function docstrings.

-

+
This section contains the automatically generated API documentation for the
+hallmark package.

+

+
Core Repository

+

 

 class hallmark.repo.Repo(path: Path | str)[source]

-
Hallmark repository.

+
Bases: object

+
Hallmark repository.

 
This is the Python API boundary.
 It loads the in-memory State from repository Dothm, and
 potentially populate the Worktree.

@@ -48,11 +49,15 @@ 
Repository State
 add(fstr: str, encoding: bool = False) → ParaFrame[source]
 
Stage files or updated repository indecing from the worktree.

-

-
Args:
fstr (string): Format string or “.” for full directory scan.
-encoding (boolean): Whether to apply encoding rules.

+

+
Parameters:

+
    
+
  • fstr (string) – Format string or “.” for full directory scan.
    • 
+
  • encoding (boolean) – Whether to apply encoding rules.
    • 
+

 

-
Returns:
paraframe Parsed and filtered file index (without checksums).

+
Returns:

+
paraframe Parsed and filtered file index (without checksums).

 

 

 

@@ -67,12 +72,18 @@ 
Repository State
 

 add_worktree(target_branch: str) → bool[source]

-
Create or link a new worktree for a branch. Raises ValueError if branch name is invalid.
+
Create or link a new worktree for a branch. Raises ValueError if branch name
+is invalid.
 Rasies Runtime error if called in a bare repository or worktree creation fails.

-

-
Args:
target_branch (string): Name of the branch to attach.

+

+
Parameters:

+
target_branch (string) – Name of the branch to attach.

+

+
Returns:

+
True if the worktree was successfully created.

 

-
Returns:
boolean: True if the worktree was successfully created.

+
Return type:

+
boolean

 

 

 

@@ -81,14 +92,19 @@ 
Repository State
 branches() → dict[str, object][source]
 
List repository branches.

-

-
Returns:

-
dictionary[string, object]: Dictionary containing:
    
+
    
+
    Returns:
    
+
    
+
    Dictionary containing:
      
 
    • current (string): Active branch name
      • 
 
    • names: All branch names
      • 
 
    
 
    
 
    
+
    
+
    
+
    Return type:
    
+
    dictionary[string, object]
    
 
    
 
    
 

@@ -99,10 +115,15 @@ 
Repository State
-
Args:
target_branch (string): Branch to switch to.

+

+
Parameters:

+
target_branch (string) – Branch to switch to.

 

-
Returns: 
boolean: True if checkout succeeds.

+
Returns:

+
True if checkout succeeds.

+

+
Return type:

+
boolean

 

 

 

@@ -111,11 +132,18 @@ 
Repository State
 static checksum(path: Path, chunk_size: int = 1048576) → str[source]
 
Computes the SHA1 checksum of a file.

-

-
Args:
path (path): path to the file to hash.
-chunk_size (integer): size of chunks used for streaming reads.

+

+
Parameters:

+
    
+
  • path (path) – path to the file to hash.
    • 
+
  • chunk_size (integer) – size of chunks used for streaming reads.
    • 
+

+

+
Returns:

+
Hexadecimal SHA1 digest of the file contents.

 

-
Returns:
String: Hexadecimal SHA1 digest of the file contents.

+
Return type:

+
String

 

 

 

@@ -124,16 +152,23 @@ 
Repository State
 classmethod clone(url: str, path: Path | str, *, fetch_data: bool = True, max_workers: int = 4, show_progress: bool = False) → Repo[source]
 
Clone a remote hallmark repository. Raises DestinationExistsError
-if the destination path already exists. Raises DownloadError if 
+if the destination path already exists. Raises DownloadError if
 data download is enabled and files fail to download.

-

-
Args:
url(string): remote repository URL.
-path(path|string): destination path for clone.
-fetch_data (boolean): if true, downloads associated data files. 
-max_workers (integer): Number of parallel workers for downloading data.
-show_progress (boolean): wether to display download progress.

+

+
Parameters:

+
    
+
  • url (string) – remote repository URL.
    • 
+
  • path (path|string) – destination path for clone.
    • 
+
  • fetch_data (boolean) – if true, downloads associated data files.
    • 
+
  • max_workers (integer) – Number of parallel workers for downloading data.
    • 
+
  • show_progress (boolean) – wether to display download progress.
    • 
+

+

+
Returns:

+
Cloned Repository instance

 

-
Returns:
Repo: Cloned Repository instance

+
Return type:

+
Repo

 

 

 

@@ -143,11 +178,18 @@ 
Repository Statecommit(msg: str, allow_empty: bool = False) → bool[source]
 
Commit staged changes to the repository. Raises ValueError if commit
 message is empty or invalid.

-

-
Args:
msg (string): commit message.
-allow_empty (boolean): Allow comitting even if no changes exists.

+

+
Parameters:

+
    
+
  • msg (string) – commit message.
    • 
+
  • allow_empty (boolean) – Allow comitting even if no changes exists.
    • 
+

+

+
Returns:

+
True if a commit was created, false otherwise.

 

-
Returns:
boolean: True if a commit was created, false otherwise.

+
Return type:

+
boolean

 

 

 

@@ -156,10 +198,15 @@ 
Repository State
 classmethod init(path: Path | str) → Repo[source]
 
Initialize a new hallmark repository.

-

-
Args:
paht(paht|string): path to initialize a worktree or .hm repository.

+

+
Parameters:

+
paht (paht|string) – path to initialize a worktree or .hm repository.

+

+
Returns:

+
newly created repository instance

 

-
Returns:
Repo: newly created repository instance

+
Return type:

+
Repo

 

 

 

@@ -168,8 +215,12 @@ 
Repository State
 log() → str[source]
 
Return commit history log.

-

-
Returns:
string: Git log output, or an empty string if no valid HEAD exists.

+

+
Returns:

+
Git log output, or an empty string if no valid HEAD exists.

+

+
Return type:

+
string

 

 

 

@@ -178,13 +229,20 @@ 
Repository State
 static lwpaths(path: Path | str) → Tuple[Path, Path | None][source]
 
Resolve repository and worktree paths.

-

-
Args:
path (Path | str): Path to either a worktree or a 
-.hm repository.

+

+
Parameters:

+
    
+
  • path (Path | str) – Path to either a worktree or a
    • 
+
  • repository. (.hm)
    • 
+

 

-
Returns:
tuple[Path, Path | None]: A (dothm_path, worktree_path) tuple. 
+
Returns:

+
A (dothm_path, worktree_path) tuple.
 If path refers to a .hm directory, worktree_path is None.

 

+
Return type:

+
tuple[Path, Path | None]

+

 

 

 
@@ -192,13 +250,20 @@ 
Repository State
 set_config(*, fmt: str | None = None, remote_name: str | None = None, remote_url: str | None = None, encoding_updates: Dict[str, str] | None = None) → dict[source]
 
Update repository configuration values.

-

-
Args:
fmt (str, optional): Data format specification.
-remote_name (str, optional): Name of the remote repository.
-remote_url (str, optional): URL of the remote repository.
-encoding_updates (dict[str, str], optional): Updates to encoding rules.

+

+
Parameters:

+
    
+
  • fmt (str, optional) – Data format specification.
    • 
+
  • remote_name (str, optional) – Name of the remote repository.
    • 
+
  • remote_url (str, optional) – URL of the remote repository.
    • 
+
  • encoding_updates (dict[str, str], optional) – Updates to encoding rules.
    • 
+

+

+
Returns:

+
Updated configuration dictionary.

 

-
Returns:
dict: Updated configuration dictionary.

+
Return type:

+
dict

 

 

 

@@ -206,17 +271,22 @@ 
Repository State
 

 status() → dict[str, object][source]

-
Return repository status information. Includes staged changes, 
+
Return repository status information. Includes staged changes,
 orktree modifications, deletions, and untracked files.

-

-
Args: 
none?

+

+
Parameters:

+
none?

 

-
Returns:
dict[str, object]: Status summary including:
+
Returns:

+
Status summary including:
 - branch (str)
 - staged changes (dict)
 - worktree changes (dict)
 - untracked files (list[str])

 

+
Return type:

+
dict[str, object]

+

 

 

 
@@ -226,10 +296,12 @@ 
Repository State
 hallmark.repo.chdir(path)[source]
 
Temporarily change the current working directory. No Yields.

-

-
Args:
Path: Directory to dwitch to while inside the context.

+

+
Parameters:

+
Path – Directory to dwitch to while inside the context.

 

-
Returns:
None.

+
Returns:

+
None.

 

 

 

@@ -238,11 +310,15 @@ 
Repository State
 hallmark.repo_state.load_branch_config(repo, branch: str) → dict[source]
 
Load the configuration for a specific branch

-

-
Args: 
repo (Repo): repository object containing Git and state information/
-branch (String): Name of the branch whose configuration should be loaded

+

+
Parameters:

+
    
+
  • repo (Repo) – repository object containing Git and state information/
    • 
+
  • branch (String) – Name of the branch whose configuration should be loaded
    • 
+

 

-
Returns:
(dict) A dictionary contaiing the branch configuration. Returns the 
+
Returns:

+
(dict) A dictionary contaiing the branch configuration. Returns the
 parsed contents of config.yml from the specified branch when
 available, otherwise returns a copy of repo.state.config.

 

@@ -253,11 +329,15 @@ 
Repository State
 hallmark.repo_state.load_branch_data(repo, branch: str) → State[source]
 
Load the state associated with a branch

-

-
Args: 
repo (Repo): repository object
-branch (String): branch name

+

+
Parameters:

+
    
+
  • repo (Repo) – repository object
    • 
+
  • branch (String) – branch name
    • 
+

 

-
Returns:
(State) A State consturcted from the branch’s config.yml,
+
Returns:

+
(State) A State consturcted from the branch’s config.yml,
 meta.yml, and data.tsv files. If the branch does
 not exist, returns a copy of the current repository state.

 

@@ -268,11 +348,15 @@ 
Repository State
 hallmark.repo_state.load_branch_meta(repo, branch: str) → dict[source]
 
Load the metadata for a specific branch

-

-
Args: 
repo (Repo): repository object
-branch (String): Name of the branch

+

+
Parameters:

+
    
+
  • repo (Repo) – repository object
    • 
+
  • branch (String) – Name of the branch
    • 
+

 

-
Returns:
The contents of meta.yml from the specified branch. If 
+
Returns:

+
The contents of meta.yml from the specified branch. If
 metadata can’t be loaded, returns repo.state.meta.

 

 

@@ -282,38 +366,423 @@ 
Repository State
 hallmark.repo_state.load_head_state(repo) → State[source]
 
Load the state stored at Head.

-

-
Args:
repo (Repo): Repository object.

+

+
Parameters:

+
repo (Repo) – Repository object.

 

-
Returns:
(State) A State constructed from the config.yml, meta.yml, 
-and``data.tsv`` files at HEAD. If no state can be loaded 
-from HEAD, returns a state with the current configuration 
+
Returns:

+
(State) A State constructed from the config.yml, meta.yml,
+and``data.tsv`` files at HEAD. If no state can be loaded
+from HEAD, returns a state with the current configuration
 and metadata and an empty data table.

 

 

 

 
-

+
Utilities for managing Hallmark repository configuration.

+
This module provides helper functions for reading, validating, and
+updating repository configuration values stored in config.yml.

+

+

+hallmark.repo_config.branch_data_spec(repo) → dict[source]

+
Return the branch data specification. Raises RuntimeError if
+the configuration does not define exactly one entry under data.

+

+
Parameters:

+
repo – Repository object.

+

+
Returns:

+
The branch data specification.

+

+
Return type:

+
dict

+

+

+

+
+

+

+hallmark.repo_config.branch_encodings(repo) → list[dict][source]

+
Return the configured filename encodings.

+

+
Parameters:

+
repo – Repository object.

+

+
Returns:

+
A list containing the encoding specification, or an
+empty list if no encodings are defined.

+

+
Return type:

+
list[dict]

+

+

+

+
+

+

+hallmark.repo_config.branch_fmt(repo) → str[source]

+
Return the configured filename format. Raises RuntimeError if
+no valid format string is defined.

+

+
Parameters:

+
repo – Repository object.

+

+
Returns:

+
The format string stored in data[0].fmt.

+

+
Return type:

+
str

+

+

+

+
+

+

+hallmark.repo_config.coerce_fmt_value(value: str, spec: str)[source]

+
Convert a value according to a format specification.

+

+
Parameters:

+
    
+
  • value (str) – Value to convert.
    • 
+
  • spec (str) – Format specification.
    • 
+

+

+
Returns:

+
The converted value.

+

+

+

+
+

+

+hallmark.repo_config.ensure_branch_data_spec(config: dict) → dict[source]

+
Ensure the configuration contains a valid data specification.

+
If the data entry is missing or malformed, it is initialized with
+a single empty dictionary.

+

+
Parameters:

+
config (dict) – Repository configuration.

+

+
Returns:

+
The branch data specification.

+

+
Return type:

+
dict

+

+

+

+
+

+

+hallmark.repo_config.fmt_fields(fmt: str) → list[str][source]

+
Extract field names from a format string.

+

+
Parameters:

+
fmt (str) – Format string containing replacement fields.

+

+
Returns:

+
Unique field names in the order they appear.

+

+
Return type:

+
list[str]

+

+

+

+
+

+

+hallmark.repo_config.path_from_row(repo, row, fmt: str | None = None) → Path[source]

+
Construct a file path from a table row.

+
If no format string is provided, the repository’s configured format
+is used.

+

+
Parameters:

+
    
+
  • repo – Repository object.
    • 
+
  • row – Table row containing field values.
    • 
+
  • fmt (str, optional) – Format string to use.
    • 
+

+

+
Returns:

+
Path generated from the row values.

+

+
Return type:

+
Path

+

+

+

+
+

+

+hallmark.repo_config.row_to_path(row, fmt: str) → Path[source]

+
Construct a file path from a table row.

+

+
Parameters:

+
    
+
  • row – Table row containing field values.
    • 
+
  • fmt (str) – Format string used to build the path.
    • 
+

+

+
Returns:

+
Path generated from the row values.

+

+
Return type:

+
Path

+

+

+

+
+

+

+hallmark.repo_config.set_branch_fmt(repo, fmt: str) → None[source]

+
Set the branch filename format.

+

+
Parameters:

+
    
+
  • repo – Repository object.
    • 
+
  • fmt (str) – New filename format.
    • 
+

+

+

+

+
+

+

+hallmark.repo_config.set_config(repo, *, fmt: str | None = None, remote_name: str | None = None, remote_url: str | None = None, encoding_updates: Dict[str, str] | None = None) → dict[source]

+
Update the repository configuration.

+
Existing configuration values are preserved unless explicitly
+replaced.

+

+
Parameters:

+
    
+
  • repo – Repository object.
    • 
+
  • fmt (str, optional) – Filename format.
    • 
+
  • remote_name (str, optional) – Remote repository name.
    • 
+
  • remote_url (str, optional) – Remote repository URL.
    • 
+
  • encoding_updates (dict, optional) – Encoding values to merge into
+the existing configuration.
    • 
+

+

+
Returns:

+
The updated configuration.

+

+
Return type:

+
dict

+

+

+

+
+

+

+hallmark.repo_manifest.frame_from_paths(repo, rel_paths: list[Path])[source]

+
Build a ParaFrame from repository paths.

+
Each path is converted into a row containing the relative path and
+its SHA-1 checksum.

+

+
Parameters:

+
    
+
  • repo – Repository object.
    • 
+
  • rel_paths (list[Path]) – Relative paths within the repository.
    • 
+

+

+
Returns:

+
ParaFrame containing the indexed paths and their
+checksums.

+

+
Return type:

+
ParaFrame

+

+

+

+
+

+

+hallmark.repo_manifest.manifest_frame_from_pf(pf, fmt: str) → DataFrame[source]

+
Build a manifest table from a ParaFrame. Raises RuntimeError
+if a file path cannot be parsed using fmt.

+
The returned table contains a sha1 column together with the
+fields extracted from the configured filename format.

+

+
Parameters:

+
    
+
  • pf – ParaFrame containing indexed file paths.
    • 
+
  • fmt (str) – Filename format used to parse the paths.
    • 
+

+

+
Returns:

+
Manifest table containing sha1 values and
+parsed filename fields.

+

+
Return type:

+
pandas.DataFrame

+

+

+

+
+

+

+hallmark.repo_manifest.manifest_map(state) → dict[str, str][source]

+
Create a mapping from file paths to SHA-1 checksums.

+

+
Parameters:

+
state – Repository state.

+

+
Returns:

+
Dictionary mapping relative file paths to their
+corresponding SHA-1 checksums.

+

+
Return type:

+
dict[str, str]

+

+

+

+
+

+

+
Repository Worktrees

+

+

+class hallmark.worktree.Worktree(path: Path | str)[source]

+
Bases: PosixPath

+
Materialized data root used by indexing and consumer tools.

+
Worktree is where file objects are discovered by format string
+and later consumed by downstream software.

+

+
+

+

+hallmark.repo_worktree.effective_cwd(repo) → Path[source]

+
Determine the effective working directory for repository operations.
+Raises RuntimeError if the repository has no worktree

+

+
Parameters:

+
repo (repo) – repository object

+

+
Returns:

+
The current working directory if it is inside the repository
+worktree; otherwise, the worktree root.

+

+
Return type:

+
path

+

+

+

+
+

+

+hallmark.repo_worktree.ensure_clean_tracked_files(repo) → None[source]

+
Verify that tracked files and repository state are clean. No returns.
+Raises CheckoutError if the reposiotry has no worktree, a tracked
+file is missing, a tracked file has uncommited changes, or the
+hallmark state contains uncommitted changes.

+

+
Parameters:

+
repo (Repo) – Repository object

+

+
Returns:

+
None.

+

+

+

+
+

+

+hallmark.repo_worktree.filtered_paraframe(repo, pf)[source]

+
Filter a paraframe to the effective working directory.

+

+
Parameters:

+
    
+
  • repo (Repo) – Repository object.
    • 
+
  • pf (ParaFrame) – paraframe to filter.
    • 
+

+

+
Returns:

+
The original paraframe if operating at the worktree root.
+Otherwise, only rows whose paths lie within the effective working
+directory.

+

+
Return type:

+
Paraframe

+

+

+

+
+

+

+hallmark.repo_worktree.tracked_paths(repo) → set[Path][source]

+
Return the set of tracked file paths.

+

+
Parameters:

+
repo (Repo) – Repository object.

+

+
Returns:

+
Paths of all files tracked in the current
+repository state.

+

+
Return type:

+
set[Path]

+

+

+

+
+

+

+
State Management

+

 

 class hallmark.state.State(config: dict = <factory>, meta: dict = <factory>, data: ~pandas.core.frame.DataFrame = <factory>)[source]

-
In-memory Hallmark state database.

-

-
Attributes:
config: Repository configuration values.
-meta: Repository metadata.
-data: Tabular file index containing indexed object checksums
-(sha1) and associated metadata.

+
Bases: object

+
In-memory Hallmark state database.

+

+

+config

+
Repository configuration values.

+

+
Type:

+
dict

+

+

+

+
+

+

+meta

+
Repository metadata.

+

+
Type:

+
dict

+

+

+

+
+

+

+data

+
Tabular file index containing indexed object checksums

+

+
Type:

+
pandas.core.frame.DataFrame

 

 

+

+
+

+

+(``sha1``) and associated metadata.

+

+
 

 

 replace(pf)[source]

 
Replace the contents of the state database.

 
Existing rows are discarded and replaced with the rows from the
 provided ParaFrame.

-

-
Args:
pf (ParaFrame): ParaFrame containing the replacement rows.

+

+
Parameters:

+
pf (ParaFrame) – ParaFrame containing the replacement rows.

 

-
Returns:
None.

+
Returns:

+
None.

 

 

 

@@ -324,36 +793,22 @@ 
Repository StateParaFrame rows into the state database.

 
Existing rows with matching keys are updated, while new rows are
 appended.

-

-
Args:
pf (ParaFrame): ParaFrame containing rows to add or update.

+

+
Parameters:

+
pf (ParaFrame) – ParaFrame containing rows to add or update.

 

-
Returns:
None.

+
Returns:

+
None.

 

 

 

 
 

 
-
Remote data file downloader for hallmark repositories.

-

-

-exception hallmark.downloader.DownloadError[source]

-
Raised when remote data download fails.

-

-
-

-

-class hallmark.downloader.DownloadProgress(filename: str, total_bytes: int, downloaded_bytes: int = 0)[source]

-
Compatibility wrapper for older callers.

-

-
-

-

-hallmark.downloader.download_remote_data(repo, worktree_path: Path, max_workers: int = 4, show_progress: bool = False) → dict[source]

-
Download remote data files for a cloned repository.

-

-
-
Utilities for discovering and parsing parameterized file collections.

+

+

+
Data Handling

+
Utilities for discovering and parsing parameterized file collections.

 
This module defines ParaFrame, a subclass of
 pandas.DataFrame that provides convenient methods for locating,
 parsing, and filtering files whose names follow parameterized naming
@@ -361,7 +816,8 @@ 
Repository State
 

 class hallmark.paraframe.ParaFrame(data=None, encodings=None, base_path=None, **kwargs)[source]

-
A subclass of pandas.DataFrame with additional methods for
+
Bases: DataFrame

+
A subclass of pandas.DataFrame with additional methods for
 parameterized file discovery and filtering.

 
ParaFrame behaves like a standard pandas.DataFrame while
 providing the following additional functionality:

@@ -383,14 +839,21 @@ 
Repository StateParaFrame by applying one or more
 conditions on its columns. Rows satisfying any of the specified
 conditions are returned.

-

-
Args:
**kwargs: Keyword arguments specifying column names and values to
-filter by. Values may be scalars or sequences (lists or tuples).
-Sequence values match any element in the sequence.

+

+
Parameters:

+
    
+
  • **kwargs – Keyword arguments specifying column names and values to
    • 
+
  • sequences (filter by. Values may be scalars or)
    • 
+
  • sequence. (Sequence values match any element in the)
    • 
+

 

-
Returns:
pandas.DataFrame: A filtered DataFrame containing only the rows
+
Returns:

+
A filtered DataFrame containing only the rows
 that satisfy the requested conditions.

 

+
Return type:

+
pandas.DataFrame

+

 

 

 
@@ -401,71 +864,388 @@ 
Repository Stateencoding=True, regular-expression encodings defined in the
 YAML configuration are also applied.

-

-
Args:

-
fmt (str):
Format string describing the expected filename pattern.

+

+
Parameters:

+
    
+
  • fmt (str) – Format string describing the expected filename pattern.
    • 
+
  • *args – Positional arguments used to fill the format string.
    • 
+
  • encodings (dict) – Encoding specifications from config.yml.
    • 
+
  • base_path (Path) – Root directory to search.
    • 
+
  • debug (bool) – If True, prints debugging information.
    • 
+
  • return_pattern (bool) – If True, returns both the glob pattern and the matched
+files.
    • 
+
  • encoding (bool) – If True, applies regex encodings defined in the YAML
+configuration.
    • 
+
  • **kwargs – Keyword arguments used to fill the format string.
+Missing values are replaced with the wildcard *.
    • 
+

 

-
*args:
Positional arguments used to fill the format string.

+
Returns:

+
If return_pattern is True, returns
+(globbed_files, pattern).

+
Otherwise returns
+(yaml_encodings, fmt_g, globbed_files).

+

 

-
encodings (dict):
Encoding specifications from config.yml.

+
Return type:

+
tuple

 

-
base_path (Path):
Root directory to search.

+

+

+
+

+

+classmethod parse(fmt, *args, encodings=None, base_path=None, debug=False, encoding=False, **kwargs)[source]

+
Build a ParaFrame by parsing file paths that match a format
+string.

+

+
Parameters:

+
    
+
  • fmt (str) – Format string containing {parameter} fields.
    • 
+
  • encodings (dict) – Encoding specifications from config.yml.
+Defaults to {}.
    • 
+
  • base_path (Path) – Root directory to search.
+Defaults to Path.cwd().
    • 
+
  • debug (bool) – If True, prints debugging information.
+Defaults to False.
    • 
+
  • encoding (bool) – If True, applies regex encoding.
+Defaults to False.
    • 
+

 

-
debug (bool):
If True, prints debugging information.

+
Returns:

+
A ParaFrame whose rows correspond to matched files.
+Parsed parameters are stored as columns together with a path column.

 

-
return_pattern (bool):
If True, returns both the glob pattern and the matched
-files.

+
Return type:

+
ParaFrame

 

-
encoding (bool):
If True, applies regex encodings defined in the YAML
-configuration.

+

+

+
+

+
+
Utilities for storing and restoring content-addressed objects.

+
This module defines the Objects class, which manages the Hallmark
+object store. Files are stored using their SHA-1 checksum and can later
+be restored to a specified location.

+

+

+class hallmark.objects.Objects(path: Path | str)[source]

+
Bases: object

+
Manage the Hallmark object store.

+
Files are stored in a content-addressed directory structure based on
+their SHA-1 checksum, allowing them to be efficiently retrieved and
+restored.

+

+

+restore(sha1: str, dest: Path) → Path[source]

+
Restore an object from the object store. Raises FileNotFoundError if
+the requested object does not exist in the object store.

+
The stored object is hard-linked to the destination path.

+

+
Parameters:

+
    
+
  • sha1 (str) – SHA-1 checksum of the stored object.
    • 
+
  • dest (Path) – Destination path for the restored file.
    • 
+

+

+
Returns:

+
Path to the restored file.

 

-
**kwargs:
Keyword arguments used to fill the format string.
-Missing values are replaced with the wildcard *.

+
Return type:

+
Path

 

 

+

+
+

+

+store(src: Path, sha1: str) → Path[source]

+
Store a file in the object store.

+
The file is copied only if an object with the same checksum does not
+already exist.

+

+
Parameters:

+
    
+
  • src (Path) – Source file to store.
    • 
+
  • sha1 (str) – SHA-1 checksum of the file.
    • 
+

 

-
Returns:

-
tuple:
If return_pattern is True, returns
-(globbed_files, pattern).

-
Otherwise returns
-(yaml_encodings, fmt_g, globbed_files).

+
Returns:

+
Path to the stored object.

+

+
Return type:

+
Path

+

+

+

+
+

+
+

+

+hallmark.eht_datatree.build_tree(root: Path, fmt: str | list[str] | None = None, data_type: str = 'L2') → dict[source]

+
Build an in-memory pytree for an EHT dataset directory.

+

+
Parameters:

+
    
+
  • root – Path to the EHT dataset root directory.
    • 
+
  • fmt – Format string for parsing data files.
    • 
+
  • data_type – Type of data to build (“L1” or “L2”)
    • 
+

+

+
Returns:

+
    
+
  • “meta”   : ParaFrame of housekeeping files
    • 
+
  • ”drives” : ParaFrame of compressed archives
    • 
+
  • ”data”   : dict of {stem -> ParaFrame}
    • 
+

+

+

+
Return type:

+
A dictionary with keys

+

+

+

+
+

+

+
Downloading

+
Remote data file downloader for hallmark repositories.

+

+

+exception hallmark.downloader.DownloadError[source]

+
Bases: HallmarkError

+
Raised when remote data download fails.

+

+
+

+

+class hallmark.downloader.DownloadProgress(filename: str, total_bytes: int, downloaded_bytes: int = 0)[source]

+
Bases: object

+
Compatibility wrapper for older callers.

+

+
+

+

+hallmark.downloader.download_remote_data(repo, worktree_path: Path, max_workers: int = 4, show_progress: bool = False) → dict[source]

+
Download remote data files for a cloned repository.

+

+
+

+

+
Utilities

+

+

+hallmark.helper_functions.find_spec_by_fmt(fmt, encodings)[source]

+
Find the encoding spec for a given format string.

+

+
Parameters:

+
    
+
  • fmt – Format string to look up.
    • 
+
  • encodings – The encodings list from State (i.e., the contents
+of config.yml).
    • 
+

+

+
Returns:

+
The matching spec dict, or None if not found.

 

 

+

+
+

+

+hallmark.helper_functions.regex_sub(value, yaml_encodings)[source]

+
Apply regex substitution defined in an encoding spec.

+

+
Parameters:

+
    
+
  • value – Format string / file path to transform.
    • 
+
  • yaml_encodings – A single encoding spec dict (one entry from the
+data list in hallmark.yml), or None.
    • 
+

+

+
Returns:

+
The (possibly transformed) format string.

 

 

 

 
-

-

-classmethod parse(fmt, *args, encodings=None, base_path=None, debug=False, encoding=False, **kwargs)[source]

-
Build a ParaFrame by parsing file paths that match a format
-string.

+

+

+hallmark.helper_functions.try_numeric_conversion(series)[source]

+
Attempt to convert a pandas Series to numeric.

 

-
Args:

-
fmt (str):
Format string containing {parameter} fields.

+
Converts the series to numeric iff:
    
+
  1. All values are numeric
    2. 
+
  2. Converting back to string matches the original values to avoid
+unintended conversions (e.g., “001” -> 1)
    3. 
+

 

-
encodings (dict):
Encoding specifications from config.yml.
-Defaults to {}.

+

+

+
Parameters:

+
series – A pandas Series of strings to attempt conversion on.

 

-
base_path (Path):
Root directory to search.
-Defaults to Path.cwd().

+
Returns:

+
The converted numeric Series if both conditions are met,
+otherwise returns original series.

 

-
debug (bool):
If True, prints debugging information.
-Defaults to False.

+

+

+
+

+

+hallmark.fmt_detection.detect_fmt(root: Path) → list[str][source]

+
Auto-detect format strings from the files in the directory.

+

+
Parameters:

+
root – Path to the dataset root directory containing the files.

 

-
encoding (bool):
If True, applies regex encoding.
-Defaults to False.

+
Returns:

+
List of fmt strings without extensions, one per distinct file
+structure found in the files list.

 

 

+

+
+

+

+hallmark.fmt_detection.scan_inventory(root: Path) → list[str][source]

+
Scan a directory and return all file paths found, recursively.

+

+
Parameters:

+
root – Path to the dataset root directory to scan.

 

-
Returns:
ParaFrame: A ParaFrame whose rows correspond to matched files.
-Parsed parameters are stored as columns together with a path column.

+
Returns:

+
List of relative file path strings found under root.

+

+
Raises:

+
FileNotFoundError – If root does not exist.

 

 

 

 
+

+

+class hallmark.dothm.Dothm(*args, **kwargs)[source]

+
Bases: Repo

+
Local .hm storage backend.

+
The backend version controls the hallmark State database files
+(config.yml, meta.yml, data.tsv) on-disk.
+It is itself a git worktree.

+

+

+classmethod clone(url: str, to_path: Path | str, display_path: Path | str | None = None) → Dothm[source]

+
Create a clone from this repository.

+

+
Parameters:

+
    
+
  • path – The full path of the new repo (traditionally ends with ./<name>.git).
    • 
+
  • progress – See Remote.push.
    • 
+
  • multi_options – 
    A list of git-clone(1) options that can be provided multiple
+times.
    
+
    One option per list item which is passed exactly as specified to clone.
+For example:
    
+
    [
+    "--config core.filemode=false",
+    "--config core.ignorecase",
+    "--recurse-submodule=repo1_path",
+    "--recurse-submodule=repo2_path",
+]
+
    
+
    
+
    • 
+
  • allow_unsafe_protocols – Allow unsafe protocols to be used, like ext.
    • 
+
  • allow_unsafe_options – Allow unsafe options to be used, like --upload-pack.
    • 
+
  • kwargs – 
      
+
    • odbt = ObjectDatabase Type, allowing to determine the object database
+implementation used by the returned Repo instance.
      • 
+
    • All remaining keyword arguments are given to the git-clone(1)
+command.
      • 
+
    
+
    • 
+

+

+
Returns:

+
Repo (the newly cloned repo)

+

+

 

 
+

+

+classmethod init(*args, **kwargs) → Dothm[source]

+
Initialize a git repository at the given path if specified.

+

+
Parameters:

+
    
+
  • path – The full path to the repo (traditionally ends with /<name>.git). Or
+None, in which case the repository will be created in the current
+working directory.
    • 
+
  • mkdir – If specified, will create the repository directory if it doesn’t already
+exist. Creates the directory with a mode=0755.
+Only effective if a path is explicitly given.
    • 
+
  • odbt – Object DataBase type - a type which is constructed by providing the
+directory containing the database objects, i.e. .git/objects. It will be
+used to access all object data.
    • 
+
  • expand_vars – If specified, environment variables will not be escaped. This can lead to
+information disclosure, allowing attackers to access the contents of
+environment variables.
    • 
+
  • kwargs – Keyword arguments serving as additional options to the
+git-init(1) command.
    • 
+

+

+
Returns:

+
Repo (the newly created repo)

+

+

+

+
+

+
+
Error hierarchy for Hallmark.

+

+

+exception hallmark.error.CheckoutError[source]

+
Bases: HallmarkError

+
Raised when a hallmark checkout cannot proceed safely.

+

+
+

+

+exception hallmark.error.CloneError[source]

+
Bases: HallmarkError, GitError

+
Raised for hallmark clone failures.

+

+
+

+

+exception hallmark.error.DestinationExistsError[source]

+
Bases: CloneError

+
Raised when clone destination already exists.

+

+
+

+

+exception hallmark.error.DothmError[source]

+
Bases: HallmarkError, GitError

+
Raised for .hm repository validation and access failures.

+

+
+

+

+exception hallmark.error.HallmarkError[source]

+
Bases: RuntimeError

+
Base exception for Hallmark-specific failures.

+

+
+

+

+
Command Line Interface

+
The Hallmark command-line interface provides commands for creating, managing,
+and interacting with Hallmark repositories.

+
Hallmark CLI entrypoint and command wiring.

 

 

 
@@ -491,7 +1271,13 @@ 
Navigation

 
  • Design
    • 
 
  • Use Cases
    • 
 
  • API Reference
 
    • 
 
diff --git a/doc/_build/html/design.html b/doc/_build/html/design.html
index d36da67..3ab1ff7 100644
--- a/doc/_build/html/design.html
+++ b/doc/_build/html/design.html
@@ -178,7 +178,6 @@ 
    Navigation
    
 
 
 
  • Use Cases
    • 
-
  • API Reference
    • 
 
 
 
    
diff --git a/doc/_build/html/genindex.html b/doc/_build/html/genindex.html
index 82a0f90..9161e37 100644
--- a/doc/_build/html/genindex.html
+++ b/doc/_build/html/genindex.html
@@ -37,16 +37,20 @@ 
    Index
    
  | B
  | C
  | D
+ | E
  | F
  | G
  | H
  | I
  | L
  | M
+ | O
  | P
  | R
  | S
+ | T
  | U
+ | W
  
 
    
 
    A
    
@@ -66,7 +70,17 @@ 
    A
    
 
    B
    
 
+  
    
   
 
    
@@ -78,13 +92,25 @@ 
    C
    
 
       
  • checkout() (hallmark.repo.Repo method)
 
    • 
+      
  • CheckoutError
+
    • 
+      
  • checksum() (hallmark.repo.Repo static method)
+
    • 
+      
  • clone() (hallmark.dothm.Dothm class method)
+
+      
    • 
   
   
 
@@ -92,10 +118,20 @@ 
    C
    
 
    D
    
 
    
   
   
 
    
 
+
    E
    
+
+  
+  
+
    
+
 
    F
    
 
+  
    
   
 
    
@@ -123,6 +183,20 @@ 
    H
    
 
+  
-  
    
   
      
       
    • 
+    hallmark.cli
+
+      
      • 
+      
    • 
+    hallmark.dothm
+
+      
      • 
+      
    • 
     hallmark.downloader
 
       
        
@@ -130,6 +204,43 @@ 
        H
        
 
       
      • 
       
    • 
+    hallmark.eht_datatree
+
+      
      • 
+      
    • 
+    hallmark.error
+
+      
      • 
+      
    • 
+    hallmark.fmt_detection
+
+      
      • 
+      
    • 
+    hallmark.helper_functions
+
+      
      • 
+      
    • 
+    hallmark.objects
+
+      
      • 
+  
      
+      
    • 
     hallmark.paraframe
 
       
        
@@ -143,13 +254,32 @@ 
        H
        
         
      • module
 
        • 
       
      • 
-  
      
+      
    • 
+    hallmark.repo_config
+
+      
      • 
+      
    • 
+    hallmark.repo_manifest
+
+      
      • 
       
    • 
     hallmark.repo_state
 
       
      • 
+      
    • 
+    hallmark.repo_worktree
+
+      
      • 
       
    • 
@@ -159,14 +289,27 @@ 
      H
      
         
    • module
 
      • 
       
    
+      
  • 
+    hallmark.worktree
+
+      
    • 
+      
  • HallmarkError
+
    • 
       		
 
    
 
 
    I
    
 
    
   
 
    
 
@@ -193,24 +336,60 @@ 
    L
    
 
    M
    
 
    
   
 
    
 
+
    O
    
+
+  
+
    
+
 
    P
    
 
    
   
      
@@ -219,6 +398,8 @@ 
      P
      
   
    		
   
 
    
@@ -226,11 +407,17 @@ 
    P
    
 
    R
    
 
    
   
   
 
    
@@ -238,13 +425,35 @@ 
    R
    
 
    S
    
 
+
    
   
   
    
+
+
    T
    
+
+  
+  
    
 
    
@@ -257,6 +466,14 @@ 
    U
    
   
 
 
+
    W
    
+
+  
+
    
+
 
 
           
    
diff --git a/doc/_build/html/index.html b/doc/_build/html/index.html
index c006953..5de9e8e 100644
--- a/doc/_build/html/index.html
+++ b/doc/_build/html/index.html
@@ -107,7 +107,13 @@ 
    Installation
 
 
  • API Reference
 
    • 
 
diff --git a/doc/_build/html/py-modindex.html b/doc/_build/html/py-modindex.html
index 50d1e90..93c89ca 100644
--- a/doc/_build/html/py-modindex.html
+++ b/doc/_build/html/py-modindex.html
@@ -49,11 +49,46 @@ 
    Python Module Index
    
        
        hallmark
        
+     
+       
+          
+       hallmark.cli
+       
+     
+       
+          
+       hallmark.dothm
+       
      
        
           
        hallmark.downloader
        
+     
+       
+          
+       hallmark.eht_datatree
+       
+     
+       
+          
+       hallmark.error
+       
+     
+       
+          
+       hallmark.fmt_detection
+       
+     
+       
+          
+       hallmark.helper_functions
+       
+     
+       
+          
+       hallmark.objects
+       
      
        
           
@@ -64,16 +99,36 @@ 
    Python Module Index
    
           
        hallmark.repo
        
+     
+       
+          
+       hallmark.repo_config
+       
+     
+       
+          
+       hallmark.repo_manifest
+       
      
        
           
        hallmark.repo_state
        
+     
+       
+          
+       hallmark.repo_worktree
+       
      
        
           
        hallmark.state
        
+     
+       
+          
+       hallmark.worktree
+       
    
 
 
diff --git a/doc/_build/html/usecase.html b/doc/_build/html/usecase.html
index 8433015..112730b 100644
--- a/doc/_build/html/usecase.html
+++ b/doc/_build/html/usecase.html
@@ -13,7 +13,7 @@
     
     
     
-    
+    
     
    
   
@@ -168,7 +168,6 @@ 
    Navigation
    
 
  • 5. Python: In-Memory State Workflows
    • 
 
 
-
  • API Reference
    • 
 
 
 
    
@@ -176,7 +175,7 @@ 
    Related Topics
    
 
 
    
diff --git a/doc/api.rst b/doc/api.rst
index 8fb2579..e142f0f 100644
--- a/doc/api.rst
+++ b/doc/api.rst
@@ -1,85 +1,93 @@
-# API Reference
+API Reference
+=============
 
 This section contains the automatically generated API documentation for the
-`hallmark` package.
+``hallmark`` package.
 
-## Core Repository
+Core Repository
+---------------
 
 .. automodule:: hallmark.repo
-:members:
-:show-inheritance:
+   :members:
+   :show-inheritance:
 
 .. automodule:: hallmark.repo_state
-:members:
-:show-inheritance:
+   :members:
+   :show-inheritance:
 
 .. automodule:: hallmark.repo_config
-:members:
-:show-inheritance:
+   :members:
+   :show-inheritance:
 
 .. automodule:: hallmark.repo_manifest
-:members:
-:show-inheritance:
+   :members:
+   :show-inheritance:
 
-## Repository Worktrees
+Repository Worktrees
+--------------------
 
 .. automodule:: hallmark.worktree
-:members:
-:show-inheritance:
+   :members:
+   :show-inheritance:
 
 .. automodule:: hallmark.repo_worktree
-:members:
-:show-inheritance:
+   :members:
+   :show-inheritance:
 
-## State Management
+State Management
+----------------
 
 .. automodule:: hallmark.state
-:members:
-:show-inheritance:
+   :members:
+   :show-inheritance:
 
-## Data Handling
+Data Handling
+-------------
 
 .. automodule:: hallmark.paraframe
-:members:
-:show-inheritance:
+   :members:
+   :show-inheritance:
 
 .. automodule:: hallmark.objects
-:members:
-:show-inheritance:
+   :members:
+   :show-inheritance:
 
 .. automodule:: hallmark.eht_datatree
-:members:
-:show-inheritance:
+   :members:
+   :show-inheritance:
 
-## Downloading
+Downloading
+-----------
 
 .. automodule:: hallmark.downloader
-:members:
-:show-inheritance:
+   :members:
+   :show-inheritance:
 
-## Utilities
+Utilities
+---------
 
 .. automodule:: hallmark.helper_functions
-:members:
-:show-inheritance:
+   :members:
+   :show-inheritance:
 
 .. automodule:: hallmark.fmt_detection
-:members:
-:show-inheritance:
+   :members:
+   :show-inheritance:
 
 .. automodule:: hallmark.dothm
-:members:
-:show-inheritance:
+   :members:
+   :show-inheritance:
 
 .. automodule:: hallmark.error
-:members:
-:show-inheritance:
+   :members:
+   :show-inheritance:
 
-## Command Line Interface
+Command Line Interface
+----------------------
 
 The Hallmark command-line interface provides commands for creating, managing,
 and interacting with Hallmark repositories.
 
 .. automodule:: hallmark.cli
-:members:
-:show-inheritance:
+   :members:
+   :show-inheritance:
\ No newline at end of file

From 11f599982d19e730864359e8dfc14bf171b1df4c Mon Sep 17 00:00:00 2001
From: YaqingSu 
Date: Mon, 20 Jul 2026 15:47:05 +0800
Subject: [PATCH 5/5] make clean to remove stale files

---
 doc/_build/html/design.html  | 1 +
 doc/_build/html/usecase.html | 5 +++--
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/doc/_build/html/design.html b/doc/_build/html/design.html
index 3ab1ff7..d36da67 100644
--- a/doc/_build/html/design.html
+++ b/doc/_build/html/design.html
@@ -178,6 +178,7 @@ 
    Navigation
    
 
 
 
  • Use Cases
    • 
+
  • API Reference
    • 
 
 
 
    
diff --git a/doc/_build/html/usecase.html b/doc/_build/html/usecase.html
index 112730b..8433015 100644
--- a/doc/_build/html/usecase.html
+++ b/doc/_build/html/usecase.html
@@ -13,7 +13,7 @@
     
     
     
-    
+    
     
    
   
@@ -168,6 +168,7 @@ 
    Navigation
    
 
  • 5. Python: In-Memory State Workflows
    • 
 
 
+
  • API Reference
    • 
 
 
 
    
@@ -175,7 +176,7 @@ 
    Related Topics