diff --git a/.gitignore b/.gitignore index ca5850a..1b3932c 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,8 @@ !*file !demo/demo_cli.ipynb !demo/demo_python.ipynb +!demo/demo_datatree.ipynb +!*.py.old __pycache__/ *.egg-info/ diff --git a/demo/demo_datatree.ipynb b/demo/demo_datatree.ipynb new file mode 100755 index 0000000..7d0f90e --- /dev/null +++ b/demo/demo_datatree.ipynb @@ -0,0 +1,360 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "77d8504a", + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "from hallmark.eht_datatree import build_tree\n", + "from hallmark.fmt_detection import scan_inventory\n", + "import pandas as pd\n", + "pd.set_option(\"display.width\", 300)\n", + "pd.set_option(\"display.max_columns\", None)\n", + "pd.set_option(\"display.max_colwidth\", 50)" + ] + }, + { + "cell_type": "markdown", + "id": "7bb3a37b", + "metadata": {}, + "source": [ + "a dataree can be built based on the path to the root and the data's fmts. Multiple fmts can be inputted. If no fmts are inputted, the program will try to detect them from the file name patterns, but this functionality still isn't perfect. Flag for if the data is L1 or L2 as the two trees are built slightly different" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "91c2e301", + "metadata": {}, + "outputs": [], + "source": [ + "datasets = [\"EHTC_CenA2017_Jul2021\", \"EHTC_M87-2018_Mar2024\",\"EHTC_SgrAmwl2017_May2022\",\n", + " \"EHTC_First3C279Results_May2020\", \"EHTC_M87mwl2017_Apr2021\", \n", + " \"EHTC_metadata2018_Dec2023\", \"EHTC_FirstM87Results_Apr2019\",\n", + " \"EHTC_M87mwl2018_Dec2024\", \"EHTC_FirstSgrAPol_Mar2024\",\n", + " \"EHTC_M87pol2017_Nov2023\", \"EHTC_FirstSgrAResults_May2022\",\n", + " \"EHTC_MonitoringM87_Sep2020\"]\n", + "root = Path(f\"~/{datasets[10]}\").expanduser()\n", + "tree = build_tree(root, None, data_type=\"L2\")\n", + "print(\"tree keys:\", list(tree.keys()))\n", + "#files = scan_inventory(root)\n", + "#ames = sorted(Path(f).name for f in files)\n", + "#for name in names:\n", + "# print(name)" + ] + }, + { + "cell_type": "markdown", + "id": "766ae073", + "metadata": {}, + "source": [ + "\"meta\" contains all the housekeeping files, which will be any files besides drives or files that have fmt naming conventions" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0cb26601", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"=== META ===\")\n", + "print(tree[\"meta\"])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c1b7eb90", + "metadata": {}, + "outputs": [], + "source": [ + "if \"drives\" in tree:\n", + " print(\"=== DRIVES ===\")\n", + " print(tree[\"drives\"])" + ] + }, + { + "cell_type": "markdown", + "id": "4e6d7164", + "metadata": {}, + "source": [ + "\"data\" will be a nested dict that has an outer list of all the different fmts" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dbc50055", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"=== DATA FMTS ===\")\n", + "fmt_keys = list(tree[\"data\"].keys())\n", + "for stem in fmt_keys:\n", + " print(f\"{stem}\")" + ] + }, + { + "cell_type": "markdown", + "id": "e961e8bc", + "metadata": {}, + "source": [ + "each fmt has a different Paraframe for each stem, as well as a Paraframe of all the stems merged incase someone wants to view all the data for one fmt" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d3747445", + "metadata": {}, + "outputs": [], + "source": [ + "print(f\"=== {fmt_keys[0]} STEMS ===\")\n", + "stem_keys = list(tree[\"data\"][fmt_keys[0]].keys())\n", + "for key in tree[\"data\"][fmt_keys[0]].keys():\n", + " print(key)" + ] + }, + { + "cell_type": "markdown", + "id": "41e091d0", + "metadata": {}, + "source": [ + "files that share fmts besides a missing parameter will also be grouped with that fmt, despite technically being parsed with a seperate fmt." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9a270ad4", + "metadata": {}, + "outputs": [], + "source": [ + "print(f\"=== ALL PARAFRAME ===\")\n", + "print(tree[\"data\"][fmt_keys[0]][\"all\"])" + ] + }, + { + "cell_type": "markdown", + "id": "dfc324ef", + "metadata": {}, + "source": [ + "The invindual stem Paraframes contain the same data with the different file formats" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "866848c8", + "metadata": {}, + "outputs": [], + "source": [ + "print(f\"=== {stem_keys[0]} STEM ===\")\n", + "print(\n", + "tree[\"data\"][fmt_keys[0]][stem_keys[0]])" + ] + }, + { + "cell_type": "markdown", + "id": "ee06d6c6", + "metadata": {}, + "source": [ + "The merged paraframe can be filtered to produce the same output as any given stem" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5c0ee919", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"=== MERGED PARAFRAME FILTERED ===\")\n", + "pf = tree[\"data\"][fmt_keys[0]][\"all\"]\n", + "\n", + "# get cols and params to ease testing and avoid hardcoding values\n", + "param_columns = [column for column in pf.columns if column not in (\"path\", \"ext\")]\n", + "filter_kwargs = {col: pf.iloc[0][col] for col in param_columns}\n", + "for column, value in filter_kwargs.items():\n", + " pf = pf(**{column: value})\n", + " \n", + "# can also print by doing print(pf(column0=\"value0\")(column1=\"value1\"),...)\n", + "print(pf)" + ] + }, + { + "cell_type": "markdown", + "id": "1d58a4af", + "metadata": {}, + "source": [ + "An L1 datatree is built similarly to the L2 case. Each zip drive is treated as its own L2 dataset, and prior to that will rely on recursive build_tree calls creating new branches for each folder until the zip drives are hit" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "19e53ab2", + "metadata": {}, + "outputs": [], + "source": [ + "root = Path(\"~/eht_local_copy\").expanduser()\n", + "drive_fmts = [\n", + " \"e17a10-7-hi-{p0}.{p1}\",\n", + " \"e17b06-7-hi_{p2}.{p3}\",\n", + " \"ff-{p0}-{p1}-{p2}-{p3}-{p4}.{p5}\",\n", + " \"{p0}.B.{p1}.{p2}\",]\n", + "tree = build_tree(root, drive_fmts, data_type=\"L1\")\n", + "root_keys = list(tree.keys())\n", + "print(\"=== ROOT KEYS ===\")\n", + "print(root_keys)" + ] + }, + { + "cell_type": "markdown", + "id": "a217d6bc", + "metadata": {}, + "source": [ + "No \"data\" branches will be made until we reach a folder that has compressed drives" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0f76048a", + "metadata": {}, + "outputs": [], + "source": [ + "all_project_keys = {}\n", + "for key in root_keys:\n", + " if key == \"meta\":\n", + " print(\"=== META PARAFRAME ===\")\n", + " print(tree[key])\n", + " print()\n", + " elif key == \"drives\":\n", + " print(\"=== DRIVES PARAFRAME ===\")\n", + " print(tree[key])\n", + " print()\n", + " else:\n", + " project_keys = list(tree[key].keys())\n", + " all_project_keys[key] = project_keys\n", + " print(f\"=== {key} KEYS ===\")\n", + " for key in project_keys:\n", + " print(key)\n", + " print()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "14ee04ae", + "metadata": {}, + "outputs": [], + "source": [ + "all_extract_keys = {}\n", + "for project_name in all_project_keys:\n", + " print(f\"=== {project_name} PROJECT ===\")\n", + " for extract_name in all_project_keys[project_name]:\n", + " if extract_name == \"meta\":\n", + " print(f\"=== {extract_name} PARAFRAME ===\")\n", + " print(tree[project_name][extract_name])\n", + " print()\n", + " elif extract_name == \"drives\":\n", + " print(f\"=== {extract_name} PARAFRAME ===\")\n", + " print(tree[project_name][extract_name])\n", + " print()\n", + " else:\n", + " extract_keys = list(tree[project_name][extract_name].keys())\n", + " all_extract_keys[f\"{project_name}/{extract_name}\"] = extract_keys\n", + " print(f\"=== {extract_name} KEYS ===\")\n", + " for key in extract_keys:\n", + " print(key)\n", + " print()\n", + " print()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "366eb8f2", + "metadata": {}, + "outputs": [], + "source": [ + "for project_name in all_project_keys:\n", + " print(f\"=== {project_name} PROJECT ===\")\n", + " print()\n", + " if \"meta\" in tree[project_name]:\n", + " print(\"=== meta PARAFRAME ===\")\n", + " print(tree[project_name][\"meta\"])\n", + " print()\n", + " elif \"drives\" in tree[project_name]:\n", + " print(\"=== drives PARAFRAME ===\")\n", + " print(tree[project_name][\"drives\"])\n", + " print()\n", + " for extract_name in all_project_keys[project_name]:\n", + " extract_dict = tree[project_name][extract_name]\n", + " if \"data\" in extract_dict:\n", + " print(f\"=== {extract_name} FMTS ===\")\n", + " for fmt in extract_dict[\"data\"]:\n", + " print(f\"{fmt}\")\n", + " print()\n", + " print()" + ] + }, + { + "cell_type": "markdown", + "id": "557196c6", + "metadata": {}, + "source": [ + "inconsistant naming conventions makes it difficult to fair all files to the correct fmt" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e810a4f4", + "metadata": {}, + "outputs": [], + "source": [ + "for project_name in all_project_keys:\n", + " print(f\"=== {project_name} PROJECT ===\")\n", + " print()\n", + " for extract_name in all_project_keys[project_name]:\n", + " if extract_name != \"meta\" and extract_name != \"drives\":\n", + " print(f\"=== {extract_name} EXTRACT ===\")\n", + " print()\n", + " extract_dict = tree[project_name][extract_name]\n", + " if \"data\" in extract_dict:\n", + " for fmt_str, stems in extract_dict[\"data\"].items():\n", + " print(f\"=== {fmt_str} FMT PARAFRAME ===\")\n", + " print(stems[\"all\"])\n", + " print()\n", + " print()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "base", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/doc/api.rst b/doc/api.rst new file mode 100644 index 0000000..1e09ba6 --- /dev/null +++ b/doc/api.rst @@ -0,0 +1,22 @@ +API Reference +============= + +Repository State +---------------- +# This tells Sphinx to import hallmark.filename and extract the module +# and function docstrings. + +.. automodule:: hallmark.repo + :members: + +.. automodule:: hallmark.repo_state + :members: + +.. automodule:: hallmark.state + :members: + +.. automodule:: hallmark.downloader + :members: + +.. automodule:: hallmark.paraframe + :members: \ No newline at end of file diff --git a/doc/conf.py b/doc/conf.py index 93b72c4..b15b3b1 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -5,6 +5,10 @@ # -- Project information ----------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information +import os +import sys + +sys.path.insert(0, os.path.abspath("../mod")) project = 'hallmark' copyright = '2025, the Hallmark Authors' diff --git a/doc/index.rst b/doc/index.rst index 3f0a695..d1b0f53 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -11,6 +11,7 @@ design usecase + api Indices and tables diff --git a/mod/hallmark/cli.py b/mod/hallmark/cli.py index 02c3b54..74b0433 100644 --- a/mod/hallmark/cli.py +++ b/mod/hallmark/cli.py @@ -238,7 +238,8 @@ def checkout(repo, target_branch): try: if repo.checkout(target_branch): click.echo(f'Switched to branch "{target_branch}".') - except (GitError, RuntimeError, ValueError, FileNotFoundError, CheckoutError) as e: + except (GitError, RuntimeError, ValueError, FileNotFoundError, + CheckoutError) as e: raise ClickException(str(e)) diff --git a/mod/hallmark/dothm.py b/mod/hallmark/dothm.py index 277acbb..cb9d7e0 100644 --- a/mod/hallmark/dothm.py +++ b/mod/hallmark/dothm.py @@ -42,12 +42,14 @@ def path(self) -> Path: def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) if self.working_tree_dir is None: - raise DothmError('The ".hm" directory must be a valid git worktree.') + raise DothmError('The ".hm" directory must be a valid git ' \ + 'worktree.') @classmethod def init(cls, *args, **kwargs) -> "Dothm": if kwargs.get('bare', False): - raise DothmError('A ".hm" directory must not be a bare git repository') + raise DothmError('A ".hm" directory must not be a bare git ' \ + 'repository') kwargs.setdefault("initial_branch", "main") dothm = super().init(*args, **kwargs) @@ -139,7 +141,8 @@ def dump_yml(self, data: dict, stem: Union[Path, str]) -> None: yaml.dump(data, f, sort_keys=False) def load_tsv(self, stem: Union[Path, str]) -> pd.DataFrame: - return pd.read_csv((self.path/stem).with_suffix(".tsv"), sep="\t", dtype=str) + return pd.read_csv((self.path/stem).with_suffix(".tsv"), sep="\t", + dtype=str) def dump_tsv(self, data: pd.DataFrame, stem: Union[Path, str]) -> None: data.to_csv((self.path/stem).with_suffix(".tsv"), sep="\t", index=False) diff --git a/mod/hallmark/downloader.py b/mod/hallmark/downloader.py index a61b9f9..3149d2a 100644 --- a/mod/hallmark/downloader.py +++ b/mod/hallmark/downloader.py @@ -146,7 +146,8 @@ def download_remote_data( try: with ThreadPoolExecutor(max_workers=max_workers) as executor: futures = { - executor.submit(_download_file, url, destination, sha1): destination + executor.submit(_download_file, + url, destination, sha1): destination for url, destination, sha1 in files_to_download } for future in as_completed(futures): diff --git a/mod/hallmark/eht_datatree.py b/mod/hallmark/eht_datatree.py new file mode 100755 index 0000000..33d4c38 --- /dev/null +++ b/mod/hallmark/eht_datatree.py @@ -0,0 +1,215 @@ +from __future__ import annotations +from pathlib import Path +import shutil +from hallmark import ParaFrame +from .fmt_detection import detect_fmt, DRIVE_EXTENSIONS +import parse +import re + +def _extract_drive(drive_path: Path) -> Path: + # remove extension from drive name to create the extraction directory + extract_dir = drive_path.parent / drive_path.stem + # check that the drive has not already been extracted + if not extract_dir.exists(): + shutil.unpack_archive(str(drive_path), str(extract_dir)) + return extract_dir + +# private function used by build_tree to create nested data branch +def _build_data_branches(root: Path, fmt: str | list[str] | None, tracked: set[str],) \ + -> dict: + """ + Build the fmt/stem data structure for files matching the given fmt(s). + + Args: + root: Path to search for matching files. + fmt: Format string(s) for parsing data files. If None, + fmts are auto-detected from files list + tracked: Set of relative file paths already accounted for. + + Returns: + Dict of {fmt_str: {stem_key: ParaFrame, ..., "all": ParaFrame}}. + """ + ### FMT STEMS OUTER DICT ### + # if no fmt was entered, parse the files to find them + if fmt is None: + fmts = detect_fmt(root) + else: + # check if there is only one fmt + fmts = [fmt] if isinstance(fmt, str) else fmt + # make a parser for each fmt with normalized deliminators + parsers = [] + # normalize the deliminators for parsing + for f in fmts: + # normalize the deliminators for parsing + stem = re.sub(r"[\-.]", "_", f) + tokens = stem.split("_") + # keep the full stem in the variants list + variants = [stem] + for i, token in enumerate(tokens): + if re.fullmatch(r"\{p\d+\}", token): + # remove one parameter token at a time to create a variant + dropped = tokens[:i] + tokens[i + 1:] + variants.append("_".join(dropped)) + + for v in variants: + # create a parser for each variant + parsers.append((f, len(v.split("_")), parse.compile(v))) + + # dict of the different fmt stems initialization + fmt_stems = {} + # search all subdirectories from root + for file in root.rglob("*"): + # if the file isn't a dir or a drive + relative_path = str(file.relative_to(root)) + if relative_path in tracked or not file.is_file(): + continue + # reset parse and fmt for each file + parsed = None + matched_fmt = None + + stem_only = re.sub(r"[\-.]", "_", file.stem) + stem_with_ext = stem_only + re.sub(r"[\-.]", "_", file.suffix) + # counts to see if ext was included in fmt + stem_only_token_count = len(stem_only.split("_")) + stem_with_ext_token_count = len(stem_with_ext.split("_")) + + # parse the file name to extract fields, skip if it doesn't match the format + for fmt_str, variant_token_count, parser in parsers: + # if these are equal, there was no ext in the fmt + if variant_token_count == stem_only_token_count: + candidate = stem_only + # if these are equal, there was an ext in the fmt + elif variant_token_count == stem_with_ext_token_count: + candidate = stem_with_ext + # if neither are equal, this fmt is not compatible with this file + else: + continue + parsed = parser.parse(candidate) + # break out once correct parser is found + if parsed: + matched_fmt = fmt_str + break + + if parsed: + # get path to file from data root + relative_path = str(file.relative_to(root)) + # create unique stem name based on fmt parameters excluding extension + stem_key = "_".join(str(value) for key, value in parsed.named.items() + if key != "ext") + # create a row for this file to add to relevant Paraframe + row = { + "path": relative_path, + # normalize ext to not have period + "ext": Path(relative_path).suffix.lstrip("."), + # one column for each different parsed field + **{key: value for key, value in parsed.named.items() if key != "ext"},} + # add row to dict, and create the dict if it doesn't exist yet + fmt_stems.setdefault(matched_fmt, {}).setdefault(stem_key, []).append(row) + tracked.add(relative_path) + + ### FMT STEMS INNER DICTS ### + # data structure initialization + data_branches = {} + # for every fmt and its stem dict + for fmt_str, stems in fmt_stems.items(): + fmt_dict = {} + all_rows = [] + # loop thru every stem for this fmt + for stem_key, rows in stems.items(): + # create a Paraframe for each different stem + stem_pf = ParaFrame(rows, base_path=root) + # stem Paraframes are nested inside the fmt dict + fmt_dict[stem_key] = stem_pf + all_rows.extend(rows) + # create a Paraframe with all the stems for this dict merged + fmt_dict["all"] = ParaFrame(all_rows, base_path=root) + # each fmt broken into different data branches + data_branches[fmt_str] = fmt_dict + + return data_branches + + +def build_tree(root: Path, fmt: str | list[str] | None = None, data_type: str = "L2")\ + -> dict: + """ + Build an in-memory pytree for an EHT dataset directory. + + Args: + root: Path to the EHT dataset root directory. + fmt: Format string for parsing data files. + data_type: Type of data to build ("L1" or "L2") + + Returns: + A dictionary with keys: + - "meta" : ParaFrame of housekeeping files + - "drives" : ParaFrame of compressed archives + - "data" : dict of {stem -> ParaFrame} + """ + # create clean root path + root = Path(root).expanduser().resolve() + # track files that are included in the tree to avoid double counting + tracked = set() + + # L1 data makes recursive calls for each directory, so use glob instead of rglob + glob_fn = root.glob if data_type == "L1" else root.rglob + + ### DRIVES ### + # collect all drive paths matching any supported extension + drive_paths = [] + extracted_dir_names = set() + for ext in DRIVE_EXTENSIONS: + # add any file that has this ext to the drive paths list and tracked set + for path in glob_fn(f"*{ext}"): + extract_dir = _extract_drive(path) + # only L1 data needs to track the extracted directory names for recursion + if data_type == "L1": + extracted_dir_names.add(extract_dir.name) + drive_paths.append(str(path.relative_to(root))) + tracked.add(str(path.relative_to(root))) + # build a single ParaFrame from all drive paths with extensions column + drives_pf = ParaFrame( + [{"path": path, "ext": Path(path).suffix.lstrip(".")} + for path in sorted(drive_paths)], + base_path=root,) + + ### DATA ### + data_branches = _build_data_branches(root, fmt, tracked) \ + if data_type == "L2" else {} + + ### META ### + # create a list of all files under root that aren't tracked + meta_files = { + str(file.relative_to(root)) + for file in glob_fn("*") + # add inventory item if its not a dir, not the .hm, and not in tracked + if file.is_file() + and ".hm" not in file.parts + and str(file.relative_to(root)) not in tracked + } + # create a paraframe for the meta files with extensions column + meta_pf = ParaFrame( + [{"path": file, "ext": Path(file).suffix.lstrip(".")} + for file in sorted(meta_files)], + base_path=root,) + for _, row in meta_pf.iterrows(): + tracked.add(row["path"]) + + # create dict with three keys, only data has subbranches + tree = {} + if not meta_pf.empty: + tree["meta"] = meta_pf + if not drives_pf.empty: + tree["drives"] = drives_pf + if data_branches: + tree["data"] = data_branches + + # recursive L1 tree construction + if data_type == "L1": + # call build_tree for each subdirectory + for subdir in sorted(p for p in root.iterdir() if p.is_dir()): + # calls with drives will end recursion + next_level = "L2" if subdir.name in extracted_dir_names else "L1" + # append each subdirectory to the tree + tree[subdir.name] = build_tree(subdir, fmt, data_type=next_level) + + return tree \ No newline at end of file diff --git a/mod/hallmark/fmt_detection.py b/mod/hallmark/fmt_detection.py new file mode 100644 index 0000000..a20c7da --- /dev/null +++ b/mod/hallmark/fmt_detection.py @@ -0,0 +1,289 @@ +import re +from pathlib import Path + +# common meta extensions to look for when building the fmts +META_EXTENSIONS = [".py", ".sh", ".md", ".pdf", ".rst", ".cfg", ".ini", ".yml", ".yaml"\ + , ".sl", ".par", ".xcm"] +# common meta files to look for when building fmts +KNOWN_META_FILES = {"README", "LICENSE", "LICENCE", "INVENTORY", "run"} +# common drive extensions to look for when building fmts +DRIVE_EXTENSIONS = [".tgz", ".tar", ".gz", ".zip", ".bz2", ".xz", ".zst", ".7z", ".rar"] +# delimiter characters fmt detection splits on +_DELIM_PATTERN = r"[_\-.]" + + +def _stems_to_fmts(stems: list[str]) -> list[str]: + """Cluster same-token-count stems into one fmt per shared literal pattern. + + Recursively call until all fmts are found + + Args: + stems: Stems (with extension reattached if present) that all have + the same number of tokens. + + Returns: + List of fmt strings, one per distinct cluster found. + """ + tokenized = {s: re.split(_DELIM_PATTERN, s) for s in stems} + # keep track of delimitors for fmt reconstruction + delimiters = {s: re.findall(_DELIM_PATTERN, s) for s in stems} + remaining_stems = list(stems) + fmts = [] + + while remaining_stems: + # check that every stem shares at least one token + reference_tokens = tokenized[remaining_stems[0]] + token_count = len(reference_tokens) + global_has_fixed = any( + # set will be length 1 if all stems have the same token at this index + len({tokenized[s][index] for s in remaining_stems}) == 1 + for index in range(token_count)) + # cluster found + if global_has_fixed: + cluster = list(remaining_stems) + + else: + # base the anchor around the first non-assigned stem + anchor = remaining_stems[0] + anchor_tokens = tokenized[anchor] + + # group remaining stems by which token positions they share with the anchor + stems_by_shared_positions : dict[tuple, list[str]] = {} + for stem in remaining_stems[1:]: + tokens = tokenized[stem] + shared_positions = tuple( + i for i in range(len(anchor_tokens)) + if tokens[i] == anchor_tokens[i]) + # if there are any tokens that match the anchor + if shared_positions: + stems_by_shared_positions.setdefault(shared_positions, \ + []).append(stem) + + if not stems_by_shared_positions: + # anchor matches nobody on any position and stem doesn't contain fmt + remaining_stems.remove(anchor) + continue + + # find the cluster with most members that share positions with the anchor + _, best_members = max(stems_by_shared_positions.items(), \ + key=lambda kv: len(kv[1])) + cluster = [anchor] + best_members + + # build the fmt directly from the cluster + cluster_tokenized = [tokenized[stem] for stem in cluster] + fmt_tokens = [] + field_count = 0 + has_fixed = False + for index, token in enumerate(cluster_tokenized[0]): + values = [tokens[index] for tokens in cluster_tokenized] + # if every stem has the same token at this position + if len(set(values)) == 1: + fmt_tokens.append(token) + has_fixed = True + else: + # if the token is different for any, its a parameter + fmt_tokens.append(f"{{p{field_count}}}") + field_count += 1 + + # concatenate the fmt from the tokens and delimiters + if has_fixed and field_count > 0: + cluster_delims = delimiters[cluster[0]] + fmt_parts = [fmt_tokens[0]] + for token, delim in zip(fmt_tokens[1:], cluster_delims): + fmt_parts.append(delim) + fmt_parts.append(token) + fmts.append("".join(fmt_parts)) + # remove all the stems that fit this fmt + for stem in cluster: + remaining_stems.remove(stem) + + return fmts + +def _is_extendable_variant(longer: str, shorter: str) -> str | None: + """ + Check if `shorter` and `longer` are compatible fmt strings, where `longer` + has one additional token that can be removed to match `shorter`. + + Args: + longer: A fmt string with one more token than `shorter`. + shorter: A fmt string with one less token than `longer`. + + Returns: + A new fmt string with the additional token in `longer` replaced by a + parameter placeholder, if the two fmt strings are compatible. Otherwise, + return None. + """ + longer_tokens = re.split(_DELIM_PATTERN, longer) + shorter_tokens = re.split(_DELIM_PATTERN, shorter) + + # if longer doesn't have a single extra token, they can't be compatible + if len(longer_tokens) != len(shorter_tokens) + 1: + return None + + # collect every compatible dropped position + candidates = [] + # check each token position in the longer fmt for compatibility with the shorter fmt + for i in range(len(longer_tokens)): + # remove the token at index i from longer and compare to shorter + candidate = longer_tokens[:i] + longer_tokens[i + 1:] + compatible = True + param_positions = {i} + # check that every token in candidate matches the corresponding token in shorter + for k, (candidate_token, shorter_token) in enumerate( + zip(candidate, shorter_tokens)): + # correcting for the removed token index + longer_idx = k if k < i else k + 1 + + # check if either token is already a parameter placeholder + if re.fullmatch(r"\{p\d+\}", candidate_token) or \ + re.fullmatch(r"\{p\d+\}", shorter_token): + # track the index of the parameter placeholder + param_positions.add(longer_idx) + continue + # if the tokens are not equal, they are incompatible + if candidate_token != shorter_token: + compatible = False + break + if compatible: + # record the num of params, the index, and the positions + candidates.append((len(param_positions), i, param_positions)) + # don't merge if it will result in a fmt with no fixed tokens + if not candidates: + # the shorter fmt is not a compatible variant of the longer fmt + return None + + # find the candidate with the fewest parameter placeholders + min_params = min(count for count, _, _ in candidates) + # only keep candidates with the minimum number of parameter placeholders + best = [c for c in candidates if c[0] == min_params] + # sort by the index of the dropped token in the longer fmt, descending + best.sort(key=lambda c: c[1], reverse=True) + # select the candidate with the highest dropped token index to keep leftmost tokens + _, _, param_positions = best[0] + + # safety check that they aren't the same length, meaning there is no variance + if len(param_positions) == len(longer_tokens): + return None + + # rebuild the fmt string with the parameter placeholders renumbered + new_tokens = [] + field_count = 0 + for index, token in enumerate(longer_tokens): + # if we are at the removed token index or a parameter placeholder + if index in param_positions: + new_tokens.append(f"{{p{field_count}}}") + field_count += 1 + else: + new_tokens.append(token) + + # get the delimiters from the longer fmt string to reconstruct the final fmt + delims = re.findall(_DELIM_PATTERN, longer) + tokens = [new_tokens[0]] + # reconstruct the fmt string with the delimiters in between the tokens + for token, delim in zip(new_tokens[1:], delims): + tokens.append(delim) + tokens.append(token) + return "".join(tokens) + + +def scan_inventory(root: Path) -> list[str]: + """ + Scan a directory and return all file paths found, recursively. + + Args: + root: Path to the dataset root directory to scan. + + Returns: + List of relative file path strings found under root. + + Raises: + FileNotFoundError: If root does not exist. + """ + root = Path(root).expanduser().resolve() + if not root.is_dir(): + raise FileNotFoundError(f"{root} is not a directory") + + return sorted( + # convert to relative path strings for consistency with fmt detection + str(file.relative_to(root)) + for file in root.rglob("*") + # only include files, and ignore any hallmark meta files + if file.is_file() and ".hm" not in file.parts + ) + + +def detect_fmt(root: Path) -> list[str]: + """ + Auto-detect format strings from the files in the directory. + + Args: + root: Path to the dataset root directory containing the files. + + Returns: + List of fmt strings without extensions, one per distinct file + structure found in the files list. + """ + root = Path(root).expanduser().resolve() + inventory_files = scan_inventory(root) + + drive_exts = {ext.lstrip(".").lower() for ext in DRIVE_EXTENSIONS} + meta_exts = {ext.lstrip(".").lower() for ext in META_EXTENSIONS} + + data_stems = [] + seen = set() + for file in inventory_files: + path = Path(file) + # separate the extension from the stem + extension = path.suffix.lstrip(".").lower() + stem = path.stem + + # skip any files that are known meta files or have known meta/drive extensions + if extension in drive_exts or extension in meta_exts: + continue + if stem.split(".")[0] in KNOWN_META_FILES: + continue + # check this stem hasn't already been added to the list + if stem not in seen: + seen.add(stem) + data_stems.append(stem) + + # filter stems by token count to group them for fmt detection + by_token_count: dict[int, list[str]] = {} + for stem in data_stems: + # split the stem up by delimiters + token_count = len(re.split(_DELIM_PATTERN, stem)) + # add the stem to the list of stems with the same token count + by_token_count.setdefault(token_count, []).append(stem) + + group_fmts = [] + for token_count, same_count_stems in sorted(by_token_count.items()): + # create the fmt strings for each group of stems with the same token count + group_fmts.extend(_stems_to_fmts(same_count_stems)) + + # merge fmts that are the same except for an optional param + group_fmts.sort(key=lambda f: len(re.split(_DELIM_PATTERN, f)), reverse=True) + merged = [] + # check if the fmt has already been merged to avoid duplicates + used = [False] * len(group_fmts) + for i, longer in enumerate(group_fmts): + if used[i]: + continue + current = longer + for j, shorter in enumerate(group_fmts): + # skip if the fmt is the same or has already been merged + if i == j or used[j]: + continue + # check if the shorter fmt is an extendable variant of the current fmt + extended = _is_extendable_variant(current, shorter) + if extended is not None: + current = extended + used[j] = True + merged.append(current) + used[i] = True + + fmts = [] + for fmt in merged: + # avoid adding duplicate fmts to the final list + if fmt not in fmts: + fmts.append(fmt) + return fmts \ No newline at end of file diff --git a/mod/hallmark/helper_functions.py b/mod/hallmark/helper_functions.py index 3ed5618..0802c4d 100644 --- a/mod/hallmark/helper_functions.py +++ b/mod/hallmark/helper_functions.py @@ -13,6 +13,7 @@ # limitations under the License. import re +import pandas as pd def find_spec_by_fmt(fmt, encodings): @@ -59,3 +60,34 @@ def regex_sub(value, yaml_encodings): result = re.sub(match.group(0), "-" + str(match.group(1)), result) return result + +def try_numeric_conversion(series): + """ + Attempt to convert a pandas Series to numeric. + + Converts the series to numeric iff: + 1. All values are numeric + 2. Converting back to string matches the original values to avoid + unintended conversions (e.g., "001" -> 1) + + Args: + series: A pandas Series of strings to attempt conversion on. + + Returns: + The converted numeric Series if both conditions are met, + otherwise returns original series. + """ + # replace unconvertible values with NaN + converted = pd.to_numeric(series, errors="coerce") + # if any values were unconvertible, return original series + if converted.isna().any(): + return series + # if converting back to str doesn't match original, return original series + # prevents unintended conversions like "001" -> 1 + if not all(str(int(numeric_val)) == str(original_val) + or str(numeric_val) == str(original_val) + # check each pair of converted and original values + for numeric_val, original_val in zip(converted, series)): + return series + return converted + diff --git a/mod/hallmark/objects.py b/mod/hallmark/objects.py index 0c5dd30..8e518ea 100644 --- a/mod/hallmark/objects.py +++ b/mod/hallmark/objects.py @@ -1,3 +1,11 @@ +""" +Utilities for storing and restoring content-addressed objects. + +This module defines the ``Objects`` class, which manages the Hallmark +object store. Files are stored using their SHA-1 checksum and can later +be restored to a specified location. +""" + import os import shutil from pathlib import Path @@ -5,13 +13,52 @@ class Objects: + """ + Manage the Hallmark object store. + + Files are stored in a content-addressed directory structure based on + their SHA-1 checksum, allowing them to be efficiently retrieved and + restored. + """ def __init__(self, path: Union[Path, str]): + """ + Initialize the object store. + + Args: + path (Path or str): Path to the Hallmark repository. The object + store is located in the ``objects`` subdirectory. + """ self.root = Path(path) / "objects" def _split_checksum(self, sha1: str) -> Path: + """ + Convert a SHA-1 checksum into its storage path. + + The first two characters of the checksum form the directory name, + while the remaining characters form the filename. + + Args: + sha1 (str): SHA-1 checksum. + + Returns: + Path: Path where the object is stored. + """ return self.root / sha1[:2] / sha1[2:] def store(self, src: Path, sha1: str) -> Path: + """ + Store a file in the object store. + + The file is copied only if an object with the same checksum does not + already exist. + + Args: + src (Path): Source file to store. + sha1 (str): SHA-1 checksum of the file. + + Returns: + Path: Path to the stored object. + """ stored_checksum = self._split_checksum(sha1) if not stored_checksum.exists(): stored_checksum.parent.mkdir(parents=True, exist_ok=True) @@ -19,6 +66,19 @@ def store(self, src: Path, sha1: str) -> Path: return stored_checksum def restore(self, sha1: str, dest: Path) -> Path: + """ + Restore an object from the object store. Raises FileNotFoundError if + the requested object does not exist in the object store. + + The stored object is hard-linked to the destination path. + + Args: + sha1 (str): SHA-1 checksum of the stored object. + dest (Path): Destination path for the restored file. + + Returns: + Path: Path to the restored file. + """ stored = self._split_checksum(sha1) if not stored.exists(): raise FileNotFoundError(f"object {sha1} not found in objects store") diff --git a/mod/hallmark/paraframe.py b/mod/hallmark/paraframe.py index ef50874..915c397 100644 --- a/mod/hallmark/paraframe.py +++ b/mod/hallmark/paraframe.py @@ -12,6 +12,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +""" +Utilities for discovering and parsing parameterized file collections. + +This module defines :class:`ParaFrame`, a subclass of +:class:`pandas.DataFrame` that provides convenient methods for locating, +parsing, and filtering files whose names follow parameterized naming +conventions. +""" from glob import glob from pathlib import Path @@ -21,37 +29,56 @@ import pandas as pd import numpy as np -from .helper_functions import find_spec_by_fmt, regex_sub +from .helper_functions import find_spec_by_fmt, regex_sub, try_numeric_conversion class ParaFrame(pd.DataFrame): """ - A subclass of :class:`pandas.DataFrame` with added methods for + A subclass of :class:`pandas.DataFrame` with additional methods for parameterized file discovery and filtering. - ``ParaFrame`` instances behave like ordinary DataFrames but add: - - - * ``__init__``: Initialises the class and stores ``encodings`` and - ``base_path`` as metadata - to the ParaFrame. - * ``_constructor``: Returns the subclassed ParaFrame with repo_path as a - default keyword argument. - * ``glob_search'': Returns files found in the directory using format string. - * ``parse``: a classmethod that builds a table of file paths and parsed - parameters from a format pattern (using ``glob`` + ``parse``). - * ``__call__``/``filter``: convenience filtering by column values. + ``ParaFrame`` behaves like a standard ``pandas.DataFrame`` while + providing the following additional functionality: + + * ``__init__``: Initializes the class and stores ``encodings`` and + ``base_path`` as metadata. + * ``_constructor``: Returns a new ``ParaFrame`` while preserving + ``encodings`` and ``base_path``. + * ``glob_search``: Finds files matching a parameterized format string. + * ``parse``: Builds a ``ParaFrame`` by parsing file paths that match + a format pattern. + * ``__call__`` and ``filter``: Convenience methods for filtering rows + by column values. """ _metadata = ["encodings", "base_path"] def __init__(self, data=None, encodings=None, base_path=None, **kwargs): + """ + Initialize a ``ParaFrame``. + + Args: + data: Data used to initialize the ``ParaFrame``. + encodings (dict, optional): Filename encoding specifications. + Defaults to ``{}``. + base_path (Path or str, optional): Root directory associated with + the ``ParaFrame``. Defaults to the current working directory. + **kwargs: Additional arguments passed to + :class:`pandas.DataFrame`. + """ super().__init__(data, **kwargs) self.encodings = encodings or {} self.base_path = Path(base_path) if base_path is not None else Path.cwd() @property def _constructor(self): + """ + Return the constructor used by pandas operations. + + Returns: + callable: A constructor that preserves ``encodings`` and + ``base_path`` metadata. + """ def _c(*args, **kwargs): kwargs.setdefault("encodings", self.encodings) kwargs.setdefault("base_path", self.base_path) @@ -59,28 +86,35 @@ def _c(*args, **kwargs): return _c def __call__(self, **kwds): + """ + Filter the ``ParaFrame``. + + Equivalent to calling :meth:`filter`. + + Args: + **kwds: Column names and values used for filtering. + + Returns: + pandas.DataFrame: The filtered ``ParaFrame``. + """ return self.filter(**kwds) def filter(self, **kwargs): """ - Filter a pandas ``DataFrame`` by matching column values. + Filter a ``ParaFrame`` by matching column values. - This function utlizes provided **kwargs to filter an existing - ``ParaFrame`` by masking based on column values. Filtering supports - single- and multi-conditioned queries, returning rows that satisfy - any of the provided conditions. + This method filters the current ``ParaFrame`` by applying one or more + conditions on its columns. Rows satisfying any of the specified + conditions are returned. Args: - **kwargs: Arbitrary keyword arguments specifying column names - and values to filter by. - * If the value is a tuple or list, rows where the column - matches any of those values are selected. - * If the value is a scalar, rows where the column equals - the value are selected. + ``**kwargs``: Keyword arguments specifying column names and values to + filter by. Values may be scalars or sequences (lists or tuples). + Sequence values match any element in the sequence. Returns: - pandas.DataFrame: A filtered DataFrame containing only rows - that match the given conditions. + pandas.DataFrame: A filtered DataFrame containing only the rows + that satisfy the requested conditions. """ mask = np.zeros(len(self), dtype=bool) for k, v in kwargs.items(): @@ -103,38 +137,47 @@ def glob_search( **kwargs, ): """ - Find all the files specified in a directory using the format string specified. + Find files matching a parameterized format string. - This function utilizes the provided format string to find the specified - files. This function also looks through .yaml files if encoding = True - and evaluates conditionals. + This method searches for files using the supplied format string. + When ``encoding=True``, regular-expression encodings defined in the + YAML configuration are also applied. Args: - fmt (str): A format string specifying the expected file naming - pattern. - Fields wrapped in ``{}`` will be extracted into columns. - *args: Positional arguments used to fill the format string. - encodings (dict): The ``encodings`` list from ``State`` - (contents of ``config.yml``). - Defaults to ``{}``. - base_path (Path): Root directory to search from. - Defaults to ``Path.cwd()``. - debug (bool, optional): If True, prints debugging information - about the matching process. - Defaults to False. - return_pattern (bool, optional): Returns the pattern and globbed files. - Defaults to False. - encodings (bool,optional): If True, looks for the .yaml file and - extracts user specified format information. - Defaults to False. - **kwargs: Keyword arguments used to fill the format string. - If missing keys are encountered, they will be replaced by - a wildcard ``*`` for globbing. + fmt (str): + Format string describing the expected filename pattern. + + ``*args``: + Positional arguments used to fill the format string. + + encodings (dict): + Encoding specifications from ``config.yml``. + + base_path (Path): + Root directory to search. + + debug (bool): + If ``True``, prints debugging information. + + return_pattern (bool): + If ``True``, returns both the glob pattern and the matched + files. + + encoding (bool): + If ``True``, applies regex encodings defined in the YAML + configuration. + + ``**kwargs``: + Keyword arguments used to fill the format string. + Missing values are replaced with the wildcard ``*``. Returns: - if return_pattern = True, it returns the pattern and the globbed files. - Else, it returns the globbed files, the format string with the wildcards - and the user specification in the .yaml file (None, if encoding = False). + tuple: + If ``return_pattern`` is ``True``, returns + ``(globbed_files, pattern)``. + + Otherwise returns + ``(yaml_encodings, fmt_g, globbed_files)``. """ encodings = encodings or {} base_path = Path(base_path) if base_path is not None else Path.cwd() @@ -187,8 +230,8 @@ def glob_search( break except KeyError as e: k = e.args[0] - pattern = re.sub(r"\{" + k + r":?.*?\}", "{" + k + ":s}", pattern) - fmt_g = re.sub(r"\{" + k + r":?.*?\}", "{" + k + ":g}", fmt_g) + pattern = re.sub(r"\{" + k + r":?.*?\}", "{" + k + "}", pattern) + fmt_g = re.sub(r"\{" + k + r":?.*?\}", "{" + k + "}", fmt_g) kwargs[k] = "*" # Obtain list of files based on the glob pattern @@ -221,28 +264,33 @@ def parse( encoding=False, **kwargs, ): - """Build a ``ParaFrame`` by parsing file paths that match a pattern. + """ + Build a ``ParaFrame`` by parsing file paths that match a format + string. Args: - fmt (str): Format string with ``{param}`` fields. - encodings (dict): The ``encodings`` dict from ``State`` - (contents of ``hallmark.yml``). - Defaults to ``{}``. - base_path (Path): Root directory to search from. - Defaults to ``Path.cwd()``. - debug (bool): Print debug info. Defaults to ``False``. - encoding (bool): Apply regex encoding. Defaults to ``False``. + fmt (str): + Format string containing ``{parameter}`` fields. + + encodings (dict): + Encoding specifications from ``config.yml``. + Defaults to ``{}``. + + base_path (Path): + Root directory to search. + Defaults to ``Path.cwd()``. + + debug (bool): + If ``True``, prints debugging information. + Defaults to ``False``. + + encoding (bool): + If ``True``, applies regex encoding. + Defaults to ``False``. Returns: - ``ParaFrame`` where each row is a matched file with parsed - parameters as columns, plus a ``path`` column. - - Example: - >>> from hallmark import ParaFrame - >>> pf = ParaFrame.parse( - ... "/custom_parameter{custom_parameter}_p{parameter}.h5", - ... encoding=True - ... ) + ParaFrame: A ``ParaFrame`` whose rows correspond to matched files. + Parsed parameters are stored as columns together with a ``path`` column. """ base_path = Path(base_path) if base_path is not None else Path.cwd() @@ -272,4 +320,9 @@ def parse( print(f'Failed to parse "{f}"') else: frame.append({"path": f_short, **r.named}) - return cls(frame, encodings=encodings, base_path=base_path) + # attempt to convert each parameter column to numeric + # if conversion fails the column stays as string + result = cls(frame, encodings=encodings, base_path=base_path) + for col in result.columns: + result[col] = try_numeric_conversion(result[col]) + return result diff --git a/mod/hallmark/repo.py b/mod/hallmark/repo.py index f35278f..f333932 100644 --- a/mod/hallmark/repo.py +++ b/mod/hallmark/repo.py @@ -38,6 +38,14 @@ @contextmanager def chdir(path): + ''' + Temporarily change the current working directory. No Yields. + + Args: + Path: Directory to dwitch to while inside the context. + Returns: + None. + ''' old = os.getcwd() os.chdir(path) try: @@ -48,7 +56,8 @@ def chdir(path): @dataclass(init=False) class Repo: - """Hallmark repository. + """ + Hallmark repository. This is the Python API boundary. It loads the in-memory ``State`` from repository ``Dothm``, and @@ -61,12 +70,31 @@ class Repo: @staticmethod def lwpaths(path: Union[Path, str]) -> Tuple[Path, Optional[Path]]: + ''' + Resolve repository and worktree paths. + + Args: + path (Path | str): Path to either a worktree or a + ``.hm`` repository. + + Returns: + tuple[Path, Path | None]: A ``(dothm_path, worktree_path)`` tuple. + If ``path`` refers to a ``.hm`` directory, ``worktree_path`` is ``None``. + ''' path = Path(path).resolve() if path.suffix == ".hm": return path, None return path / ".hm", path def __init__(self, path: Union[Path, str]) -> None: + ''' + Open an existing hallmark repository. + + Args: + path: path to a worktree or '.hm repository'/ + Returns: + none. + ''' dothm_path, worktree_path = self.lwpaths(path) self.dothm = Dothm(dothm_path) self.worktree = worktree_path and Worktree(worktree_path) @@ -83,6 +111,15 @@ def __init__(self, path: Union[Path, str]) -> None: @classmethod def init(cls, path: Union[Path, str]) -> "Repo": + ''' + Initialize a new hallmark repository. + + Args: + paht(paht|string): path to initialize a worktree or ``.hm`` repository. + + Returns: + Repo: newly created repository instance + ''' dothm_path, worktree_path = cls.lwpaths(path) dothm = Dothm.init(dothm_path) (dothm.path/"config.yml").write_text(Dothm.config_template(),encoding="utf-8") @@ -103,6 +140,20 @@ def clone( max_workers: int = 4, show_progress: bool = False, ) -> "Repo": + ''' + Clone a remote hallmark repository. Raises DestinationExistsError + if the destination path already exists. Raises DownloadError if + data download is enabled and files fail to download. + + Args: + url(string): remote repository URL. + path(path|string): destination path for clone. + fetch_data (boolean): if true, downloads associated data files. + max_workers (integer): Number of parallel workers for downloading data. + show_progress (boolean): wether to display download progress. + Returns: + Repo: Cloned Repository instance + ''' clone_path = Path(path) if clone_path.exists(): raise DestinationExistsError( @@ -146,6 +197,15 @@ def clone( @staticmethod def checksum(path: Path, chunk_size: int = 1024 * 1024) -> str: + ''' + Computes the SHA1 checksum of a file. + + Args: + path (path): path to the file to hash. + chunk_size (integer): size of chunks used for streaming reads. + Returns: + String: Hexadecimal SHA1 digest of the file contents. + ''' digest = sha1() with path.open("rb") as f: for block in iter(lambda: f.read(chunk_size), b""): @@ -153,6 +213,10 @@ def checksum(path: Path, chunk_size: int = 1024 * 1024) -> str: return digest.hexdigest() def add_paths(self, paths: List[Union[Path, str]]) -> ParaFrame: + ''' + Add explicit file paths to the repository index. Raises RuntimeError. + Operation not supported in Hallmark. + ''' raise RuntimeError( 'explicit path add is not supported while data.tsv ' \ 'stores only sha1 plus fmt fields') @@ -165,6 +229,18 @@ def set_config( remote_url: Optional[str] = None, encoding_updates: Optional[Dict[str, str]] = None, ) -> dict: + """ + Update repository configuration values. + + Args: + fmt (str, optional): Data format specification. + remote_name (str, optional): Name of the remote repository. + remote_url (str, optional): URL of the remote repository. + encoding_updates (dict[str, str], optional): Updates to encoding rules. + + Returns: + dict: Updated configuration dictionary. + """ repo_set_config( self, fmt=fmt, @@ -176,6 +252,20 @@ def set_config( return self.state.config def status(self) -> dict[str, object]: + """ + Return repository status information. Includes staged changes, + orktree modifications, deletions, and untracked files. + + Args: + none? + + Returns: + dict[str, object]: Status summary including: + - branch (str) + - staged changes (dict) + - worktree changes (dict) + - untracked files (list[str]) + """ head_state = load_head_state(self) head_map = manifest_map(head_state) staged_map = manifest_map(self.state) @@ -230,6 +320,15 @@ def status(self) -> dict[str, object]: } def add(self, fstr: str, encoding: bool = False) -> ParaFrame: + ''' + Stage files or updated repository indecing from the worktree. + + Args: + fstr (string): Format string or "." for full directory scan. + encoding (boolean): Whether to apply encoding rules. + Returns: + paraframe Parsed and filtered file index (without checksums). + ''' if self.worktree is None: raise RuntimeError( "cannot add files in a bare repository without a worktree") @@ -284,6 +383,16 @@ def add(self, fstr: str, encoding: bool = False) -> ParaFrame: return pf.drop(columns=["sha1"], errors="ignore") def commit(self, msg: str, allow_empty: bool = False) -> bool: + ''' + Commit staged changes to the repository. Raises ValueError if commit + message is empty or invalid. + + Args: + msg (string): commit message. + allow_empty (boolean): Allow comitting even if no changes exists. + Returns: + boolean: True if a commit was created, false otherwise. + ''' if not isinstance(msg, str) or not msg.strip(): raise ValueError("commit message must be a non-empty string") @@ -296,16 +405,42 @@ def commit(self, msg: str, allow_empty: bool = False) -> bool: return False def log(self) -> str: + ''' + Return commit history log. + + Returns: + string: Git log output, or an empty string if no valid HEAD exists. + ''' if not self.dothm.head.is_valid(): return "" return self.dothm.git.log() def branches(self) -> dict[str, object]: + ''' + List repository branches. + + Returns: + dictionary[string, object]: Dictionary containing: + - ``current`` (string): Active branch name + - ``names``: All branch names + ''' current = self.dothm.active_branch.name names = sorted(head.name for head in self.dothm.heads) return {"current": current, "names": names} def checkout(self, target_branch: str) -> bool: + ''' + Switch to a different branch and update the worktree. Raises ValueError if + branch name is invalid. Raises CheckoutError if the workign directoary + is not clean or checkout can't be completed safely. + + Args: + target_branch (string): Branch to switch to. + Returns: + boolean: True if checkout succeeds. + + + ''' if not isinstance(target_branch, str) or not target_branch.strip(): raise ValueError("branch name must be a non-empty string") @@ -364,6 +499,16 @@ def checkout(self, target_branch: str) -> bool: return True def add_worktree(self, target_branch: str) -> bool: + ''' + Create or link a new worktree for a branch. Raises ValueError if branch name + is invalid. + Rasies Runtime error if called in a bare repository or worktree creation fails. + + Args: + target_branch (string): Name of the branch to attach. + Returns: + boolean: True if the worktree was successfully created. + ''' from shutil import copy2 from git.exc import GitCommandError diff --git a/mod/hallmark/repo_config.py b/mod/hallmark/repo_config.py index 49a3ed0..a94981e 100644 --- a/mod/hallmark/repo_config.py +++ b/mod/hallmark/repo_config.py @@ -1,11 +1,30 @@ +""" +Utilities for managing Hallmark repository configuration. + +This module provides helper functions for reading, validating, and +updating repository configuration values stored in ``config.yml``. +""" + from __future__ import annotations from string import Formatter from pathlib import Path from typing import Dict, Optional - +#test def ensure_branch_data_spec(config: dict) -> dict: + """ + Ensure the configuration contains a valid data specification. + + If the ``data`` entry is missing or malformed, it is initialized with + a single empty dictionary. + + Args: + config (dict): Repository configuration. + + Returns: + dict: The branch data specification. + """ data = config.get("data") if not isinstance(data, list) or len(data) != 1 or not isinstance(data[0], dict): config["data"] = [{}] @@ -13,6 +32,16 @@ def ensure_branch_data_spec(config: dict) -> dict: def branch_data_spec(repo) -> dict: + """ + Return the branch data specification. Raises RuntimeError if + the configuration does not define exactly one entry under ``data``. + + Args: + repo: Repository object. + + Returns: + dict: The branch data specification. + """ data = repo.state.config.get("data") if not isinstance(data, list) or len(data) != 1 or not isinstance(data[0], dict): raise RuntimeError('branch config must define exactly ' \ @@ -21,6 +50,16 @@ def branch_data_spec(repo) -> dict: def branch_fmt(repo) -> str: + """ + Return the configured filename format. Raises RuntimeError if + no valid format string is defined. + + Args: + repo: Repository object. + + Returns: + str: The format string stored in ``data[0].fmt``. + """ fmt = branch_data_spec(repo).get("fmt") if not isinstance(fmt, str) or not fmt.strip(): raise RuntimeError('branch config must define one ' \ @@ -29,6 +68,13 @@ def branch_fmt(repo) -> str: def set_branch_fmt(repo, fmt: str) -> None: + """ + Set the branch filename format. + + Args: + repo: Repository object. + fmt (str): New filename format. + """ set_config(repo, fmt=fmt) @@ -40,6 +86,23 @@ def set_config( remote_url: Optional[str] = None, encoding_updates: Optional[Dict[str, str]] = None, ) -> dict: + """ + Update the repository configuration. + + Existing configuration values are preserved unless explicitly + replaced. + + Args: + repo: Repository object. + fmt (str, optional): Filename format. + remote_name (str, optional): Remote repository name. + remote_url (str, optional): Remote repository URL. + encoding_updates (dict, optional): Encoding values to merge into + the existing configuration. + + Returns: + dict: The updated configuration. + """ config = repo.state.config spec = ensure_branch_data_spec(config) @@ -88,11 +151,30 @@ def set_config( def branch_encodings(repo) -> list[dict]: + """ + Return the configured filename encodings. + + Args: + repo: Repository object. + + Returns: + list[dict]: A list containing the encoding specification, or an + empty list if no encodings are defined. + """ spec = branch_data_spec(repo) return [spec] if isinstance(spec.get("encoding"), dict) else [] def fmt_fields(fmt: str) -> list[str]: + """ + Extract field names from a format string. + + Args: + fmt (str): Format string containing replacement fields. + + Returns: + list[str]: Unique field names in the order they appear. + """ fields: list[str] = [] for _, field_name, _, _ in Formatter().parse(fmt): if field_name and field_name not in fields: @@ -101,6 +183,16 @@ def fmt_fields(fmt: str) -> list[str]: def coerce_fmt_value(value: str, spec: str): + """ + Convert a value according to a format specification. + + Args: + value (str): Value to convert. + spec (str): Format specification. + + Returns: + The converted value. + """ if not spec: return value if spec.endswith("d"): @@ -111,6 +203,16 @@ def coerce_fmt_value(value: str, spec: str): def row_to_path(row, fmt: str) -> Path: + """ + Construct a file path from a table row. + + Args: + row: Table row containing field values. + fmt (str): Format string used to build the path. + + Returns: + Path: Path generated from the row values. + """ values = {} for _, field_name, format_spec, _ in Formatter().parse(fmt): if field_name: @@ -119,4 +221,18 @@ def row_to_path(row, fmt: str) -> Path: def path_from_row(repo, row, fmt: Optional[str] = None) -> Path: + """ + Construct a file path from a table row. + + If no format string is provided, the repository's configured format + is used. + + Args: + repo: Repository object. + row: Table row containing field values. + fmt (str, optional): Format string to use. + + Returns: + Path: Path generated from the row values. + """ return row_to_path(row, fmt or branch_fmt(repo)) diff --git a/mod/hallmark/repo_manifest.py b/mod/hallmark/repo_manifest.py index 72914e7..1edde50 100644 --- a/mod/hallmark/repo_manifest.py +++ b/mod/hallmark/repo_manifest.py @@ -9,6 +9,21 @@ def manifest_frame_from_pf(pf, fmt: str) -> pd.DataFrame: + """ + Build a manifest table from a ``ParaFrame``. Raises RuntimeError + if a file path cannot be parsed using ``fmt``. + + The returned table contains a ``sha1`` column together with the + fields extracted from the configured filename format. + + Args: + pf: ``ParaFrame`` containing indexed file paths. + fmt (str): Filename format used to parse the paths. + + Returns: + pandas.DataFrame: Manifest table containing ``sha1`` values and + parsed filename fields. + """ if pf.empty: return pd.DataFrame(columns=["sha1", *fmt_fields(fmt)]) @@ -25,6 +40,16 @@ def manifest_frame_from_pf(pf, fmt: str) -> pd.DataFrame: def manifest_map(state) -> dict[str, str]: + """ + Create a mapping from file paths to SHA-1 checksums. + + Args: + state: Repository state. + + Returns: + dict[str, str]: Dictionary mapping relative file paths to their + corresponding SHA-1 checksums. + """ if state.data.empty: return {} data = state.config.get("data") @@ -40,6 +65,20 @@ def manifest_map(state) -> dict[str, str]: def frame_from_paths(repo, rel_paths: list[Path]): + """ + Build a ``ParaFrame`` from repository paths. + + Each path is converted into a row containing the relative path and + its SHA-1 checksum. + + Args: + repo: Repository object. + rel_paths (list[Path]): Relative paths within the repository. + + Returns: + ParaFrame: ``ParaFrame`` containing the indexed paths and their + checksums. + """ records = [{"path": str(path)} for path in rel_paths] pf = repo.paraframe_cls(records, base_path=repo.worktree) if not pf.empty: diff --git a/mod/hallmark/repo_state.py b/mod/hallmark/repo_state.py index 7b4ba9e..e7467b4 100644 --- a/mod/hallmark/repo_state.py +++ b/mod/hallmark/repo_state.py @@ -10,6 +10,17 @@ def load_branch_config(repo, branch: str) -> dict: + ''' + Load the configuration for a specific branch + + Args: + repo (Repo): repository object containing Git and state information/ + branch (String): Name of the branch whose configuration should be loaded + Returns: + (dict) A dictionary contaiing the branch configuration. Returns the + parsed contents of ``config.yml`` from the specified branch when + available, otherwise returns a copy of ``repo.state.config``. + ''' try: return yaml.safe_load(repo.dothm.git.show(f"{branch}:config.yml")) or {} except GitCommandError: @@ -17,6 +28,16 @@ def load_branch_config(repo, branch: str) -> dict: def load_branch_meta(repo, branch: str) -> dict: + ''' + Load the metadata for a specific branch + + Args: + repo (Repo): repository object + branch (String): Name of the branch + Returns: + The contents of ``meta.yml`` from the specified branch. If + metadata can't be loaded, returns ``repo.state.meta``. + ''' try: return yaml.safe_load(repo.dothm.git.show(f"{branch}:meta.yml")) or {} except GitCommandError: @@ -24,6 +45,17 @@ def load_branch_meta(repo, branch: str) -> dict: def load_branch_data(repo, branch: str) -> State: + ''' + Load the state associated with a branch + + Args: + repo (Repo): repository object + branch (String): branch name + Returns: + (State) A ``State`` consturcted from the branch's ``config.yml``, + ``meta.yml``, and ``data.tsv`` files. If the branch does + not exist, returns a copy of the current repository state. + ''' if branch in {head.name for head in repo.dothm.heads}: data = repo.dothm.git.show(f"{branch}:data.tsv") frame = State().data if not data.strip() else None @@ -45,6 +77,18 @@ def load_branch_data(repo, branch: str) -> State: def load_head_state(repo) -> State: + ''' + Load the state stored at ``Head``. + + Args: + repo (Repo): Repository object. + + Returns: + (State) A ``State`` constructed from the ``config.yml``, ``meta.yml``, + and``data.tsv`` files at ``HEAD``. If no state can be loaded + from ``HEAD``, returns a state with the current configuration + and metadata and an empty data table. + ''' try: data = repo.dothm.git.show("HEAD:data.tsv") except GitCommandError: diff --git a/mod/hallmark/repo_worktree.py b/mod/hallmark/repo_worktree.py index 1122bc3..5ebefc7 100644 --- a/mod/hallmark/repo_worktree.py +++ b/mod/hallmark/repo_worktree.py @@ -8,6 +8,16 @@ def effective_cwd(repo) -> Path: + ''' + Determine the effective working directory for repository operations. + Raises RuntimeError if the repository has no worktree + + Args: + repo (repo): repository object + Returns: + path: The current working directory if it is inside the repository + worktree; otherwise, the worktree root. + ''' if repo.worktree is None: raise RuntimeError("cannot inspect files in a bare repository " \ "without a worktree") @@ -22,6 +32,17 @@ def effective_cwd(repo) -> Path: def filtered_paraframe(repo, pf): + ''' + Filter a paraframe to the effective working directory. + + Args: + repo (Repo): Repository object. + pf (ParaFrame): paraframe to filter. + Returns: + Paraframe: The original paraframe if operating at the worktree root. + Otherwise, only rows whose paths lie within the effective working + directory. + ''' root = effective_cwd(repo) worktree = Path(repo.worktree) if root == worktree: @@ -33,11 +54,32 @@ def filtered_paraframe(repo, pf): def tracked_paths(repo) -> set[Path]: + ''' + Return the set of tracked file paths. + + Args: + repo (Repo): Repository object. + + Returns: + set[Path]: Paths of all files tracked in the current + repository state. + ''' fmt = branch_fmt(repo) return {path_from_row(repo, row, fmt) for _, row in repo.state.data.iterrows()} def ensure_clean_tracked_files(repo) -> None: + ''' + Verify that tracked files and repository state are clean. No returns. + Raises CheckoutError if the reposiotry has no worktree, a tracked + file is missing, a tracked file has uncommited changes, or the + hallmark state contains uncommitted changes. + + Args: + repo (Repo): Repository object + Returns: + None. + ''' if repo.worktree is None: raise CheckoutError("cannot checkout without a worktree") diff --git a/mod/hallmark/state.py b/mod/hallmark/state.py index 967dd0e..8701115 100644 --- a/mod/hallmark/state.py +++ b/mod/hallmark/state.py @@ -4,7 +4,7 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -23,13 +23,14 @@ @dataclass class State: - """In-memory hallmark state database. + """ + In-memory Hallmark state database. Attributes: config: Repository configuration values. - meta: Metadata. - data: Tabular file index; each row is keyed by path/change and - stores the indexed object checksum (``sha1``). + meta: Repository metadata. + data: Tabular file index containing indexed object checksums + (``sha1``) and associated metadata. """ config: dict = field(default_factory=dict) @@ -39,6 +40,18 @@ class State: ) def update(self, pf): + """ + Merge ``ParaFrame`` rows into the state database. + + Existing rows with matching keys are updated, while new rows are + appended. + + Args: + pf (ParaFrame): ``ParaFrame`` containing rows to add or update. + + Returns: + None. + """ if pf.empty: incoming = pd.DataFrame(columns=self.data.columns if len(self.data.columns) else COLUMNS) @@ -51,11 +64,12 @@ def update(self, pf): for column in incoming.columns: if column != "sha1": incoming[column] = incoming[column].astype(str) - + # Merge the incoming rows with the existing state. merged = pd.concat([self.data, incoming], ignore_index=True, sort=False) key_columns = [column for column in merged.columns if column != "sha1"] if key_columns: + # Remove duplicate entries while keeping the most recent row. deduped = merged.drop_duplicates(subset=key_columns, keep="last") else: deduped = merged @@ -63,6 +77,18 @@ def update(self, pf): self.data = deduped.loc[:, ["sha1", *key_columns]] def replace(self, pf): + """ + Replace the contents of the state database. + + Existing rows are discarded and replaced with the rows from the + provided ``ParaFrame``. + + Args: + pf (ParaFrame): ``ParaFrame`` containing the replacement rows. + + Returns: + None. + """ if pf.empty: self.data = pd.DataFrame(columns=self.data.columns if len(self.data.columns) else COLUMNS) diff --git a/test/conftest.py b/test/conftest.py index f8fdb84..9e78767 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -25,7 +25,7 @@ def _write_text_files(root: Path, files: list[str]) -> None: def _encoded_data_spec() -> list[dict]: return [ { - "fmt": "a{aspin}_i{i}.h5", + "fmt": "encoded/a{aspin}_i{i}.h5", "encoding": { "aspin": r"m([0-9]+(\.[0-9]+)?|\.[0-9]+)" }, @@ -45,14 +45,17 @@ def hallmark_test_suite_dictionary(tmp_path_factory): # Actually write out listed files in the temporary directory _write_text_files(repo_path, Standard_files) - _write_text_files(repo_path, Encoded_files) + # write encoded files to separate directory to avoid globbing issues + encoded_path = repo_path / "encoded" + encoded_path.mkdir() + _write_text_files(encoded_path, Encoded_files) encoded_specs = _encoded_data_spec() # Create paraframes, glob files, glob pattern and repo behavior objects standard_pf = ParaFrame.parse("a{a}_i{i}.h5", base_path=repo.worktree) encoded_pf = ParaFrame.parse( - "a{aspin}_i{i}.h5", + "encoded/a{aspin}_i{i}.h5", base_path=repo.worktree, encodings=encoded_specs, encoding=True, @@ -65,7 +68,7 @@ def hallmark_test_suite_dictionary(tmp_path_factory): ) encoded_globbed_files, encoded_glob_pattern = ParaFrame.glob_search( - "a{aspin}_i{i}.h5", + "encoded/a{aspin}_i{i}.h5", base_path=repo.worktree, encodings=encoded_specs, encoding=True, diff --git a/test/fmt_detection.py b/test/fmt_detection.py new file mode 100644 index 0000000..e69de29 diff --git a/test/test_eht_datatree.py.old b/test/test_eht_datatree.py.old new file mode 100755 index 0000000..a886990 --- /dev/null +++ b/test/test_eht_datatree.py.old @@ -0,0 +1,540 @@ +from pathlib import Path +import pytest +from hallmark import ParaFrame +from hallmark.eht_datatree import scan_inventory, build_tree, detect_fmt + +### TODO: Test suite needs reconfigured for all the recent funciton changes ### +### will not pass current tests as functionality has been changed/removed ### + +# sample inventory content for testing based off real EHT inventory +INVENTORY_CONTENT = """\ +README.md +INVENTORY.txt +LICENSE.txt +run.sh* +uvfits/ +uvfits/convert_stokesI.py* +uvfits/SR1_M87_2017_095_hi_hops_netcal_StokesI.uvfits +uvfits/SR1_M87_2017_095_lo_hops_netcal_StokesI.uvfits +txt/ +txt/dump_txt.py* +txt/SR1_M87_2017_095_hi_hops_netcal_StokesI.txt +txt/SR1_M87_2017_095_lo_hops_netcal_StokesI.txt +csv/ +csv/dump_csv.py* +csv/SR1_M87_2017_095_hi_hops_netcal_StokesI.csv +csv/SR1_M87_2017_095_lo_hops_netcal_StokesI.csv +EHTC_FirstM87Results_Apr2019_uvfits.tgz +EHTC_FirstM87Results_Apr2019_txt.tgz +EHTC_FirstM87Results_Apr2019_csv.tgz +""" + +# list of expected files from the raw inventory content +EXPECTED_FILES = [ + line.strip().rstrip("*") + for line in INVENTORY_CONTENT.splitlines() + if line.strip() and not line.strip().endswith("/")] + +# sample format string based on a real EHT dataset +sample_fmt = "SR1_M87_{year}_{day}_{band}_hops_netcal_StokesI.{ext}" + +# helper function to create test inventory file +def _write_inventory(root: Path, content: str) -> None: + (root / "INVENTORY.txt").write_text(content, encoding="utf-8") + +# write inventory file fixture into temporary directory +@pytest.fixture +def inventory_dir(tmp_path): + _write_inventory(tmp_path, INVENTORY_CONTENT) + return tmp_path + +# create fixture for scan_inventory result to use in multiple tests +@pytest.fixture +def inventory_result(inventory_dir): + return scan_inventory(inventory_dir) + +# create test dataset fixture +@pytest.fixture +def eht_dataset(tmp_path): + + # create subdirectories + (tmp_path / "csv").mkdir() + (tmp_path / "txt").mkdir() + (tmp_path / "uvfits").mkdir() + + # write meta files + for file_name in ["README.md", "LICENSE.txt", "run.sh"]: + (tmp_path / file_name).write_text(file_name, encoding="utf-8") + + # write INVENTORY.txt mirroring the real dataset structure + _write_inventory(tmp_path, INVENTORY_CONTENT) + + # write script files + for file_name in ["csv/dump_csv.py", + "txt/dump_txt.py", + "uvfits/convert_stokesI.py"]: + (tmp_path / file_name).write_text(file_name, encoding="utf-8") + + # write drive files + for file_name in [ + "EHTC_FirstM87Results_Apr2019_csv.tgz", + "EHTC_FirstM87Results_Apr2019_txt.tgz", + "EHTC_FirstM87Results_Apr2019_uvfits.tgz", + ]: + (tmp_path / file_name).write_text(file_name, encoding="utf-8") + + # write data files — 2 stems, 3 formats each + for stem in [ + "SR1_M87_2017_095_hi_hops_netcal_StokesI", + "SR1_M87_2017_095_lo_hops_netcal_StokesI", + ]: + for ext in ["csv", "txt", "uvfits"]: + (tmp_path / ext / f"{stem}.{ext}").write_text(stem, encoding="utf-8") + + return tmp_path + +# create fixture for build_tree result to use in multiple tests +@pytest.fixture +def sample_tree(eht_dataset): + return build_tree(eht_dataset, sample_fmt) + +### read_inventory tests ### + +def test_read_inventory_parsing_properties(inventory_result): + assert isinstance(inventory_result, list), \ + f"Expected list for inventory result, got {type(inventory_result)}" + dirs = [entry for entry in inventory_result if entry.endswith("/")] + assert not dirs, f"Directories not skipped: {dirs}" + execs = [entry for entry in inventory_result if entry.endswith("*")] + assert not execs, f"Exec markers not stripped: {execs}" + +# test that all files are parsed from the inventory +@pytest.mark.parametrize("expected_file", EXPECTED_FILES) +def test_read_inventory_contents(inventory_result, expected_file): + assert expected_file in inventory_result, f"{expected_file} not found in inventory" + +# needs its own fixture to test blank lines handling +def test_read_inventory_skips_blank_lines(tmp_path): + _write_inventory(tmp_path, "\nREADME.md\n\nLICENSE.txt\n") + result = read_inventory(tmp_path) + assert result == ["README.md", "LICENSE.txt"], \ + f"Expected only README.md and LICENSE.txt, got {result}" + +# needs its own fixture to test for directories without extensions +def test_read_inventory_skips_no_extension(tmp_path): + _write_inventory(tmp_path, "run\nrun.sh\n") + result = read_inventory(tmp_path) + assert "run" not in result, "run not skipped" + assert "run.sh" in result, "run.sh not found" + +# needs its own fixture to test for missing inventory file +def test_read_inventory_raises_if_missing(tmp_path): + with pytest.raises(FileNotFoundError): + read_inventory(tmp_path) + + +#### validate tests #### + +def test_validate_returns_true_when_all_present(inventory_dir, inventory_result): + # convert inventory_result to a set to work with validate's expected set input + tracked = set(inventory_result) + assert validate(inventory_dir, tracked) is True,\ + "validate did not return True when all inventory files were tracked" + + +def test_validate_returns_false_when_files_missing(inventory_dir): + assert validate(inventory_dir, set()) is False, \ + "validate did not return False when files were missing" + + +def test_validate_reports_success(inventory_dir, inventory_result, capsys): + tracked = set(inventory_result) + validate(inventory_dir, tracked) + output = capsys.readouterr().out + assert "✓ all inventory files are present" in output,\ + "success message not in output" + + +def test_validate_reports_only_missing_files(inventory_dir, capsys): + tracked = {"README.md", "INVENTORY.txt", "LICENSE.txt"} + validate(inventory_dir, tracked) + # capture printed output of missing files + output = capsys.readouterr().out + assert "run.sh" in output, "run.sh not in missing files" + assert "README.md" not in output, "unexpected README.md file in missing files" + + +def test_validate_fuzzy_matching(tmp_path): + _write_inventory(tmp_path, "LICENSE.txt\ndata.uvfits\n") + # check alternative spelling support (LICENSE vs LICENCE) + assert validate(tmp_path, {"LICENCE.txt", "data.uvfits"}) is True + # check suffix stripping support (data.uvfits matches data.uvfits.extra) + assert validate(tmp_path, {"LICENSE.txt", "data.uvfits.extra"}) is True + +### build_tree structure tests ### + +def test_build_tree_raises_if_root_not_found(tmp_path): + with pytest.raises(FileNotFoundError): + build_tree(tmp_path / "nonexistent", sample_fmt) + +# verify build_tree resolves string inputs for the root path +def test_build_tree_accepts_string_path(eht_dataset): + tree = build_tree(str(eht_dataset), sample_fmt) + assert isinstance(tree, dict) + assert set(tree.keys()) == {"meta", "drives", "data"} + +# test that each branch and stem are the write type +def test_build_tree_structure(sample_tree): + assert isinstance(sample_tree, dict), \ + f"Expected dict for tree, got {type(sample_tree)}" + assert set(sample_tree.keys()) == {"meta", "drives", "data"}, \ + f"expected meta, drives, and data keys, got {sample_tree.keys()}" + assert isinstance(sample_tree["meta"], ParaFrame) + assert isinstance(sample_tree["drives"], ParaFrame) + assert isinstance(sample_tree["data"], dict), \ + f"data branch is {type(sample_tree['data'])}, not a dict" + +# needs unique dataset to test for ext column presence when not in fmt string +def test_build_tree_ext_column_always_present(tmp_path): + (tmp_path / "csv").mkdir() + (tmp_path / "csv" / "SR1_M87_2017_095_hi.csv").write_text( + "data", encoding="utf-8") + _write_inventory(tmp_path, "csv/SR1_M87_2017_095_hi.csv\n") + fmt = "SR1_M87_{year}_{day}_{band}.csv" + tree = build_tree(tmp_path, fmt) + for stem, fmt_dict in tree["data"].items(): + assert "ext" in fmt_dict["all"].columns, f"ext column not present in {stem}" + +### build_tree meta branch tests ### + +# needs its own fixture to test hallmark's .hm is ignored +def test_build_tree_ignores_dot_hm_directory(tmp_path): + # create a file inside .hm and verify it's not picked up by meta + hm_dir = tmp_path / ".hm" + hm_dir.mkdir() + (hm_dir / "config.yml").write_text("config", encoding="utf-8") + (tmp_path / "visible.txt").write_text("visible", encoding="utf-8") + _write_inventory(tmp_path, "visible.txt\n") + tree = build_tree(tmp_path, "data/{name}.txt") + meta_paths = list(tree["meta"]["path"]) + assert "visible.txt" in meta_paths, "visible.txt not in meta" + assert not any(".hm" in p for p in meta_paths), ".hm directory leaked into meta" + + +# verify that meta and drives branches contain only the file path and are sorted +def test_build_tree_simple_branch_schema(sample_tree): + for branch in ["meta", "drives"]: + columns = sorted(list(sample_tree[branch].columns)) + assert columns == ["ext", "path"], f"{branch} has unexpected columns: {columns}" + branch_list = list(sample_tree[branch]["path"]) + assert branch_list == sorted(sample_tree[branch]["path"]), \ + f"{branch} branch is not sorted alphabetically" + +### build_tree drive branch tests ### + +def test_build_tree_drives_only_contains_archive_files(sample_tree): + # verify every file in drives has a recognized archive extension + archive_extensions = {".tgz", ".tar", ".gz", ".zip", ".bz2", ".xz", ".zst", ".7z", + ".rar"} + for path in sample_tree["drives"]["path"]: + ext = Path(path).suffix + assert ext in archive_extensions, \ + f"file '{path}' is not a recognized archive file (extension: {ext})" + + +# needs its own dataset to test for multiple drive extensions handling +def test_build_tree_drives_finds_multiple_extensions(tmp_path): + # verify drives branch finds files with different archive extensions + (tmp_path / "data.tgz").write_text("tgz", encoding="utf-8") + (tmp_path / "data.zip").write_text("zip", encoding="utf-8") + _write_inventory(tmp_path, "data.tgz\ndata.zip\n") + tree = build_tree(tmp_path, "{name}") + assert len(tree["drives"]) == 2, \ + f"length should be 2 but len={len(tree['drives'])}" + # create list of all unique extensions in drives branch + extensions = {Path(path).suffix for path in tree["drives"]["path"]} + assert extensions == {".tgz", ".zip"}, \ + f"drives has unexpected extensions: {extensions}, should be ['.tgz', '.zip']" + + +def test_build_tree_drives_empty_when_no_archives(tmp_path): + (tmp_path / "README.md").write_text("readme", encoding="utf-8") + _write_inventory(tmp_path, "README.md\n") + tree = build_tree(tmp_path, "{name}") + assert len(tree["drives"]) == 0, \ + f"length should be zero but len={len(tree['drives'])}" + + +def test_build_tree_drives_found_in_subdirectories(tmp_path): + subdir = tmp_path / "archives" + subdir.mkdir() + (subdir / "data.tgz").write_text("tgz", encoding="utf-8") + _write_inventory(tmp_path, "archives/data.tgz\n") + tree = build_tree(tmp_path, "{name}") + assert any("data.tgz" in path for path in tree["drives"]["path"]), \ + "drives in subdirectory not found" + +#### build_tree data branch tests ### + +# verify that every file is assigned to only one branch and are accounted for +def test_build_tree_partitioning_and_completeness(sample_tree): + meta_paths = set(sample_tree["meta"]["path"]) + drive_paths = set(sample_tree["drives"]["path"]) + data_paths = { + path + for fmt_dict in sample_tree["data"].values() + for path in fmt_dict["all"]["path"]} + # checks that the three branches have no common files + assert meta_paths.isdisjoint(drive_paths), \ + f"meta and drives have common files: {sorted(meta_paths & drive_paths)}" + assert meta_paths.isdisjoint(data_paths), \ + f"meta and data have common files: {sorted(meta_paths & data_paths)}" + assert drive_paths.isdisjoint(data_paths), \ + f"drives and data have common files: {sorted(drive_paths & data_paths)}" + # the union of all branches must match the inventory exactly + all_tracked = meta_paths | drive_paths | data_paths + expected = set(EXPECTED_FILES) + assert all_tracked == expected, \ + f"Missing: {expected - all_tracked}. Unexpected: {all_tracked - expected}" + + +def test_build_tree_internal_validation_reports_success(eht_dataset, capsys): + build_tree(eht_dataset, sample_fmt) + assert "✓ all inventory files are present" in capsys.readouterr().out, \ + "internal validation did not report success" + + +def test_build_tree_data_empty_when_no_fmt_matches(eht_dataset): + # use a format that doesn't match any files to verify data branch is empty + tree = build_tree(eht_dataset, "{ext}/NONEXISTENT_{year}_{day}.{ext}") + assert len(tree["data"]) == 0, \ + f"length should be zero but len={len(tree['data'])}" + +def test_build_tree_data_matches_mixed_delimiters(tmp_path): + (tmp_path / "ER6_SGRA_2017_096_hi_hops_netcal-LMTcal_StokesI.csv").write_text( + "data", encoding="utf-8") + _write_inventory( + tmp_path, + "ER6_SGRA_2017_096_hi_hops_netcal-LMTcal_StokesI.csv\n") + fmt = "ER6_SGRA_2017_{day}_{band}_{pipeline}_netcal_LMTcal_StokesI" + tree = build_tree(tmp_path, fmt) + assert len(tree["data"]) == 1, \ + f"expected 1 fmt despite delimiter mismatch, got {len(tree['data'])}" + + +def test_build_tree_auto_detected_fmt_matches_mixed_delimiters(tmp_path): + (tmp_path / "ER6_SGRA_2017_096_hi_hops_netcal-LMTcal_StokesI.csv").write_text( + "data", encoding="utf-8") + (tmp_path / "ER6_SGRA_2017_097_lo_casa_netcal-LMTcal_StokesI.csv").write_text( + "data", encoding="utf-8") + _write_inventory( + tmp_path, + "ER6_SGRA_2017_096_hi_hops_netcal-LMTcal_StokesI.csv\n" + "ER6_SGRA_2017_097_lo_casa_netcal-LMTcal_StokesI.csv\n") + tree = build_tree(tmp_path) # fmt is None, triggers auto-detection + assert len(tree["data"]) == 1, \ + f"expected 1 fmt, got {len(tree['data'])}" + fmt_dict = list(tree["data"].values())[0] + assert len(fmt_dict["all"]) == 2, \ + f"expected 2 files, got {len(fmt_dict['all'])}" + +# needs unique dataset to test for single stem handling +def test_build_tree_single_stem(tmp_path): + (tmp_path / "csv").mkdir() + (tmp_path / "csv" / "SR1_M87_2017_095_hi_hops_netcal_StokesI.csv").write_text( + "data", encoding="utf-8") + _write_inventory( + tmp_path, + "csv/SR1_M87_2017_095_hi_hops_netcal_StokesI.csv\n") + tree = build_tree(tmp_path, sample_fmt) + assert len(tree["data"]) == 1, \ + f"length should be 1 but len={len(tree['data'])}" + +# test that each data stem contains the expected number of files and schema +def test_build_tree_data_content(sample_tree): + data = sample_tree["data"] + assert len(data) == 1, f"expected 1 fmt, got {len(data)}" + fmt_dict = list(data.values())[0] + stems = [k for k in fmt_dict if k != "all"] + assert len(stems) == 2, f"expected 2 stems, got {len(stems)}" + for stem in stems: + assert len(fmt_dict[stem]) == 3, \ + f"{stem} has {len(fmt_dict[stem])} files, expected 3" + assert len(fmt_dict["all"]) == 6, \ + f"all has {len(fmt_dict['all'])} files, expected 6" + + +### fmt variations tests ### + +# needs unique dataset to test for nested subdirectory handling +def test_build_tree_nested_subdir_fmt(tmp_path): + (tmp_path / "casa_data" / "April05").mkdir(parents=True) + (tmp_path / "hops_data" / "April05").mkdir(parents=True) + (tmp_path / "casa_data" / "April05" / "SR2_M87_2017_095_hi_casa.uvfits").write_text( + "data", encoding="utf-8") + (tmp_path / "hops_data" / "April05" / "SR2_M87_2017_095_hi_hops.uvfits").write_text( + "data", encoding="utf-8") + _write_inventory( + tmp_path, + "casa_data/April05/SR2_M87_2017_095_hi_casa.uvfits\n" + "hops_data/April05/SR2_M87_2017_095_hi_hops.uvfits\n") + fmt = "SR2_M87_{year}_{day}_{band}_{pipeline}.uvfits" + tree = build_tree(tmp_path, fmt) + assert len(tree["data"]) == 1 + fmt_dict = list(tree["data"].values())[0] + stems = [k for k in fmt_dict if k != "all"] + assert len(stems) == 2, \ + f"expected 2 stems but got {len(stems)}, couldn't parse nested directories" + +### detect_fmt tests ### + +def test_detect_fmt_finds_single_pattern(tmp_path): + _write_inventory( + tmp_path, + "csv/SR1_M87_2017_095_hi_hops_netcal_StokesI.csv\n" + "csv/SR1_M87_2017_096_lo_hops_netcal_StokesI.csv\n") + fmts = detect_fmt(tmp_path) + assert fmts == ["SR1_M87_2017_{p0}_{p1}_hops_netcal_StokesI"], f"unexpected fmts: \ + {fmts}, expected 'SR1_M87_2017_{{p0}}_{{p1}}_hops_netcal_StokesI'" + + +def test_detect_fmt_excludes_script_extensions(tmp_path): + _write_inventory( + tmp_path, + "csv/SR1_M87_2017_095_hi.csv\n" + "csv/SR1_M87_2017_096_lo.csv\n" + "csv/dump_csv.py\n" + "csv/convert.py\n") + fmts = detect_fmt(tmp_path) + assert all("dump" not in fmt for fmt in fmts), \ + f"dump files leaked into fmts: {fmts}" + assert all("convert" not in fmt for fmt in fmts), \ + f"convert files leaked into fmts: {fmts}" + + +def test_detect_fmt_excludes_known_meta_files(tmp_path): + _write_inventory( + tmp_path, + "README.txt\n" + "LICENSE.txt\n" + "data/SR1_M87_2017_095_hi.txt\n" + "data/SR1_M87_2017_096_lo.txt\n") + fmts = detect_fmt(tmp_path) + assert all("README" not in fmt for fmt in fmts), \ + f"README file leaked into fmts: {fmts}" + assert all("LICENSE" not in fmt for fmt in fmts), \ + f"LICENSE file leaked into fmts: {fmts}" + + +def test_detect_fmt_separates_by_first_token(tmp_path): + _write_inventory( + tmp_path, + "csv/SR1_M87_2017_095.csv\n" + "csv/SR1_M87_2017_096.csv\n" + "csv/ER6_SGRA_2017_097.csv\n" + "csv/ER6_SGRA_2017_098.csv\n") + fmts = detect_fmt(tmp_path) + assert len(fmts) == 2, f"expected 2 fmts, got {len(fmts)}: {fmts}" + assert any("SR1" in fmt for fmt in fmts), f"SR1 fmt not found in fmts: {fmts}" + assert any("ER6" in fmt for fmt in fmts), f"ER6 fmt not found in fmts: {fmts}" + + +def test_detect_fmt_separates_by_token_count(tmp_path): + # same first token but different number of tokens (optional field case) + _write_inventory( + tmp_path, + "csv/ER6_SGRA_2017_095_hi_netcal_StokesI.csv\n" + "csv/ER6_SGRA_2017_096_lo_netcal_StokesI.csv\n" + "csv/ER6_SGRA_2017_097_hi_netcal_norm_StokesI.csv\n" + "csv/ER6_SGRA_2017_098_lo_netcal_besttime_StokesI.csv\n" + ) + fmts = detect_fmt(tmp_path) + assert len(fmts) == 2, f"expected 2 fmts, got {len(fmts)}: {fmts}" + + +def test_detect_fmt_returns_empty_when_no_data_files(tmp_path): + _write_inventory( tmp_path, "README.md\nLICENSE.txt\nrun.sh\n") + fmts = detect_fmt(tmp_path) + assert fmts == [], f"expected empty list, got {fmts}" + + +def test_detect_fmt_used_automatically_when_fmt_is_none(tmp_path): + (tmp_path / "SR1_M87_2017_095_hi.csv").write_text( + "data", encoding="utf-8") + (tmp_path / "SR1_M87_2017_096_lo.csv").write_text( + "data", encoding="utf-8") + _write_inventory( + tmp_path, + "SR1_M87_2017_095_hi.csv\n" + "SR1_M87_2017_096_lo.csv\n") + tree = build_tree(tmp_path) + assert len(tree["data"]) == 1, \ + f"expected 1 fmt, got {len(tree['data'])}" + +### data branch "all" paraframe tests ### + +def test_build_tree_data_all_merges_every_stem(tmp_path): + for stem_name in ["SR1_M87_2017_095_hi", "SR1_M87_2017_096_lo"]: + (tmp_path / f"{stem_name}.csv").write_text("data", encoding="utf-8") + _write_inventory( + tmp_path, + "SR1_M87_2017_095_hi.csv\n" + "SR1_M87_2017_096_lo.csv\n" + ) + fmt = "SR1_M87_{year}_{day}_{band}.csv" + tree = build_tree(tmp_path, fmt) + fmt_dict = list(tree["data"].values())[0] + stems = [k for k in fmt_dict if k != "all"] + total_stem_rows = sum(len(fmt_dict[stem]) for stem in stems) + assert len(fmt_dict["all"]) == total_stem_rows, \ + f"'all' has {len(fmt_dict['all'])} rows, expected {total_stem_rows}" + + +def test_build_tree_data_multiple_fmts_produce_separate_branches(tmp_path): + (tmp_path / "SR1_M87_2017_095_hi.csv").write_text("data", encoding="utf-8") + (tmp_path / "ER6_SGRA_2017_096_lo.csv").write_text("data", encoding="utf-8") + _write_inventory( + tmp_path, + "SR1_M87_2017_095_hi.csv\n" + "ER6_SGRA_2017_096_lo.csv\n" + ) + fmts = [ + "SR1_M87_{year}_{day}_{band}", + "ER6_SGRA_{year}_{day}_{band}", + ] + tree = build_tree(tmp_path, fmts) + assert len(tree["data"]) == 2, \ + f"expected 2 fmt branches, got {len(tree['data'])}" + for fmt_dict in tree["data"].values(): + assert "all" in fmt_dict, "'all' paraframe missing from fmt dict" + + +def test_detect_fmt_separates_when_first_token_varies(tmp_path): + _write_inventory( + tmp_path, + "AA_B_1_0AXG5H.dat\n" + "AA_B_2_0AXG5H.dat\n" + "AP_B_3_0AXG5H.dat\n" + "AP_B_4_0AXG5H.dat\n" + ) + fmts = detect_fmt(tmp_path) + assert len(fmts) == 2, f"expected 2 fmts, got {len(fmts)}: {fmts}" + assert any(fmt.startswith("AA_") for fmt in fmts), \ + f"AA_ fmt not found in fmts: {fmts}" + assert any(fmt.startswith("AP_") for fmt in fmts), \ + f"AP_ fmt not found in fmts: {fmts}" + + +def test_build_tree_data_ext_column_has_no_period(tmp_path): + (tmp_path / "SR1_M87_2017_095_hi.csv").write_text("data", encoding="utf-8") + (tmp_path / "SR1_M87_2017_096_lo.csv").write_text("data", encoding="utf-8") + _write_inventory( + tmp_path, + "SR1_M87_2017_095_hi.csv\n" + "SR1_M87_2017_096_lo.csv\n" + ) + fmt = "SR1_M87_{year}_{day}_{band}.csv" + tree = build_tree(tmp_path, fmt) + fmt_dict = list(tree["data"].values())[0] + exts = set(fmt_dict["all"]["ext"]) + assert exts == {"csv"}, f"expected ext without period, got {exts}" \ No newline at end of file diff --git a/test/test_hallmark.py b/test/test_hallmark.py index a9bd798..6c2f287 100644 --- a/test/test_hallmark.py +++ b/test/test_hallmark.py @@ -26,7 +26,8 @@ def test_standard_pf_column_names(hallmark_test_suite_dictionary): def test_standard_pf_column_value_types(hallmark_test_suite_dictionary): pf = hallmark_test_suite_dictionary["standard_pf"] assert pd.api.types.is_float_dtype(pf["a"]) - assert pd.api.types.is_float_dtype(pf["i"]) + # i values are whole numbers so they convert to int64, not float64 + assert pd.api.types.is_numeric_dtype(pf["i"]) def test_standard_pf_values(hallmark_test_suite_dictionary): pf = hallmark_test_suite_dictionary["standard_pf"] @@ -48,7 +49,7 @@ def test_standard_glob_pattern_created_properly(hallmark_test_suite_dictionary): def test_standard_glob_returns_expected_files(hallmark_test_suite_dictionary): files = hallmark_test_suite_dictionary["standard_globbed_files"] - assert len(files) == 16 + assert len(files) == 12 def test_standard_pf_single_filter_argument(hallmark_test_suite_dictionary): pf = hallmark_test_suite_dictionary["standard_pf"] @@ -72,7 +73,7 @@ def test_standard_pf_filter_multiple_conditions(hallmark_test_suite_dictionary): ### Test encoded pf def test_encoded_pf_shape(hallmark_test_suite_dictionary): pf = hallmark_test_suite_dictionary["encoded_pf"] - assert pf.shape == (16, 3) + assert pf.shape == (4, 3) def test_encoded_pf_column_names(hallmark_test_suite_dictionary): pf = hallmark_test_suite_dictionary["encoded_pf"] @@ -84,7 +85,7 @@ def test_encoded_pf_has_custom_spin_type(hallmark_test_suite_dictionary): def test_encoded_pf_spin_values(hallmark_test_suite_dictionary): pf = hallmark_test_suite_dictionary["encoded_pf"] - assert set(pf["aspin"].unique()) == {-0.5, 0, 0.75, 0.975} + assert set(pf["aspin"].unique()) == {-0.5} def test_encoded_pf_i_values(hallmark_test_suite_dictionary): pf = hallmark_test_suite_dictionary["encoded_pf"] @@ -99,8 +100,8 @@ def test_encoded_pf_filter_single_value(hallmark_test_suite_dictionary): def test_encoded_pf_filter_multiple_values(hallmark_test_suite_dictionary): pf = hallmark_test_suite_dictionary["encoded_pf"] filtered = pf(aspin=[-0.5, 0.0]) - assert len(filtered) == 8 - assert set(filtered["aspin"].unique()) == {-0.5, 0.0} + assert len(filtered) == 4 + assert set(filtered["aspin"].unique()) == {-0.5} def test_encoded_pf_filter_multiple_conditions(hallmark_test_suite_dictionary): pf = hallmark_test_suite_dictionary["encoded_pf"] @@ -115,7 +116,21 @@ def test_encoded_glob_pattern_created_properly(hallmark_test_suite_dictionary): def test_encoded_glob_returns_expected_files(hallmark_test_suite_dictionary): files = hallmark_test_suite_dictionary["encoded_globbed_files"] - assert len(files) == 16 + assert len(files) == 4 + +# new tests for the two different subdirectories of encoded vs standard files +def test_encoded_pf_paths_are_in_encoded_subdirectory(hallmark_test_suite_dictionary): + pf = hallmark_test_suite_dictionary["encoded_pf"] + assert all(path.startswith("encoded/") for path in pf["path"]) + +def test_standard_pf_not_in_encoded_subdir(hallmark_test_suite_dictionary): + pf = hallmark_test_suite_dictionary["standard_pf"] + assert not any(path.startswith("encoded/") for path in pf["path"]) + +def test_standard_and_encoded_no_overlap(hallmark_test_suite_dictionary): + standard_paths = set(hallmark_test_suite_dictionary["standard_pf"]["path"]) + encoded_paths = set(hallmark_test_suite_dictionary["encoded_pf"]["path"]) + assert standard_paths.isdisjoint(encoded_paths) ### Test repo behavior def test_repo_init_created_dot_hm(hallmark_test_suite_dictionary):