From c3c286932dca787cc953cebf1a950239dd6a4ccc Mon Sep 17 00:00:00 2001 From: Sam Jacobs Date: Tue, 16 Jun 2026 15:18:37 -0700 Subject: [PATCH 01/10] eht datatree starting implementation --- mod/hallmark/eht_datatree.py | 192 +++++++++++++++++ mod/hallmark/helper_functions.py | 32 +++ mod/hallmark/paraframe.py | 13 +- test/conftest.py | 11 +- test/test_eht_datatree.py | 353 +++++++++++++++++++++++++++++++ test/test_hallmark.py | 29 ++- 6 files changed, 615 insertions(+), 15 deletions(-) create mode 100755 mod/hallmark/eht_datatree.py create mode 100755 test/test_eht_datatree.py diff --git a/mod/hallmark/eht_datatree.py b/mod/hallmark/eht_datatree.py new file mode 100755 index 0000000..853228a --- /dev/null +++ b/mod/hallmark/eht_datatree.py @@ -0,0 +1,192 @@ +from __future__ import annotations +from pathlib import Path +from hallmark import ParaFrame +import parse + +def _strip_suffixes(path: str) -> str: + """Strip all suffixes from a path string to help with parsing inventory files.""" + p = Path(path) + while p.suffix: + p = p.with_suffix("") + return str(p) + +def _alternative_spellings(path: str) -> list[str]: + """Return alternative spellings for known filename variants.""" + alternatives = [path] + if "LICENSE" in path: + alternatives.append(path.replace("LICENSE", "LICENCE")) + if "LICENCE" in path: + alternatives.append(path.replace("LICENCE", "LICENSE")) + return alternatives + +def read_inventory(root: Path) -> list[str]: + """ + Read INVENTORY.txt and return expected relative file paths. + + Args: + root: Path to the dataset root directory containing INVENTORY.txt. + + Returns: + List of relative file path strings expected to exist under root. + + Raises: + FileNotFoundError: If INVENTORY.txt does not exist under root. + """ + # finds path to the inventory file, raises error if not found + inventory_path = Path(root) / "INVENTORY.txt" + if not inventory_path.exists(): + raise FileNotFoundError(f"INVENTORY.txt not found in {root}") + + files_list = [] + # skip directory lines ending in "/" and strip executable marker "*" + for line in inventory_path.read_text(encoding="utf-8").splitlines(): + # remove whitespace + line = line.strip() + # skip empty lines and directories + if not line or line.endswith("/"): + continue + # remove executable marker if present + line = line.rstrip("*") + # check there is a file extension + if not Path(line).suffix: + continue + # add cleaned line to files list + files_list.append(line) + return files_list + +def validate(root: Path, tracked: set[str]) -> bool: + """ + Cross-check INVENTORY.txt against a set of tracked files in tree. + + Args: + root: Path to the dataset root containing INVENTORY.txt. + tracked: Set of relative file path strings already in the tree. + + Returns: + True if all inventory files are accounted for, False otherwise. + """ + files_list = read_inventory(root) + # add files to missing if not in tracked but in inventory + # checks the end rather than exact match to handle nesting and strip suffixes + missing = [ + file for file in files_list + if not any( + t.endswith(alt) or _strip_suffixes(t).endswith(_strip_suffixes(alt)) + for t in tracked + # catch issues like licence/license + for alt in _alternative_spellings(file))] + # if the missing list is not empty, print missing files message + if missing: + # print how many files are missing and list them + print(f" missing : {len(missing)} file(s)") + for file in missing: + print(f" ✗ {file}") + return False + else: + print(" ✓ all inventory files are present in the tree") + return True + +# common drive extensions to look for when building the tree +DRIVE_EXTENSIONS = [".tgz", ".tar", ".gz", ".zip", ".bz2", ".xz", ".zst", ".7z", ".rar"] + +def build_tree(root: Path, fmt: str | list[str]) -> dict: + """ + Build an in-memory pytree for an EHT dataset directory. + + Args: + root: Path to the EHT dataset root directory. + fmt: Format string for parsing data files. + + Returns: + A dictionary with keys: + - "meta" : ParaFrame of housekeeping files + - "drives" : ParaFrame of compressed archives + - "data" : dict of {stem -> ParaFrame} + """ + # create clean root path + root = Path(root).expanduser().resolve() + # track files that are included in the tree, to cross-check against inventory + tracked = set() + + ### DRIVES ### + # collect all drive paths matching any supported extension + drive_paths = [] + for ext in DRIVE_EXTENSIONS: + # add any file that has this ext to the drive paths list and tracked set + for path in root.rglob(f"*{ext}"): + drive_paths.append(str(path.relative_to(root))) + tracked.add(str(path.relative_to(root))) + # build a single ParaFrame from all drive paths with extensions column + drives_pf = ParaFrame( + [{"path": path, "ext": Path(path).suffix.lstrip(".")} + for path in sorted(drive_paths)], + base_path=root,) + + ### DATA ### + # check if there is only one fmt + fmts = [fmt] if isinstance(fmt, str) else fmt + # make a parser for each fmt + parsers = [parse.compile(f) for f in fmts] + stems = {} + # search all subdirectories from root + for file in root.rglob("*"): + # if the file isn't a folder + if not file.is_file(): + continue + # reset parse for each file + parsed = None + # parse the path to extract fields, skip if it doesn't match the format + for parser in parsers: + parsed = (parser.parse(file.name) or + parser.parse(file.stem) or + parser.parse(file.stem.split(".")[0])) + # break out once matching parser found + if parsed: + break + if parsed: + # get path to file from data root + relative_path = str(file.relative_to(root)) + # create unique stem name based on fmt parameters excluding extension + stem_key = "_".join(str(value) for key, value in parsed.named.items() + if key != "ext") + # create stem if it doesn't already exist and add a row for this file + stems.setdefault(stem_key, []).append( + {"path": relative_path, + # normalize ext to not have period + "ext": Path(relative_path).suffix.lstrip("."), + **{key: value for key, value in parsed.named.items() if key != "ext"},} + ) + tracked.add(relative_path) + + data_branches = {} + # create a ParaFrame for each stem and add to the data branches dict + for stem_key, rows in stems.items(): + data_branches[stem_key] = ParaFrame(rows, base_path=root) + + ### META ### + # create a list of all files under root that aren't tracked + meta_files = { + str(file.relative_to(root)) + for file in root.rglob("*") + # add file if its not a dir, not the .hm, and not in tracked + if file.is_file() + and ".hm" not in file.parts + and str(file.relative_to(root)) not in tracked + } + # create a paraframe for the meta files with extensions column + meta_pf = ParaFrame( + [{"path": file, "ext": Path(file).suffix.lstrip(".")} + for file in sorted(meta_files)], + base_path=root,) + for _, row in meta_pf.iterrows(): + tracked.add(row["path"]) + + # check all files are in the tree + validate(root, tracked) + + # return dict with three keys, only data has subbranches + return { + "meta" : meta_pf, + "drives" : drives_pf, + "data" : data_branches, + } \ No newline at end of file diff --git a/mod/hallmark/helper_functions.py b/mod/hallmark/helper_functions.py index 3ed5618..0802c4d 100644 --- a/mod/hallmark/helper_functions.py +++ b/mod/hallmark/helper_functions.py @@ -13,6 +13,7 @@ # limitations under the License. import re +import pandas as pd def find_spec_by_fmt(fmt, encodings): @@ -59,3 +60,34 @@ def regex_sub(value, yaml_encodings): result = re.sub(match.group(0), "-" + str(match.group(1)), result) return result + +def try_numeric_conversion(series): + """ + Attempt to convert a pandas Series to numeric. + + Converts the series to numeric iff: + 1. All values are numeric + 2. Converting back to string matches the original values to avoid + unintended conversions (e.g., "001" -> 1) + + Args: + series: A pandas Series of strings to attempt conversion on. + + Returns: + The converted numeric Series if both conditions are met, + otherwise returns original series. + """ + # replace unconvertible values with NaN + converted = pd.to_numeric(series, errors="coerce") + # if any values were unconvertible, return original series + if converted.isna().any(): + return series + # if converting back to str doesn't match original, return original series + # prevents unintended conversions like "001" -> 1 + if not all(str(int(numeric_val)) == str(original_val) + or str(numeric_val) == str(original_val) + # check each pair of converted and original values + for numeric_val, original_val in zip(converted, series)): + return series + return converted + diff --git a/mod/hallmark/paraframe.py b/mod/hallmark/paraframe.py index ef50874..3505001 100644 --- a/mod/hallmark/paraframe.py +++ b/mod/hallmark/paraframe.py @@ -21,7 +21,7 @@ import pandas as pd import numpy as np -from .helper_functions import find_spec_by_fmt, regex_sub +from .helper_functions import find_spec_by_fmt, regex_sub, try_numeric_conversion class ParaFrame(pd.DataFrame): @@ -187,8 +187,8 @@ def glob_search( break except KeyError as e: k = e.args[0] - pattern = re.sub(r"\{" + k + r":?.*?\}", "{" + k + ":s}", pattern) - fmt_g = re.sub(r"\{" + k + r":?.*?\}", "{" + k + ":g}", fmt_g) + pattern = re.sub(r"\{" + k + r":?.*?\}", "{" + k + "}", pattern) + fmt_g = re.sub(r"\{" + k + r":?.*?\}", "{" + k + "}", fmt_g) kwargs[k] = "*" # Obtain list of files based on the glob pattern @@ -272,4 +272,9 @@ def parse( print(f'Failed to parse "{f}"') else: frame.append({"path": f_short, **r.named}) - return cls(frame, encodings=encodings, base_path=base_path) + # attempt to convert each parameter column to numeric + # if conversion fails the column stays as string + result = cls(frame, encodings=encodings, base_path=base_path) + for col in result.columns: + result[col] = try_numeric_conversion(result[col]) + return result diff --git a/test/conftest.py b/test/conftest.py index f8fdb84..9e78767 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -25,7 +25,7 @@ def _write_text_files(root: Path, files: list[str]) -> None: def _encoded_data_spec() -> list[dict]: return [ { - "fmt": "a{aspin}_i{i}.h5", + "fmt": "encoded/a{aspin}_i{i}.h5", "encoding": { "aspin": r"m([0-9]+(\.[0-9]+)?|\.[0-9]+)" }, @@ -45,14 +45,17 @@ def hallmark_test_suite_dictionary(tmp_path_factory): # Actually write out listed files in the temporary directory _write_text_files(repo_path, Standard_files) - _write_text_files(repo_path, Encoded_files) + # write encoded files to separate directory to avoid globbing issues + encoded_path = repo_path / "encoded" + encoded_path.mkdir() + _write_text_files(encoded_path, Encoded_files) encoded_specs = _encoded_data_spec() # Create paraframes, glob files, glob pattern and repo behavior objects standard_pf = ParaFrame.parse("a{a}_i{i}.h5", base_path=repo.worktree) encoded_pf = ParaFrame.parse( - "a{aspin}_i{i}.h5", + "encoded/a{aspin}_i{i}.h5", base_path=repo.worktree, encodings=encoded_specs, encoding=True, @@ -65,7 +68,7 @@ def hallmark_test_suite_dictionary(tmp_path_factory): ) encoded_globbed_files, encoded_glob_pattern = ParaFrame.glob_search( - "a{aspin}_i{i}.h5", + "encoded/a{aspin}_i{i}.h5", base_path=repo.worktree, encodings=encoded_specs, encoding=True, diff --git a/test/test_eht_datatree.py b/test/test_eht_datatree.py new file mode 100755 index 0000000..be2dd3f --- /dev/null +++ b/test/test_eht_datatree.py @@ -0,0 +1,353 @@ +from pathlib import Path +import pytest +from hallmark import ParaFrame +from hallmark.eht_datatree import read_inventory, validate, build_tree + +# sample inventory content for testing based off real EHT inventory +INVENTORY_CONTENT = """\ +README.md +INVENTORY.txt +LICENSE.txt +run.sh* +uvfits/ +uvfits/convert_stokesI.py* +uvfits/SR1_M87_2017_095_hi_hops_netcal_StokesI.uvfits +uvfits/SR1_M87_2017_095_lo_hops_netcal_StokesI.uvfits +txt/ +txt/dump_txt.py* +txt/SR1_M87_2017_095_hi_hops_netcal_StokesI.txt +txt/SR1_M87_2017_095_lo_hops_netcal_StokesI.txt +csv/ +csv/dump_csv.py* +csv/SR1_M87_2017_095_hi_hops_netcal_StokesI.csv +csv/SR1_M87_2017_095_lo_hops_netcal_StokesI.csv +EHTC_FirstM87Results_Apr2019_uvfits.tgz +EHTC_FirstM87Results_Apr2019_txt.tgz +EHTC_FirstM87Results_Apr2019_csv.tgz +""" + +# list of expected files from the raw inventory content +EXPECTED_FILES = [ + line.strip().rstrip("*") + for line in INVENTORY_CONTENT.splitlines() + if line.strip() and not line.strip().endswith("/")] + +# sample format string based on a real EHT dataset +sample_fmt = "SR1_M87_{year}_{day}_{band}_hops_netcal_StokesI.{ext}" + +# helper function to create test inventory file +def _write_inventory(root: Path, content: str) -> None: + (root / "INVENTORY.txt").write_text(content, encoding="utf-8") + +# write inventory file fixture into temporary directory +@pytest.fixture +def inventory_dir(tmp_path): + _write_inventory(tmp_path, INVENTORY_CONTENT) + return tmp_path + +# create fixture for read_inventory result to use in multiple tests +@pytest.fixture +def inventory_result(inventory_dir): + return read_inventory(inventory_dir) + +# create test dataset fixture +@pytest.fixture +def eht_dataset(tmp_path): + + # create subdirectories + (tmp_path / "csv").mkdir() + (tmp_path / "txt").mkdir() + (tmp_path / "uvfits").mkdir() + + # write meta files + for file_name in ["README.md", "LICENSE.txt", "run.sh"]: + (tmp_path / file_name).write_text(file_name, encoding="utf-8") + + # write INVENTORY.txt mirroring the real dataset structure + _write_inventory(tmp_path, INVENTORY_CONTENT) + + # write script files + for file_name in ["csv/dump_csv.py", + "txt/dump_txt.py", + "uvfits/convert_stokesI.py"]: + (tmp_path / file_name).write_text(file_name, encoding="utf-8") + + # write drive files + for file_name in [ + "EHTC_FirstM87Results_Apr2019_csv.tgz", + "EHTC_FirstM87Results_Apr2019_txt.tgz", + "EHTC_FirstM87Results_Apr2019_uvfits.tgz", + ]: + (tmp_path / file_name).write_text(file_name, encoding="utf-8") + + # write data files — 2 stems, 3 formats each + for stem in [ + "SR1_M87_2017_095_hi_hops_netcal_StokesI", + "SR1_M87_2017_095_lo_hops_netcal_StokesI", + ]: + for ext in ["csv", "txt", "uvfits"]: + (tmp_path / ext / f"{stem}.{ext}").write_text(stem, encoding="utf-8") + + return tmp_path + +# create fixture for build_tree result to use in multiple tests +@pytest.fixture +def sample_tree(eht_dataset): + return build_tree(eht_dataset, sample_fmt) + +### read_inventory tests ### + +def test_read_inventory_parsing_properties(inventory_result): + assert isinstance(inventory_result, list), \ + f"Expected list for inventory result, got {type(inventory_result)}" + dirs = [entry for entry in inventory_result if entry.endswith("/")] + assert not dirs, f"Directories not skipped: {dirs}" + execs = [entry for entry in inventory_result if entry.endswith("*")] + assert not execs, f"Exec markers not stripped: {execs}" + +# test that all files are parsed from the inventory +@pytest.mark.parametrize("expected_file", EXPECTED_FILES) +def test_read_inventory_contents(inventory_result, expected_file): + assert expected_file in inventory_result, f"{expected_file} not found in inventory" + +# needs its own fixture to test blank lines handling +def test_read_inventory_skips_blank_lines(tmp_path): + _write_inventory(tmp_path, "\nREADME.md\n\nLICENSE.txt\n") + result = read_inventory(tmp_path) + assert result == ["README.md", "LICENSE.txt"], \ + f"Expected only README.md and LICENSE.txt, got {result}" + +# needs its own fixture to test for directories without extensions +def test_read_inventory_skips_no_extension(tmp_path): + _write_inventory(tmp_path, "run\nrun.sh\n") + result = read_inventory(tmp_path) + assert "run" not in result, "run not skipped" + assert "run.sh" in result, "run.sh not found" + +# needs its own fixture to test for missing inventory file +def test_read_inventory_raises_if_missing(tmp_path): + with pytest.raises(FileNotFoundError): + read_inventory(tmp_path) + + +#### validate tests #### + +def test_validate_returns_true_when_all_present(inventory_dir, inventory_result): + # convert inventory_result to a set to work with validate's expected set input + tracked = set(inventory_result) + assert validate(inventory_dir, tracked) is True,\ + "validate did not return True when all inventory files were tracked" + + +def test_validate_returns_false_when_files_missing(inventory_dir): + assert validate(inventory_dir, set()) is False, \ + "validate did not return False when files were missing" + + +def test_validate_reports_success(inventory_dir, inventory_result, capsys): + tracked = set(inventory_result) + validate(inventory_dir, tracked) + output = capsys.readouterr().out + assert "✓ all inventory files are present" in output,\ + "success message not in output" + + +def test_validate_reports_only_missing_files(inventory_dir, capsys): + tracked = {"README.md", "INVENTORY.txt", "LICENSE.txt"} + validate(inventory_dir, tracked) + # capture printed output of missing files + output = capsys.readouterr().out + assert "run.sh" in output, "run.sh not in missing files" + assert "README.md" not in output, "unexpected README.md file in missing files" + + +def test_validate_fuzzy_matching(tmp_path): + _write_inventory(tmp_path, "LICENSE.txt\ndata.uvfits\n") + # check alternative spelling support (LICENSE vs LICENCE) + assert validate(tmp_path, {"LICENCE.txt", "data.uvfits"}) is True + # check suffix stripping support (data.uvfits matches data.uvfits.extra) + assert validate(tmp_path, {"LICENSE.txt", "data.uvfits.extra"}) is True + +### build_tree structure tests ### + +def test_build_tree_raises_if_root_not_found(tmp_path): + with pytest.raises(FileNotFoundError): + build_tree(tmp_path / "nonexistent", sample_fmt) + +# verify build_tree resolves string inputs for the root path +def test_build_tree_accepts_string_path(eht_dataset): + tree = build_tree(str(eht_dataset), sample_fmt) + assert isinstance(tree, dict) + assert set(tree.keys()) == {"meta", "drives", "data"} + +# test that each branch and stem are the write type +def test_build_tree_structure(sample_tree): + assert isinstance(sample_tree, dict), \ + f"Expected dict for tree, got {type(sample_tree)}" + assert set(sample_tree.keys()) == {"meta", "drives", "data"}, \ + f"expected meta, drives, and data keys, got {sample_tree.keys()}" + assert isinstance(sample_tree["meta"], ParaFrame) + assert isinstance(sample_tree["drives"], ParaFrame) + assert isinstance(sample_tree["data"], dict), \ + f"data branch is {type(sample_tree['data'])}, not a dict" + +# needs unique dataset to test for ext column presence when not in fmt string +def test_build_tree_ext_column_always_present(tmp_path): + (tmp_path / "csv").mkdir() + (tmp_path / "csv" / "SR1_M87_2017_095_hi.csv").write_text( + "data", encoding="utf-8") + _write_inventory(tmp_path, "csv/SR1_M87_2017_095_hi.csv\n") + fmt = "SR1_M87_{year}_{day}_{band}.csv" + tree = build_tree(tmp_path, fmt) + for stem, pf in tree["data"].items(): + assert "ext" in pf.columns, f"ext column not present in {stem}" + +### build_tree meta branch tests ### + +# needs its own fixture to test hallmark's .hm is ignored +def test_build_tree_ignores_dot_hm_directory(tmp_path): + # create a file inside .hm and verify it's not picked up by meta + hm_dir = tmp_path / ".hm" + hm_dir.mkdir() + (hm_dir / "config.yml").write_text("config", encoding="utf-8") + (tmp_path / "visible.txt").write_text("visible", encoding="utf-8") + _write_inventory(tmp_path, "visible.txt\n") + tree = build_tree(tmp_path, "data/{name}.txt") + meta_paths = list(tree["meta"]["path"]) + assert "visible.txt" in meta_paths, "visible.txt not in meta" + assert not any(".hm" in p for p in meta_paths), ".hm directory leaked into meta" + + +# verify that meta and drives branches contain only the file path and are sorted +def test_build_tree_simple_branch_schema(sample_tree): + for branch in ["meta", "drives"]: + columns = sorted(list(sample_tree[branch].columns)) + assert columns == ["ext", "path"], f"{branch} has unexpected columns: {columns}" + branch_list = list(sample_tree[branch]["path"]) + assert branch_list == sorted(sample_tree[branch]["path"]), \ + f"{branch} branch is not sorted alphabetically" + +### build_tree drive branch tests ### + +def test_build_tree_drives_only_contains_archive_files(sample_tree): + # verify every file in drives has a recognized archive extension + archive_extensions = {".tgz", ".tar", ".gz", ".zip", ".bz2", ".xz", ".zst", ".7z", + ".rar"} + for path in sample_tree["drives"]["path"]: + ext = Path(path).suffix + assert ext in archive_extensions, \ + f"file '{path}' is not a recognized archive file (extension: {ext})" + + +# needs its own dataset to test for multiple drive extensions handling +def test_build_tree_drives_finds_multiple_extensions(tmp_path): + # verify drives branch finds files with different archive extensions + (tmp_path / "data.tgz").write_text("tgz", encoding="utf-8") + (tmp_path / "data.zip").write_text("zip", encoding="utf-8") + _write_inventory(tmp_path, "data.tgz\ndata.zip\n") + tree = build_tree(tmp_path, "{name}") + assert len(tree["drives"]) == 2, \ + f"length should be 2 but len={len(tree['drives'])}" + # create list of all unique extensions in drives branch + extensions = {Path(path).suffix for path in tree["drives"]["path"]} + assert extensions == {".tgz", ".zip"}, \ + f"drives has unexpected extensions: {extensions}, should be ['.tgz', '.zip']" + + +def test_build_tree_drives_empty_when_no_archives(tmp_path): + (tmp_path / "README.md").write_text("readme", encoding="utf-8") + _write_inventory(tmp_path, "README.md\n") + tree = build_tree(tmp_path, "{name}") + assert len(tree["drives"]) == 0, \ + f"length should be zero but len={len(tree['drives'])}" + + +def test_build_tree_drives_found_in_subdirectories(tmp_path): + subdir = tmp_path / "archives" + subdir.mkdir() + (subdir / "data.tgz").write_text("tgz", encoding="utf-8") + _write_inventory(tmp_path, "archives/data.tgz\n") + tree = build_tree(tmp_path, "{name}") + assert any("data.tgz" in path for path in tree["drives"]["path"]), \ + "drives in subdirectory not found" + +#### build_tree data branch tests ### + +# verify that every file is assigned to only one branch and are accounted for +def test_build_tree_partitioning_and_completeness(sample_tree): + meta_paths = set(sample_tree["meta"]["path"]) + drive_paths = set(sample_tree["drives"]["path"]) + data_paths = { + path + for pf in sample_tree["data"].values() + for path in pf["path"]} + # checks that the three branches have no common files + assert meta_paths.isdisjoint(drive_paths), \ + f"meta and drives have common files: {sorted(meta_paths & drive_paths)}" + assert meta_paths.isdisjoint(data_paths), \ + f"meta and data have common files: {sorted(meta_paths & data_paths)}" + assert drive_paths.isdisjoint(data_paths), \ + f"drives and data have common files: {sorted(drive_paths & data_paths)}" + # the union of all branches must match the inventory exactly + all_tracked = meta_paths | drive_paths | data_paths + expected = set(EXPECTED_FILES) + assert all_tracked == expected, \ + f"Missing: {expected - all_tracked}. Unexpected: {all_tracked - expected}" + + +def test_build_tree_internal_validation_reports_success(eht_dataset, capsys): + build_tree(eht_dataset, sample_fmt) + assert "✓ all inventory files are present" in capsys.readouterr().out, \ + "internal validation did not report success" + + +def test_build_tree_data_empty_when_no_fmt_matches(eht_dataset): + # use a format that doesn't match any files to verify data branch is empty + tree = build_tree(eht_dataset, "{ext}/NONEXISTENT_{year}_{day}.{ext}") + assert len(tree["data"]) == 0, \ + f"length should be zero but len={len(tree['data'])}" + + +# needs unique dataset to test for single stem handling +def test_build_tree_single_stem(tmp_path): + (tmp_path / "csv").mkdir() + (tmp_path / "csv" / "SR1_M87_2017_095_hi_hops_netcal_StokesI.csv").write_text( + "data", encoding="utf-8") + _write_inventory( + tmp_path, + "csv/SR1_M87_2017_095_hi_hops_netcal_StokesI.csv\n") + tree = build_tree(tmp_path, sample_fmt) + assert len(tree["data"]) == 1, \ + f"length should be 1 but len={len(tree['data'])}" + +# test that each data stem contains the expected number of files and schema +def test_build_tree_data_content(sample_tree): + data = sample_tree["data"] + assert len(data) == 2, f"expected 2 stems, got {len(data)}" + for stem, pf in data.items(): + assert isinstance(pf, ParaFrame), f"{stem} is not a ParaFrame" + assert len(pf) == 3, f"{stem} has {len(pf)} files, expected 3" + assert set(pf.columns) == {"path", "ext", "year", "day", "band"}, \ + f"{stem} has {pf.columns}, expected ['path', 'ext', 'year', 'day', 'band']" + formats = set(pf["ext"].unique()) + assert formats == {"csv", "txt", "uvfits"}, \ + f"{stem} has wrong formats: {formats}, expected ['csv', 'txt', 'uvfits']" + +### fmt variations tests ### + +# needs unique dataset to test for nested subdirectory handling +def test_build_tree_nested_subdir_fmt(tmp_path): + (tmp_path / "casa_data" / "April05").mkdir(parents=True) + (tmp_path / "hops_data" / "April05").mkdir(parents=True) + (tmp_path / "casa_data" / "April05" / "SR2_M87_2017_095_hi_casa.uvfits").write_text( + "data", encoding="utf-8") + (tmp_path / "hops_data" / "April05" / "SR2_M87_2017_095_hi_hops.uvfits").write_text( + "data", encoding="utf-8") + _write_inventory( + tmp_path, + "casa_data/April05/SR2_M87_2017_095_hi_casa.uvfits\n" + "hops_data/April05/SR2_M87_2017_095_hi_hops.uvfits\n") + fmt = "SR2_M87_{year}_{day}_{band}_{pipeline}.uvfits" + tree = build_tree(tmp_path, fmt) + assert len(tree["data"]) == 2, \ + f"length should be 2 but len={len(tree['data'])}, couldn't parse nested directories" \ No newline at end of file diff --git a/test/test_hallmark.py b/test/test_hallmark.py index a9bd798..6c2f287 100644 --- a/test/test_hallmark.py +++ b/test/test_hallmark.py @@ -26,7 +26,8 @@ def test_standard_pf_column_names(hallmark_test_suite_dictionary): def test_standard_pf_column_value_types(hallmark_test_suite_dictionary): pf = hallmark_test_suite_dictionary["standard_pf"] assert pd.api.types.is_float_dtype(pf["a"]) - assert pd.api.types.is_float_dtype(pf["i"]) + # i values are whole numbers so they convert to int64, not float64 + assert pd.api.types.is_numeric_dtype(pf["i"]) def test_standard_pf_values(hallmark_test_suite_dictionary): pf = hallmark_test_suite_dictionary["standard_pf"] @@ -48,7 +49,7 @@ def test_standard_glob_pattern_created_properly(hallmark_test_suite_dictionary): def test_standard_glob_returns_expected_files(hallmark_test_suite_dictionary): files = hallmark_test_suite_dictionary["standard_globbed_files"] - assert len(files) == 16 + assert len(files) == 12 def test_standard_pf_single_filter_argument(hallmark_test_suite_dictionary): pf = hallmark_test_suite_dictionary["standard_pf"] @@ -72,7 +73,7 @@ def test_standard_pf_filter_multiple_conditions(hallmark_test_suite_dictionary): ### Test encoded pf def test_encoded_pf_shape(hallmark_test_suite_dictionary): pf = hallmark_test_suite_dictionary["encoded_pf"] - assert pf.shape == (16, 3) + assert pf.shape == (4, 3) def test_encoded_pf_column_names(hallmark_test_suite_dictionary): pf = hallmark_test_suite_dictionary["encoded_pf"] @@ -84,7 +85,7 @@ def test_encoded_pf_has_custom_spin_type(hallmark_test_suite_dictionary): def test_encoded_pf_spin_values(hallmark_test_suite_dictionary): pf = hallmark_test_suite_dictionary["encoded_pf"] - assert set(pf["aspin"].unique()) == {-0.5, 0, 0.75, 0.975} + assert set(pf["aspin"].unique()) == {-0.5} def test_encoded_pf_i_values(hallmark_test_suite_dictionary): pf = hallmark_test_suite_dictionary["encoded_pf"] @@ -99,8 +100,8 @@ def test_encoded_pf_filter_single_value(hallmark_test_suite_dictionary): def test_encoded_pf_filter_multiple_values(hallmark_test_suite_dictionary): pf = hallmark_test_suite_dictionary["encoded_pf"] filtered = pf(aspin=[-0.5, 0.0]) - assert len(filtered) == 8 - assert set(filtered["aspin"].unique()) == {-0.5, 0.0} + assert len(filtered) == 4 + assert set(filtered["aspin"].unique()) == {-0.5} def test_encoded_pf_filter_multiple_conditions(hallmark_test_suite_dictionary): pf = hallmark_test_suite_dictionary["encoded_pf"] @@ -115,7 +116,21 @@ def test_encoded_glob_pattern_created_properly(hallmark_test_suite_dictionary): def test_encoded_glob_returns_expected_files(hallmark_test_suite_dictionary): files = hallmark_test_suite_dictionary["encoded_globbed_files"] - assert len(files) == 16 + assert len(files) == 4 + +# new tests for the two different subdirectories of encoded vs standard files +def test_encoded_pf_paths_are_in_encoded_subdirectory(hallmark_test_suite_dictionary): + pf = hallmark_test_suite_dictionary["encoded_pf"] + assert all(path.startswith("encoded/") for path in pf["path"]) + +def test_standard_pf_not_in_encoded_subdir(hallmark_test_suite_dictionary): + pf = hallmark_test_suite_dictionary["standard_pf"] + assert not any(path.startswith("encoded/") for path in pf["path"]) + +def test_standard_and_encoded_no_overlap(hallmark_test_suite_dictionary): + standard_paths = set(hallmark_test_suite_dictionary["standard_pf"]["path"]) + encoded_paths = set(hallmark_test_suite_dictionary["encoded_pf"]["path"]) + assert standard_paths.isdisjoint(encoded_paths) ### Test repo behavior def test_repo_init_created_dot_hm(hallmark_test_suite_dictionary): From 50a1ce23bb6411823c72da69cd7976f09228d076 Mon Sep 17 00:00:00 2001 From: YaqingSu Date: Thu, 18 Jun 2026 08:53:43 +0800 Subject: [PATCH 02/10] test documentation --- mod/hallmark/repo_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mod/hallmark/repo_config.py b/mod/hallmark/repo_config.py index 49a3ed0..dea4757 100644 --- a/mod/hallmark/repo_config.py +++ b/mod/hallmark/repo_config.py @@ -4,7 +4,7 @@ from pathlib import Path from typing import Dict, Optional - +#test def ensure_branch_data_spec(config: dict) -> dict: data = config.get("data") if not isinstance(data, list) or len(data) != 1 or not isinstance(data[0], dict): From 677c8d4df1da5fda0d49e0ebd8e3a5d65df577b7 Mon Sep 17 00:00:00 2001 From: YaqingSu Date: Thu, 18 Jun 2026 10:42:00 +0800 Subject: [PATCH 03/10] updated documentation --- mod/hallmark/repo.py | 19 +++++++++++++++ mod/hallmark/repo_state.py | 44 +++++++++++++++++++++++++++++++++++ mod/hallmark/repo_worktree.py | 42 +++++++++++++++++++++++++++++++++ mod/hallmark/state.py | 18 ++++++++++++++ 4 files changed, 123 insertions(+) diff --git a/mod/hallmark/repo.py b/mod/hallmark/repo.py index f35278f..7436382 100644 --- a/mod/hallmark/repo.py +++ b/mod/hallmark/repo.py @@ -38,6 +38,14 @@ @contextmanager def chdir(path): + ''' + Temporarily change the current working directory. No Yields. + + Args: + Path: Directory to dwitch to while inside the context. + Returns: + None. + ''' old = os.getcwd() os.chdir(path) try: @@ -61,6 +69,17 @@ class Repo: @staticmethod def lwpaths(path: Union[Path, str]) -> Tuple[Path, Optional[Path]]: + ''' + Resolve repository and worktree paths. + + Args: + path (Path | str): Path to either a worktree or a + ``.hm`` repository. + + Returns: + tuple[Path, Path | None]: A ``(dothm_path, worktree_path)`` tuple. + If ``path`` refers to a ``.hm`` directory, ``worktree_path`` is ``None``. + ''' path = Path(path).resolve() if path.suffix == ".hm": return path, None diff --git a/mod/hallmark/repo_state.py b/mod/hallmark/repo_state.py index 7b4ba9e..e7467b4 100644 --- a/mod/hallmark/repo_state.py +++ b/mod/hallmark/repo_state.py @@ -10,6 +10,17 @@ def load_branch_config(repo, branch: str) -> dict: + ''' + Load the configuration for a specific branch + + Args: + repo (Repo): repository object containing Git and state information/ + branch (String): Name of the branch whose configuration should be loaded + Returns: + (dict) A dictionary contaiing the branch configuration. Returns the + parsed contents of ``config.yml`` from the specified branch when + available, otherwise returns a copy of ``repo.state.config``. + ''' try: return yaml.safe_load(repo.dothm.git.show(f"{branch}:config.yml")) or {} except GitCommandError: @@ -17,6 +28,16 @@ def load_branch_config(repo, branch: str) -> dict: def load_branch_meta(repo, branch: str) -> dict: + ''' + Load the metadata for a specific branch + + Args: + repo (Repo): repository object + branch (String): Name of the branch + Returns: + The contents of ``meta.yml`` from the specified branch. If + metadata can't be loaded, returns ``repo.state.meta``. + ''' try: return yaml.safe_load(repo.dothm.git.show(f"{branch}:meta.yml")) or {} except GitCommandError: @@ -24,6 +45,17 @@ def load_branch_meta(repo, branch: str) -> dict: def load_branch_data(repo, branch: str) -> State: + ''' + Load the state associated with a branch + + Args: + repo (Repo): repository object + branch (String): branch name + Returns: + (State) A ``State`` consturcted from the branch's ``config.yml``, + ``meta.yml``, and ``data.tsv`` files. If the branch does + not exist, returns a copy of the current repository state. + ''' if branch in {head.name for head in repo.dothm.heads}: data = repo.dothm.git.show(f"{branch}:data.tsv") frame = State().data if not data.strip() else None @@ -45,6 +77,18 @@ def load_branch_data(repo, branch: str) -> State: def load_head_state(repo) -> State: + ''' + Load the state stored at ``Head``. + + Args: + repo (Repo): Repository object. + + Returns: + (State) A ``State`` constructed from the ``config.yml``, ``meta.yml``, + and``data.tsv`` files at ``HEAD``. If no state can be loaded + from ``HEAD``, returns a state with the current configuration + and metadata and an empty data table. + ''' try: data = repo.dothm.git.show("HEAD:data.tsv") except GitCommandError: diff --git a/mod/hallmark/repo_worktree.py b/mod/hallmark/repo_worktree.py index 1122bc3..5ebefc7 100644 --- a/mod/hallmark/repo_worktree.py +++ b/mod/hallmark/repo_worktree.py @@ -8,6 +8,16 @@ def effective_cwd(repo) -> Path: + ''' + Determine the effective working directory for repository operations. + Raises RuntimeError if the repository has no worktree + + Args: + repo (repo): repository object + Returns: + path: The current working directory if it is inside the repository + worktree; otherwise, the worktree root. + ''' if repo.worktree is None: raise RuntimeError("cannot inspect files in a bare repository " \ "without a worktree") @@ -22,6 +32,17 @@ def effective_cwd(repo) -> Path: def filtered_paraframe(repo, pf): + ''' + Filter a paraframe to the effective working directory. + + Args: + repo (Repo): Repository object. + pf (ParaFrame): paraframe to filter. + Returns: + Paraframe: The original paraframe if operating at the worktree root. + Otherwise, only rows whose paths lie within the effective working + directory. + ''' root = effective_cwd(repo) worktree = Path(repo.worktree) if root == worktree: @@ -33,11 +54,32 @@ def filtered_paraframe(repo, pf): def tracked_paths(repo) -> set[Path]: + ''' + Return the set of tracked file paths. + + Args: + repo (Repo): Repository object. + + Returns: + set[Path]: Paths of all files tracked in the current + repository state. + ''' fmt = branch_fmt(repo) return {path_from_row(repo, row, fmt) for _, row in repo.state.data.iterrows()} def ensure_clean_tracked_files(repo) -> None: + ''' + Verify that tracked files and repository state are clean. No returns. + Raises CheckoutError if the reposiotry has no worktree, a tracked + file is missing, a tracked file has uncommited changes, or the + hallmark state contains uncommitted changes. + + Args: + repo (Repo): Repository object + Returns: + None. + ''' if repo.worktree is None: raise CheckoutError("cannot checkout without a worktree") diff --git a/mod/hallmark/state.py b/mod/hallmark/state.py index 967dd0e..2884a83 100644 --- a/mod/hallmark/state.py +++ b/mod/hallmark/state.py @@ -39,6 +39,14 @@ class State: ) def update(self, pf): + ''' + Marge paraframe rows into the state database. + + Args: + pf (paraframe): paraframe containing rows to add or update + Returns: + none. + ''' if pf.empty: incoming = pd.DataFrame(columns=self.data.columns if len(self.data.columns) else COLUMNS) @@ -63,6 +71,16 @@ def update(self, pf): self.data = deduped.loc[:, ["sha1", *key_columns]] def replace(self, pf): + ''' + Merge paraframe rows into the tsate database. + Existing rows with matching keys are replaced by the incoming rows. + + Args: + pf (paraframe): Paraframe containing rows to add or update. + + Returns: + None. + ''' if pf.empty: self.data = pd.DataFrame(columns=self.data.columns if len(self.data.columns) else COLUMNS) From 79a5d953b6683ecc8eb13dff6ee108ece899c296 Mon Sep 17 00:00:00 2001 From: YaqingSu Date: Mon, 22 Jun 2026 18:25:37 +0800 Subject: [PATCH 04/10] Updated documentation for repo.py file. --- mod/hallmark/repo.py | 120 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) diff --git a/mod/hallmark/repo.py b/mod/hallmark/repo.py index 7436382..4358924 100644 --- a/mod/hallmark/repo.py +++ b/mod/hallmark/repo.py @@ -86,6 +86,14 @@ def lwpaths(path: Union[Path, str]) -> Tuple[Path, Optional[Path]]: return path / ".hm", path def __init__(self, path: Union[Path, str]) -> None: + ''' + Open an existing hallmark repository. + + Args: + path: path to a worktree or '.hm repository'/ + Returns: + none. + ''' dothm_path, worktree_path = self.lwpaths(path) self.dothm = Dothm(dothm_path) self.worktree = worktree_path and Worktree(worktree_path) @@ -102,6 +110,15 @@ def __init__(self, path: Union[Path, str]) -> None: @classmethod def init(cls, path: Union[Path, str]) -> "Repo": + ''' + Initialize a new hallmark repository. + + Args: + paht(paht|string): path to initialize a worktree or ``.hm`` repository. + + Returns: + Repo: newly created repository instance + ''' dothm_path, worktree_path = cls.lwpaths(path) dothm = Dothm.init(dothm_path) (dothm.path/"config.yml").write_text(Dothm.config_template(),encoding="utf-8") @@ -122,6 +139,21 @@ def clone( max_workers: int = 4, show_progress: bool = False, ) -> "Repo": + ''' + Clone a remote hallmark repository. Raises DestinationExistsError + if the destination path already exists. Raises DownloadError if + data download is enabled and files fail to download. + + Args: + url(string): remote repository URL. + path(path|string): destination path for clone. + fetch_data (boolean): if true, downloads associated data files. + max_workers (integer): Number of parallel workers for + downloading data. + show_progress (boolean): wether to display download progress. + Returns: + Repo: Cloned Repository instance + ''' clone_path = Path(path) if clone_path.exists(): raise DestinationExistsError( @@ -165,6 +197,14 @@ def clone( @staticmethod def checksum(path: Path, chunk_size: int = 1024 * 1024) -> str: + ''' + Computes the SHA1 checksum of a file. + Args: + path (path):path to the file to hash. + chunk_size (integer): size of chunks used for streaming reads. + Returns: + String: Hexadecimal SHA1 digest of the file contents. + ''' digest = sha1() with path.open("rb") as f: for block in iter(lambda: f.read(chunk_size), b""): @@ -172,6 +212,10 @@ def checksum(path: Path, chunk_size: int = 1024 * 1024) -> str: return digest.hexdigest() def add_paths(self, paths: List[Union[Path, str]]) -> ParaFrame: + ''' + Add explicit file paths to the repository index. Raises RuntimeError. + Operation not supported in Hallmark. + ''' raise RuntimeError( 'explicit path add is not supported while data.tsv ' \ 'stores only sha1 plus fmt fields') @@ -184,6 +228,16 @@ def set_config( remote_url: Optional[str] = None, encoding_updates: Optional[Dict[str, str]] = None, ) -> dict: + ''' + Update repository configuration values. + Args: + fmt (string|None): Data dormat specification. + remote_name (string|None): Name of the remote repository. + remote_url (string|None): URL of the remote repository. + encoding_updates (dictioary[str,str] | None): updates to encoding rules. + Returns: + dictionary: Updated configuration dictionary + ''' repo_set_config( self, fmt=fmt, @@ -195,6 +249,19 @@ def set_config( return self.state.config def status(self) -> dict[str, object]: + """ + Return repository status information. Includes staged changes, + orktree modifications, deletions, and untracked files. + + Args: none? + + Returns: + dict[str, object]: Status summary including: + - branch (str) + - staged changes (dict) + - worktree changes (dict) + - untracked files (list[str]) + """ head_state = load_head_state(self) head_map = manifest_map(head_state) staged_map = manifest_map(self.state) @@ -249,6 +316,15 @@ def status(self) -> dict[str, object]: } def add(self, fstr: str, encoding: bool = False) -> ParaFrame: + ''' + Stage files or updated repository indecing from the worktree. + + Args: + fstr (string): Format string or "." for full directory scan. + encoding (boolean): Whether to apply encoding rules. + Returns: + paraframe Parsed and filtered file index (without checksums). + ''' if self.worktree is None: raise RuntimeError( "cannot add files in a bare repository without a worktree") @@ -303,6 +379,16 @@ def add(self, fstr: str, encoding: bool = False) -> ParaFrame: return pf.drop(columns=["sha1"], errors="ignore") def commit(self, msg: str, allow_empty: bool = False) -> bool: + ''' + Commit staged changes to the repository. Raises ValueError if commit + message is empty or invalid. + + Args: + msg (string): commit message. + allow_empty (boolean): Allow comitting even if no changes exists. + Returns: + boolean: True if a commit was created, false otherwise. + ''' if not isinstance(msg, str) or not msg.strip(): raise ValueError("commit message must be a non-empty string") @@ -315,16 +401,41 @@ def commit(self, msg: str, allow_empty: bool = False) -> bool: return False def log(self) -> str: + ''' + Return commit history log. + + Returns: + string: Git log output, or an empty string if no valid HEAD exists. + ''' if not self.dothm.head.is_valid(): return "" return self.dothm.git.log() def branches(self) -> dict[str, object]: + ''' + List repository branches. + Returns: + dictionary[string, object]: Dictionary containing: + - current (string): Active branch name + - names (list[string]): All branch names + ''' current = self.dothm.active_branch.name names = sorted(head.name for head in self.dothm.heads) return {"current": current, "names": names} def checkout(self, target_branch: str) -> bool: + ''' + Switch to a different branch and update the worktree. Raises ValueError if + branch name is invalid. Raises CheckoutError if the workign directoary + is not clean or checkout can't be completed safely. + + Args: + target_branch (string): Branch to switch to. + Returns: + boolean: True if checkout succeeds. + + + ''' if not isinstance(target_branch, str) or not target_branch.strip(): raise ValueError("branch name must be a non-empty string") @@ -383,6 +494,15 @@ def checkout(self, target_branch: str) -> bool: return True def add_worktree(self, target_branch: str) -> bool: + ''' + Create or link a new worktree for a branch. Raises ValueError if branch name is invalid. + Rasies Runtime error if called in a bare repository or worktree creation fails. + + Args: + target_branch (string): Name of the branch to attach. + Returns: + boolean: True if the worktree was successfully created. + ''' from shutil import copy2 from git.exc import GitCommandError From 9d21acb6d9fced6433544bd68ebddc1ae44ee79b Mon Sep 17 00:00:00 2001 From: Sam Jacobs Date: Sat, 27 Jun 2026 13:59:28 -0700 Subject: [PATCH 05/10] removed INVENTORY.txt dependence, added L1 dataset compatibility, started rudamentary fmt detector --- .gitignore | 2 + demo/demo_datatree.ipynb | 217 +++++++++++++ mod/hallmark/eht_datatree.py | 286 +++++++++-------- mod/hallmark/fmt_detection.py | 289 ++++++++++++++++++ test/fmt_detection.py | 0 ...t_datatree.py => test_eht_datatree.py.old} | 221 ++++++++++++-- 6 files changed, 867 insertions(+), 148 deletions(-) create mode 100755 demo/demo_datatree.ipynb create mode 100644 mod/hallmark/fmt_detection.py create mode 100644 test/fmt_detection.py rename test/{test_eht_datatree.py => test_eht_datatree.py.old} (62%) diff --git a/.gitignore b/.gitignore index ca5850a..1b3932c 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,8 @@ !*file !demo/demo_cli.ipynb !demo/demo_python.ipynb +!demo/demo_datatree.ipynb +!*.py.old __pycache__/ *.egg-info/ diff --git a/demo/demo_datatree.ipynb b/demo/demo_datatree.ipynb new file mode 100755 index 0000000..b2bed5a --- /dev/null +++ b/demo/demo_datatree.ipynb @@ -0,0 +1,217 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "77d8504a", + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "from hallmark.eht_datatree import build_tree\n", + "import pandas as pd\n", + "pd.set_option(\"display.width\", 200)\n", + "pd.set_option(\"display.max_columns\", None)\n", + "pd.set_option(\"display.max_colwidth\", 50)" + ] + }, + { + "cell_type": "markdown", + "id": "7bb3a37b", + "metadata": {}, + "source": [ + "a dataree can be built based on the path to where INVENTORY.txt is contained and the data's fmts. Multiple fmts can be inputted. Will validate that every file in the inventory ended up in a branch. Working on implementing parsing the fmts if none are entered. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "91c2e301", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " ✓ all inventory files are present in the tree\n", + "tree keys: ['meta', 'drives', 'data']\n", + "[]\n" + ] + } + ], + "source": [ + "root = Path(\"~/EHTC_FirstSgrAResults_May2022\").expanduser()\n", + "tree = build_tree(root)\n", + "print(\"tree keys:\", list(tree.keys()))" + ] + }, + { + "cell_type": "markdown", + "id": "766ae073", + "metadata": {}, + "source": [ + "\"meta\" contains all the housekeeping files, which will be any files besides drives or files that have fmt naming conventions" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "0cb26601", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "=== META ===\n", + " path ext\n", + "0 EHTC_FirstSgrAResults_May2022_csv/EHTC_FirstSg... csv\n", + "1 EHTC_FirstSgrAResults_May2022_csv/EHTC_FirstSg... csv\n", + "2 EHTC_FirstSgrAResults_May2022_csv/EHTC_FirstSg... csv\n", + "3 EHTC_FirstSgrAResults_May2022_csv/EHTC_FirstSg... csv\n", + "4 EHTC_FirstSgrAResults_May2022_csv/EHTC_FirstSg... csv\n", + ".. ... ...\n", + "68 EHTC_FirstSgrAResults_May2022_uvfits/EHTC_Firs... py\n", + "69 INVENTORY.txt txt\n", + "70 LICENSE.txt txt\n", + "71 README.md md\n", + "72 run.sh sh\n", + "\n", + "[73 rows x 2 columns]\n" + ] + } + ], + "source": [ + "print(\"=== META ===\")\n", + "print(tree[\"meta\"])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c1b7eb90", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"=== DRIVES ===\")\n", + "print(tree[\"drives\"])" + ] + }, + { + "cell_type": "markdown", + "id": "4e6d7164", + "metadata": {}, + "source": [ + "\"data\" will be a nested dict that has an outer list of all the different fmts" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dbc50055", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"=== DATA FMTS ===\")\n", + "for stem in tree[\"data\"].keys():\n", + " print(f\"{stem}\")" + ] + }, + { + "cell_type": "markdown", + "id": "e961e8bc", + "metadata": {}, + "source": [ + "each fmt has a different Paraframe for each stem, as well as a Paraframe of all the stems merged incase someone wants to view all the data for one fmt" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d3747445", + "metadata": {}, + "outputs": [], + "source": [ + "print(f\"=== {{VARIANT}} FMT STEMS ===\")\n", + "for key in tree[\"data\"]\\\n", + " [\"ER6_SGRA_2017_{date}_{band}_{pipeline}_netcal-LMTcal-{variant}_StokesI\"].keys():\n", + " print(key)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9a270ad4", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"=== PARAFRAMES MERGED ===\")\n", + "print(tree[\"data\"]\\\n", + " [\"ER6_SGRA_2017_{date}_{band}_{pipeline}_netcal-LMTcal-{variant}_StokesI\"][\"all\"])" + ] + }, + { + "cell_type": "markdown", + "id": "dfc324ef", + "metadata": {}, + "source": [ + "The invindual stem Paraframes contain the same data with the different file formats" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "866848c8", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"=== 097_hi_hops_besttime STEM ===\")\n", + "print(\n", + "tree[\"data\"][\"ER6_SGRA_2017_{date}_{band}_{pipeline}_netcal-LMTcal-{variant}_StokesI\"]\\\n", + " [\"097_hi_hops_besttime\"])" + ] + }, + { + "cell_type": "markdown", + "id": "ee06d6c6", + "metadata": {}, + "source": [ + "The merged paraframe can be filtered to produce the same output as any given stem" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5c0ee919", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"=== MERGED PARAFRAME FILTERED ===\")\n", + "pf = tree[\"data\"]\\\n", + " [\"ER6_SGRA_2017_{date}_{band}_{pipeline}_netcal-LMTcal-{variant}_StokesI\"][\"all\"]\n", + "print(pf(date=\"097\")(band=\"hi\")(pipeline=\"hops\")(variant=\"besttime\"))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "base", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/mod/hallmark/eht_datatree.py b/mod/hallmark/eht_datatree.py index 853228a..dac57c9 100755 --- a/mod/hallmark/eht_datatree.py +++ b/mod/hallmark/eht_datatree.py @@ -1,101 +1,144 @@ from __future__ import annotations from pathlib import Path +import shutil from hallmark import ParaFrame +from .fmt_detection import detect_fmt, DRIVE_EXTENSIONS +import pandas as pd import parse - -def _strip_suffixes(path: str) -> str: - """Strip all suffixes from a path string to help with parsing inventory files.""" - p = Path(path) - while p.suffix: - p = p.with_suffix("") - return str(p) - -def _alternative_spellings(path: str) -> list[str]: - """Return alternative spellings for known filename variants.""" - alternatives = [path] - if "LICENSE" in path: - alternatives.append(path.replace("LICENSE", "LICENCE")) - if "LICENCE" in path: - alternatives.append(path.replace("LICENCE", "LICENSE")) - return alternatives - -def read_inventory(root: Path) -> list[str]: +import re + +def _extract_drive(drive_path: Path) -> Path: + # remove extension from drive name to create the extraction directory + extract_dir = drive_path.parent / drive_path.stem + # check that the drive has not already been extracted + if not extract_dir.exists(): + shutil.unpack_archive(str(drive_path), str(extract_dir)) + return extract_dir + +# private function used by build_tree to create nested data branch +def _build_data_branches(root: Path, fmt: str | list[str] | None, tracked: set[str],) \ + -> dict: """ - Read INVENTORY.txt and return expected relative file paths. + Build the fmt/stem data structure for files matching the given fmt(s). Args: - root: Path to the dataset root directory containing INVENTORY.txt. + root: Path to search for matching files. + fmt: Format string(s) for parsing data files. If None, + fmts are auto-detected from files list + tracked: Set of relative file paths already accounted for. Returns: - List of relative file path strings expected to exist under root. - - Raises: - FileNotFoundError: If INVENTORY.txt does not exist under root. + Dict of {fmt_str: {stem_key: ParaFrame, ..., "all": ParaFrame}}. """ - # finds path to the inventory file, raises error if not found - inventory_path = Path(root) / "INVENTORY.txt" - if not inventory_path.exists(): - raise FileNotFoundError(f"INVENTORY.txt not found in {root}") - - files_list = [] - # skip directory lines ending in "/" and strip executable marker "*" - for line in inventory_path.read_text(encoding="utf-8").splitlines(): - # remove whitespace - line = line.strip() - # skip empty lines and directories - if not line or line.endswith("/"): - continue - # remove executable marker if present - line = line.rstrip("*") - # check there is a file extension - if not Path(line).suffix: + ### FMT STEMS OUTER DICT ### + # if no fmt was entered, parse the files to find them + if fmt is None: + fmts = detect_fmt(root) + else: + # check if there is only one fmt + fmts = [fmt] if isinstance(fmt, str) else fmt + # make a parser for each fmt with normalized deliminators + parsers = [] + # normalize the deliminators for parsing + for f in fmts: + # normalize the deliminators for parsing + stem = re.sub(r"[\-.]", "_", f) + tokens = stem.split("_") + # keep the full stem in the variants list + variants = [stem] + for i, token in enumerate(tokens): + if re.fullmatch(r"\{p\d+\}", token): + # remove one parameter token at a time to create a variant + dropped = tokens[:i] + tokens[i + 1:] + variants.append("_".join(dropped)) + + for v in variants: + # create a parser for each variant + parsers.append((f, len(v.split("_")), parse.compile(v))) + + # dict of the different fmt stems initialization + fmt_stems = {} + # search all subdirectories from root + for file in root.rglob("*"): + # if the file isn't a dir or a drive + relative_path = str(file.relative_to(root)) + if relative_path in tracked or not file.is_file(): continue - # add cleaned line to files list - files_list.append(line) - return files_list - -def validate(root: Path, tracked: set[str]) -> bool: - """ - Cross-check INVENTORY.txt against a set of tracked files in tree. - - Args: - root: Path to the dataset root containing INVENTORY.txt. - tracked: Set of relative file path strings already in the tree. + # reset parse and fmt for each file + parsed = None + matched_fmt = None + + stem_only = re.sub(r"[\-.]", "_", file.stem) + stem_with_ext = stem_only + re.sub(r"[\-.]", "_", file.suffix) + # counts to see if ext was included in fmt + stem_only_token_count = len(stem_only.split("_")) + stem_with_ext_token_count = len(stem_with_ext.split("_")) + + # parse the file name to extract fields, skip if it doesn't match the format + for fmt_str, variant_token_count, parser in parsers: + # if these are equal, there was no ext in the fmt + if variant_token_count == stem_only_token_count: + candidate = stem_only + # if these are equal, there was an ext in the fmt + elif variant_token_count == stem_with_ext_token_count: + candidate = stem_with_ext + # if neither are equal, this fmt is not compatible with this file + else: + continue + parsed = parser.parse(candidate) + # break out once correct parser is found + if parsed: + matched_fmt = fmt_str + break - Returns: - True if all inventory files are accounted for, False otherwise. - """ - files_list = read_inventory(root) - # add files to missing if not in tracked but in inventory - # checks the end rather than exact match to handle nesting and strip suffixes - missing = [ - file for file in files_list - if not any( - t.endswith(alt) or _strip_suffixes(t).endswith(_strip_suffixes(alt)) - for t in tracked - # catch issues like licence/license - for alt in _alternative_spellings(file))] - # if the missing list is not empty, print missing files message - if missing: - # print how many files are missing and list them - print(f" missing : {len(missing)} file(s)") - for file in missing: - print(f" ✗ {file}") - return False - else: - print(" ✓ all inventory files are present in the tree") - return True - -# common drive extensions to look for when building the tree -DRIVE_EXTENSIONS = [".tgz", ".tar", ".gz", ".zip", ".bz2", ".xz", ".zst", ".7z", ".rar"] + if parsed: + # get path to file from data root + relative_path = str(file.relative_to(root)) + # create unique stem name based on fmt parameters excluding extension + stem_key = "_".join(str(value) for key, value in parsed.named.items() + if key != "ext") + # create a row for this file to add to relevant Paraframe + row = { + "path": relative_path, + # normalize ext to not have period + "ext": Path(relative_path).suffix.lstrip("."), + # one column for each different parsed field + **{key: value for key, value in parsed.named.items() if key != "ext"},} + # add row to dict, and create the dict if it doesn't exist yet + fmt_stems.setdefault(matched_fmt, {}).setdefault(stem_key, []).append(row) + tracked.add(relative_path) -def build_tree(root: Path, fmt: str | list[str]) -> dict: + ### FMT STEMS INNER DICTS ### + # data structure initialization + data_branches = {} + # for every fmt and its stem dict + for fmt_str, stems in fmt_stems.items(): + fmt_dict = {} + all_rows = [] + # loop thru every stem for this fmt + for stem_key, rows in stems.items(): + # create a Paraframe for each different stem + stem_pf = ParaFrame(rows, base_path=root) + # stem Paraframes are nested inside the fmt dict + fmt_dict[stem_key] = stem_pf + all_rows.extend(rows) + # create a Paraframe with all the stems for this dict merged + fmt_dict["all"] = ParaFrame(all_rows, base_path=root) + # each fmt broken into different data branches + data_branches[fmt_str] = fmt_dict + + return data_branches + + +def build_tree(root: Path, fmt: str | list[str] | None = None, data_type: str = "L2")\ + -> dict: """ Build an in-memory pytree for an EHT dataset directory. Args: root: Path to the EHT dataset root directory. fmt: Format string for parsing data files. + data_type: Type of data to build ("L1" or "L2") Returns: A dictionary with keys: @@ -105,15 +148,23 @@ def build_tree(root: Path, fmt: str | list[str]) -> dict: """ # create clean root path root = Path(root).expanduser().resolve() - # track files that are included in the tree, to cross-check against inventory + # track files that are included in the tree to avoid double counting tracked = set() + # L1 data makes recursive calls for each directory, so use glob instead of rglob + glob_fn = root.glob if data_type == "L1" else root.rglob + ### DRIVES ### # collect all drive paths matching any supported extension drive_paths = [] + extracted_dir_names = set() for ext in DRIVE_EXTENSIONS: # add any file that has this ext to the drive paths list and tracked set - for path in root.rglob(f"*{ext}"): + for path in glob_fn(f"*{ext}"): + extract_dir = _extract_drive(path) + # only L1 data needs to track the extracted directory names for recursion + if data_type == "L1": + extracted_dir_names.add(extract_dir.name) drive_paths.append(str(path.relative_to(root))) tracked.add(str(path.relative_to(root))) # build a single ParaFrame from all drive paths with extensions column @@ -123,52 +174,15 @@ def build_tree(root: Path, fmt: str | list[str]) -> dict: base_path=root,) ### DATA ### - # check if there is only one fmt - fmts = [fmt] if isinstance(fmt, str) else fmt - # make a parser for each fmt - parsers = [parse.compile(f) for f in fmts] - stems = {} - # search all subdirectories from root - for file in root.rglob("*"): - # if the file isn't a folder - if not file.is_file(): - continue - # reset parse for each file - parsed = None - # parse the path to extract fields, skip if it doesn't match the format - for parser in parsers: - parsed = (parser.parse(file.name) or - parser.parse(file.stem) or - parser.parse(file.stem.split(".")[0])) - # break out once matching parser found - if parsed: - break - if parsed: - # get path to file from data root - relative_path = str(file.relative_to(root)) - # create unique stem name based on fmt parameters excluding extension - stem_key = "_".join(str(value) for key, value in parsed.named.items() - if key != "ext") - # create stem if it doesn't already exist and add a row for this file - stems.setdefault(stem_key, []).append( - {"path": relative_path, - # normalize ext to not have period - "ext": Path(relative_path).suffix.lstrip("."), - **{key: value for key, value in parsed.named.items() if key != "ext"},} - ) - tracked.add(relative_path) - - data_branches = {} - # create a ParaFrame for each stem and add to the data branches dict - for stem_key, rows in stems.items(): - data_branches[stem_key] = ParaFrame(rows, base_path=root) + data_branches = _build_data_branches(root, fmt, tracked) \ + if data_type == "L2" else {} ### META ### # create a list of all files under root that aren't tracked meta_files = { str(file.relative_to(root)) - for file in root.rglob("*") - # add file if its not a dir, not the .hm, and not in tracked + for file in glob_fn("*") + # add inventory item if its not a dir, not the .hm, and not in tracked if file.is_file() and ".hm" not in file.parts and str(file.relative_to(root)) not in tracked @@ -180,13 +194,23 @@ def build_tree(root: Path, fmt: str | list[str]) -> dict: base_path=root,) for _, row in meta_pf.iterrows(): tracked.add(row["path"]) - - # check all files are in the tree - validate(root, tracked) - # return dict with three keys, only data has subbranches - return { - "meta" : meta_pf, - "drives" : drives_pf, - "data" : data_branches, - } \ No newline at end of file + # create dict with three keys, only data has subbranches + tree = {} + if not meta_pf.empty: + tree["meta"] = meta_pf + if not drives_pf.empty: + tree["drives"] = drives_pf + if data_branches: + tree["data"] = data_branches + + # recursive L1 tree construction + if data_type == "L1": + # call build_tree for each subdirectory + for subdir in sorted(p for p in root.iterdir() if p.is_dir()): + # calls with drives will end recursion + next_level = "L2" if subdir.name in extracted_dir_names else "L1" + # append each subdirectory to the tree + tree[subdir.name] = build_tree(subdir, fmt, data_type=next_level) + + return tree \ No newline at end of file diff --git a/mod/hallmark/fmt_detection.py b/mod/hallmark/fmt_detection.py new file mode 100644 index 0000000..da043b8 --- /dev/null +++ b/mod/hallmark/fmt_detection.py @@ -0,0 +1,289 @@ +import re +from pathlib import Path + +# common meta extensions to look for when building the fmts +META_EXTENSIONS = [".py", ".sh", ".md", ".pdf", ".rst", ".cfg", ".ini", ".yml", ".yaml"\ + , ".sl", ".par", ".xcm"] +# common meta files to look for when building fmts +KNOWN_META_FILES = {"README", "LICENSE", "LICENCE", "INVENTORY", "run"} +# common drive extensions to look for when building fmts +DRIVE_EXTENSIONS = [".tgz", ".tar", ".gz", ".zip", ".bz2", ".xz", ".zst", ".7z", ".rar"] +# delimiter characters fmt detection splits on +_DELIM_PATTERN = r"[_\-.]" + + +def _stems_to_fmts(stems: list[str]) -> list[str]: + """Cluster same-token-count stems into one fmt per shared literal pattern. + + Recursively call until all fmts are found + + Args: + stems: Stems (with extension reattached if present) that all have + the same number of tokens. + + Returns: + List of fmt strings, one per distinct cluster found. + """ + tokenized = {s: re.split(_DELIM_PATTERN, s) for s in stems} + # keep track of delimitors for fmt reconstruction + delimiters = {s: re.findall(_DELIM_PATTERN, s) for s in stems} + remaining_stems = list(stems) + fmts = [] + + while remaining_stems: + # check that every stem shares at least one token + reference_tokens = tokenized[remaining_stems[0]] + token_count = len(reference_tokens) + global_has_fixed = any( + # set will be length 1 if all stems have the same token at this index + len({tokenized[s][index] for s in remaining_stems}) == 1 + for index in range(token_count)) + # cluster found + if global_has_fixed: + cluster = list(remaining_stems) + + else: + # base the anchor around the first non-assigned stem + anchor = remaining_stems[0] + anchor_tokens = tokenized[anchor] + + # group remaining stems by which token positions they share with the anchor + stems_by_shared_positions : dict[tuple, list[str]] = {} + for stem in remaining_stems[1:]: + tokens = tokenized[stem] + shared_positions = tuple( + i for i in range(len(anchor_tokens)) + if tokens[i] == anchor_tokens[i]) + # if there are any tokens that match the anchor + if shared_positions: + stems_by_shared_positions.setdefault(shared_positions, \ + []).append(stem) + + if not stems_by_shared_positions: + # anchor matches nobody on any position and stem doesn't contain fmt + remaining_stems.remove(anchor) + continue + + # find the cluster with the most members that share positions with the anchor + _, best_members = max(stems_by_shared_positions.items(), \ + key=lambda kv: len(kv[1])) + cluster = [anchor] + best_members + + # build the fmt directly from the cluster + cluster_tokenized = [tokenized[stem] for stem in cluster] + fmt_tokens = [] + field_count = 0 + has_fixed = False + for index, token in enumerate(cluster_tokenized[0]): + values = [tokens[index] for tokens in cluster_tokenized] + # if every stem has the same token at this position + if len(set(values)) == 1: + fmt_tokens.append(token) + has_fixed = True + else: + # if the token is different for any, its a parameter + fmt_tokens.append(f"{{p{field_count}}}") + field_count += 1 + + # concatenate the fmt from the tokens and delimiters + if has_fixed and field_count > 0: + cluster_delims = delimiters[cluster[0]] + fmt_parts = [fmt_tokens[0]] + for token, delim in zip(fmt_tokens[1:], cluster_delims): + fmt_parts.append(delim) + fmt_parts.append(token) + fmts.append("".join(fmt_parts)) + # remove all the stems that fit this fmt + for stem in cluster: + remaining_stems.remove(stem) + + return fmts + +def _is_extendable_variant(longer: str, shorter: str) -> str | None: + """ + Check if `shorter` and `longer` are compatible fmt strings, where `longer` + has one additional token that can be removed to match `shorter`. + + Args: + longer: A fmt string with one more token than `shorter`. + shorter: A fmt string with one less token than `longer`. + + Returns: + A new fmt string with the additional token in `longer` replaced by a + parameter placeholder, if the two fmt strings are compatible. Otherwise, + return None. + """ + longer_tokens = re.split(_DELIM_PATTERN, longer) + shorter_tokens = re.split(_DELIM_PATTERN, shorter) + + # if longer doesn't have a single extra token, they can't be compatible + if len(longer_tokens) != len(shorter_tokens) + 1: + return None + + # collect every compatible dropped position + candidates = [] + # check each token position in the longer fmt for compatibility with the shorter fmt + for i in range(len(longer_tokens)): + # remove the token at index i from longer and compare to shorter + candidate = longer_tokens[:i] + longer_tokens[i + 1:] + compatible = True + param_positions = {i} + # check that every token in candidate matches the corresponding token in shorter + for k, (candidate_token, shorter_token) in enumerate( + zip(candidate, shorter_tokens)): + # correcting for the removed token index + longer_idx = k if k < i else k + 1 + + # check if either token is already a parameter placeholder + if re.fullmatch(r"\{p\d+\}", candidate_token) or \ + re.fullmatch(r"\{p\d+\}", shorter_token): + # track the index of the parameter placeholder + param_positions.add(longer_idx) + continue + # if the tokens are not equal, they are incompatible + if candidate_token != shorter_token: + compatible = False + break + if compatible: + # record the num of params, the index, and the positions + candidates.append((len(param_positions), i, param_positions)) + # don't merge if it will result in a fmt with no fixed tokens + if not candidates: + # the shorter fmt is not a compatible variant of the longer fmt + return None + + # find the candidate with the fewest parameter placeholders + min_params = min(count for count, _, _ in candidates) + # only keep candidates with the minimum number of parameter placeholders + best = [c for c in candidates if c[0] == min_params] + # sort by the index of the dropped token in the longer fmt, descending + best.sort(key=lambda c: c[1], reverse=True) + # select the candidate with the highest dropped token index to keep leftmost tokens + _, _, param_positions = best[0] + + # safety check that they aren't the same length, meaning there is no variance + if len(param_positions) == len(longer_tokens): + return None + + # rebuild the fmt string with the parameter placeholders renumbered + new_tokens = [] + field_count = 0 + for index, token in enumerate(longer_tokens): + # if we are at the removed token index or a parameter placeholder + if index in param_positions: + new_tokens.append(f"{{p{field_count}}}") + field_count += 1 + else: + new_tokens.append(token) + + # get the delimiters from the longer fmt string to reconstruct the final fmt + delims = re.findall(_DELIM_PATTERN, longer) + tokens = [new_tokens[0]] + # reconstruct the fmt string with the delimiters in between the tokens + for token, delim in zip(new_tokens[1:], delims): + tokens.append(delim) + tokens.append(token) + return "".join(tokens) + + +def scan_inventory(root: Path) -> list[str]: + """ + Scan a directory and return all file paths found, recursively. + + Args: + root: Path to the dataset root directory to scan. + + Returns: + List of relative file path strings found under root. + + Raises: + FileNotFoundError: If root does not exist. + """ + root = Path(root).expanduser().resolve() + if not root.is_dir(): + raise FileNotFoundError(f"{root} is not a directory") + + return sorted( + # convert to relative path strings for consistency with fmt detection + str(file.relative_to(root)) + for file in root.rglob("*") + # only include files, and ignore any hallmark meta files + if file.is_file() and ".hm" not in file.parts + ) + + +def detect_fmt(root: Path) -> list[str]: + """ + Auto-detect format strings from the files in the directory. + + Args: + root: Path to the dataset root directory containing the files. + + Returns: + List of fmt strings without extensions, one per distinct file + structure found in the files list. + """ + root = Path(root).expanduser().resolve() + inventory_files = scan_inventory(root) + + drive_exts = {ext.lstrip(".").lower() for ext in DRIVE_EXTENSIONS} + meta_exts = {ext.lstrip(".").lower() for ext in META_EXTENSIONS} + + data_stems = [] + seen = set() + for file in inventory_files: + path = Path(file) + # separate the extension from the stem + extension = path.suffix.lstrip(".").lower() + stem = path.stem + + # skip any files that are known meta files or have known meta/drive extensions + if extension in drive_exts or extension in meta_exts: + continue + if stem.split(".")[0] in KNOWN_META_FILES: + continue + # check this stem hasn't already been added to the list + if stem not in seen: + seen.add(stem) + data_stems.append(stem) + + # filter stems by token count to group them for fmt detection + by_token_count: dict[int, list[str]] = {} + for stem in data_stems: + # split the stem up by delimiters + token_count = len(re.split(_DELIM_PATTERN, stem)) + # add the stem to the list of stems with the same token count + by_token_count.setdefault(token_count, []).append(stem) + + group_fmts = [] + for token_count, same_count_stems in sorted(by_token_count.items()): + # create the fmt strings for each group of stems with the same token count + group_fmts.extend(_stems_to_fmts(same_count_stems)) + + # merge fmts that are the same except for an optional param + group_fmts.sort(key=lambda f: len(re.split(_DELIM_PATTERN, f)), reverse=True) + merged = [] + # check if the fmt has already been merged to avoid duplicates + used = [False] * len(group_fmts) + for i, longer in enumerate(group_fmts): + if used[i]: + continue + current = longer + for j, shorter in enumerate(group_fmts): + # skip if the fmt is the same or has already been merged + if i == j or used[j]: + continue + # check if the shorter fmt is an extendable variant of the current fmt + extended = _is_extendable_variant(current, shorter) + if extended is not None: + current = extended + used[j] = True + merged.append(current) + used[i] = True + + fmts = [] + for fmt in merged: + # avoid adding duplicate fmts to the final list + if fmt not in fmts: + fmts.append(fmt) + return fmts \ No newline at end of file diff --git a/test/fmt_detection.py b/test/fmt_detection.py new file mode 100644 index 0000000..e69de29 diff --git a/test/test_eht_datatree.py b/test/test_eht_datatree.py.old similarity index 62% rename from test/test_eht_datatree.py rename to test/test_eht_datatree.py.old index be2dd3f..a886990 100755 --- a/test/test_eht_datatree.py +++ b/test/test_eht_datatree.py.old @@ -1,7 +1,10 @@ from pathlib import Path import pytest from hallmark import ParaFrame -from hallmark.eht_datatree import read_inventory, validate, build_tree +from hallmark.eht_datatree import scan_inventory, build_tree, detect_fmt + +### TODO: Test suite needs reconfigured for all the recent funciton changes ### +### will not pass current tests as functionality has been changed/removed ### # sample inventory content for testing based off real EHT inventory INVENTORY_CONTENT = """\ @@ -45,10 +48,10 @@ def inventory_dir(tmp_path): _write_inventory(tmp_path, INVENTORY_CONTENT) return tmp_path -# create fixture for read_inventory result to use in multiple tests +# create fixture for scan_inventory result to use in multiple tests @pytest.fixture def inventory_result(inventory_dir): - return read_inventory(inventory_dir) + return scan_inventory(inventory_dir) # create test dataset fixture @pytest.fixture @@ -199,8 +202,8 @@ def test_build_tree_ext_column_always_present(tmp_path): _write_inventory(tmp_path, "csv/SR1_M87_2017_095_hi.csv\n") fmt = "SR1_M87_{year}_{day}_{band}.csv" tree = build_tree(tmp_path, fmt) - for stem, pf in tree["data"].items(): - assert "ext" in pf.columns, f"ext column not present in {stem}" + for stem, fmt_dict in tree["data"].items(): + assert "ext" in fmt_dict["all"].columns, f"ext column not present in {stem}" ### build_tree meta branch tests ### @@ -279,8 +282,8 @@ def test_build_tree_partitioning_and_completeness(sample_tree): drive_paths = set(sample_tree["drives"]["path"]) data_paths = { path - for pf in sample_tree["data"].values() - for path in pf["path"]} + for fmt_dict in sample_tree["data"].values() + for path in fmt_dict["all"]["path"]} # checks that the three branches have no common files assert meta_paths.isdisjoint(drive_paths), \ f"meta and drives have common files: {sorted(meta_paths & drive_paths)}" @@ -307,6 +310,33 @@ def test_build_tree_data_empty_when_no_fmt_matches(eht_dataset): assert len(tree["data"]) == 0, \ f"length should be zero but len={len(tree['data'])}" +def test_build_tree_data_matches_mixed_delimiters(tmp_path): + (tmp_path / "ER6_SGRA_2017_096_hi_hops_netcal-LMTcal_StokesI.csv").write_text( + "data", encoding="utf-8") + _write_inventory( + tmp_path, + "ER6_SGRA_2017_096_hi_hops_netcal-LMTcal_StokesI.csv\n") + fmt = "ER6_SGRA_2017_{day}_{band}_{pipeline}_netcal_LMTcal_StokesI" + tree = build_tree(tmp_path, fmt) + assert len(tree["data"]) == 1, \ + f"expected 1 fmt despite delimiter mismatch, got {len(tree['data'])}" + + +def test_build_tree_auto_detected_fmt_matches_mixed_delimiters(tmp_path): + (tmp_path / "ER6_SGRA_2017_096_hi_hops_netcal-LMTcal_StokesI.csv").write_text( + "data", encoding="utf-8") + (tmp_path / "ER6_SGRA_2017_097_lo_casa_netcal-LMTcal_StokesI.csv").write_text( + "data", encoding="utf-8") + _write_inventory( + tmp_path, + "ER6_SGRA_2017_096_hi_hops_netcal-LMTcal_StokesI.csv\n" + "ER6_SGRA_2017_097_lo_casa_netcal-LMTcal_StokesI.csv\n") + tree = build_tree(tmp_path) # fmt is None, triggers auto-detection + assert len(tree["data"]) == 1, \ + f"expected 1 fmt, got {len(tree['data'])}" + fmt_dict = list(tree["data"].values())[0] + assert len(fmt_dict["all"]) == 2, \ + f"expected 2 files, got {len(fmt_dict['all'])}" # needs unique dataset to test for single stem handling def test_build_tree_single_stem(tmp_path): @@ -323,15 +353,16 @@ def test_build_tree_single_stem(tmp_path): # test that each data stem contains the expected number of files and schema def test_build_tree_data_content(sample_tree): data = sample_tree["data"] - assert len(data) == 2, f"expected 2 stems, got {len(data)}" - for stem, pf in data.items(): - assert isinstance(pf, ParaFrame), f"{stem} is not a ParaFrame" - assert len(pf) == 3, f"{stem} has {len(pf)} files, expected 3" - assert set(pf.columns) == {"path", "ext", "year", "day", "band"}, \ - f"{stem} has {pf.columns}, expected ['path', 'ext', 'year', 'day', 'band']" - formats = set(pf["ext"].unique()) - assert formats == {"csv", "txt", "uvfits"}, \ - f"{stem} has wrong formats: {formats}, expected ['csv', 'txt', 'uvfits']" + assert len(data) == 1, f"expected 1 fmt, got {len(data)}" + fmt_dict = list(data.values())[0] + stems = [k for k in fmt_dict if k != "all"] + assert len(stems) == 2, f"expected 2 stems, got {len(stems)}" + for stem in stems: + assert len(fmt_dict[stem]) == 3, \ + f"{stem} has {len(fmt_dict[stem])} files, expected 3" + assert len(fmt_dict["all"]) == 6, \ + f"all has {len(fmt_dict['all'])} files, expected 6" + ### fmt variations tests ### @@ -349,5 +380,161 @@ def test_build_tree_nested_subdir_fmt(tmp_path): "hops_data/April05/SR2_M87_2017_095_hi_hops.uvfits\n") fmt = "SR2_M87_{year}_{day}_{band}_{pipeline}.uvfits" tree = build_tree(tmp_path, fmt) + assert len(tree["data"]) == 1 + fmt_dict = list(tree["data"].values())[0] + stems = [k for k in fmt_dict if k != "all"] + assert len(stems) == 2, \ + f"expected 2 stems but got {len(stems)}, couldn't parse nested directories" + +### detect_fmt tests ### + +def test_detect_fmt_finds_single_pattern(tmp_path): + _write_inventory( + tmp_path, + "csv/SR1_M87_2017_095_hi_hops_netcal_StokesI.csv\n" + "csv/SR1_M87_2017_096_lo_hops_netcal_StokesI.csv\n") + fmts = detect_fmt(tmp_path) + assert fmts == ["SR1_M87_2017_{p0}_{p1}_hops_netcal_StokesI"], f"unexpected fmts: \ + {fmts}, expected 'SR1_M87_2017_{{p0}}_{{p1}}_hops_netcal_StokesI'" + + +def test_detect_fmt_excludes_script_extensions(tmp_path): + _write_inventory( + tmp_path, + "csv/SR1_M87_2017_095_hi.csv\n" + "csv/SR1_M87_2017_096_lo.csv\n" + "csv/dump_csv.py\n" + "csv/convert.py\n") + fmts = detect_fmt(tmp_path) + assert all("dump" not in fmt for fmt in fmts), \ + f"dump files leaked into fmts: {fmts}" + assert all("convert" not in fmt for fmt in fmts), \ + f"convert files leaked into fmts: {fmts}" + + +def test_detect_fmt_excludes_known_meta_files(tmp_path): + _write_inventory( + tmp_path, + "README.txt\n" + "LICENSE.txt\n" + "data/SR1_M87_2017_095_hi.txt\n" + "data/SR1_M87_2017_096_lo.txt\n") + fmts = detect_fmt(tmp_path) + assert all("README" not in fmt for fmt in fmts), \ + f"README file leaked into fmts: {fmts}" + assert all("LICENSE" not in fmt for fmt in fmts), \ + f"LICENSE file leaked into fmts: {fmts}" + + +def test_detect_fmt_separates_by_first_token(tmp_path): + _write_inventory( + tmp_path, + "csv/SR1_M87_2017_095.csv\n" + "csv/SR1_M87_2017_096.csv\n" + "csv/ER6_SGRA_2017_097.csv\n" + "csv/ER6_SGRA_2017_098.csv\n") + fmts = detect_fmt(tmp_path) + assert len(fmts) == 2, f"expected 2 fmts, got {len(fmts)}: {fmts}" + assert any("SR1" in fmt for fmt in fmts), f"SR1 fmt not found in fmts: {fmts}" + assert any("ER6" in fmt for fmt in fmts), f"ER6 fmt not found in fmts: {fmts}" + + +def test_detect_fmt_separates_by_token_count(tmp_path): + # same first token but different number of tokens (optional field case) + _write_inventory( + tmp_path, + "csv/ER6_SGRA_2017_095_hi_netcal_StokesI.csv\n" + "csv/ER6_SGRA_2017_096_lo_netcal_StokesI.csv\n" + "csv/ER6_SGRA_2017_097_hi_netcal_norm_StokesI.csv\n" + "csv/ER6_SGRA_2017_098_lo_netcal_besttime_StokesI.csv\n" + ) + fmts = detect_fmt(tmp_path) + assert len(fmts) == 2, f"expected 2 fmts, got {len(fmts)}: {fmts}" + + +def test_detect_fmt_returns_empty_when_no_data_files(tmp_path): + _write_inventory( tmp_path, "README.md\nLICENSE.txt\nrun.sh\n") + fmts = detect_fmt(tmp_path) + assert fmts == [], f"expected empty list, got {fmts}" + + +def test_detect_fmt_used_automatically_when_fmt_is_none(tmp_path): + (tmp_path / "SR1_M87_2017_095_hi.csv").write_text( + "data", encoding="utf-8") + (tmp_path / "SR1_M87_2017_096_lo.csv").write_text( + "data", encoding="utf-8") + _write_inventory( + tmp_path, + "SR1_M87_2017_095_hi.csv\n" + "SR1_M87_2017_096_lo.csv\n") + tree = build_tree(tmp_path) + assert len(tree["data"]) == 1, \ + f"expected 1 fmt, got {len(tree['data'])}" + +### data branch "all" paraframe tests ### + +def test_build_tree_data_all_merges_every_stem(tmp_path): + for stem_name in ["SR1_M87_2017_095_hi", "SR1_M87_2017_096_lo"]: + (tmp_path / f"{stem_name}.csv").write_text("data", encoding="utf-8") + _write_inventory( + tmp_path, + "SR1_M87_2017_095_hi.csv\n" + "SR1_M87_2017_096_lo.csv\n" + ) + fmt = "SR1_M87_{year}_{day}_{band}.csv" + tree = build_tree(tmp_path, fmt) + fmt_dict = list(tree["data"].values())[0] + stems = [k for k in fmt_dict if k != "all"] + total_stem_rows = sum(len(fmt_dict[stem]) for stem in stems) + assert len(fmt_dict["all"]) == total_stem_rows, \ + f"'all' has {len(fmt_dict['all'])} rows, expected {total_stem_rows}" + + +def test_build_tree_data_multiple_fmts_produce_separate_branches(tmp_path): + (tmp_path / "SR1_M87_2017_095_hi.csv").write_text("data", encoding="utf-8") + (tmp_path / "ER6_SGRA_2017_096_lo.csv").write_text("data", encoding="utf-8") + _write_inventory( + tmp_path, + "SR1_M87_2017_095_hi.csv\n" + "ER6_SGRA_2017_096_lo.csv\n" + ) + fmts = [ + "SR1_M87_{year}_{day}_{band}", + "ER6_SGRA_{year}_{day}_{band}", + ] + tree = build_tree(tmp_path, fmts) assert len(tree["data"]) == 2, \ - f"length should be 2 but len={len(tree['data'])}, couldn't parse nested directories" \ No newline at end of file + f"expected 2 fmt branches, got {len(tree['data'])}" + for fmt_dict in tree["data"].values(): + assert "all" in fmt_dict, "'all' paraframe missing from fmt dict" + + +def test_detect_fmt_separates_when_first_token_varies(tmp_path): + _write_inventory( + tmp_path, + "AA_B_1_0AXG5H.dat\n" + "AA_B_2_0AXG5H.dat\n" + "AP_B_3_0AXG5H.dat\n" + "AP_B_4_0AXG5H.dat\n" + ) + fmts = detect_fmt(tmp_path) + assert len(fmts) == 2, f"expected 2 fmts, got {len(fmts)}: {fmts}" + assert any(fmt.startswith("AA_") for fmt in fmts), \ + f"AA_ fmt not found in fmts: {fmts}" + assert any(fmt.startswith("AP_") for fmt in fmts), \ + f"AP_ fmt not found in fmts: {fmts}" + + +def test_build_tree_data_ext_column_has_no_period(tmp_path): + (tmp_path / "SR1_M87_2017_095_hi.csv").write_text("data", encoding="utf-8") + (tmp_path / "SR1_M87_2017_096_lo.csv").write_text("data", encoding="utf-8") + _write_inventory( + tmp_path, + "SR1_M87_2017_095_hi.csv\n" + "SR1_M87_2017_096_lo.csv\n" + ) + fmt = "SR1_M87_{year}_{day}_{band}.csv" + tree = build_tree(tmp_path, fmt) + fmt_dict = list(tree["data"].values())[0] + exts = set(fmt_dict["all"]["ext"]) + assert exts == {"csv"}, f"expected ext without period, got {exts}" \ No newline at end of file From 67f5866f56be9f72af2030a5c20dec6c8528c7c8 Mon Sep 17 00:00:00 2001 From: Sam Jacobs Date: Sat, 27 Jun 2026 14:00:48 -0700 Subject: [PATCH 06/10] removed INVENTORY.txt dependence, added L1 dataset compatibility, started rudamentary fmt detector --- demo/demo_datatree.ipynb | 838 +++++++++++++++++++++++++++++++--- mod/hallmark/eht_datatree.py | 1 - mod/hallmark/fmt_detection.py | 2 +- 3 files changed, 788 insertions(+), 53 deletions(-) diff --git a/demo/demo_datatree.ipynb b/demo/demo_datatree.ipynb index b2bed5a..78446a6 100755 --- a/demo/demo_datatree.ipynb +++ b/demo/demo_datatree.ipynb @@ -2,15 +2,16 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 48, "id": "77d8504a", "metadata": {}, "outputs": [], "source": [ "from pathlib import Path\n", "from hallmark.eht_datatree import build_tree\n", + "from hallmark.fmt_detection import scan_inventory\n", "import pandas as pd\n", - "pd.set_option(\"display.width\", 200)\n", + "pd.set_option(\"display.width\", 300)\n", "pd.set_option(\"display.max_columns\", None)\n", "pd.set_option(\"display.max_colwidth\", 50)" ] @@ -20,12 +21,12 @@ "id": "7bb3a37b", "metadata": {}, "source": [ - "a dataree can be built based on the path to where INVENTORY.txt is contained and the data's fmts. Multiple fmts can be inputted. Will validate that every file in the inventory ended up in a branch. Working on implementing parsing the fmts if none are entered. " + "a dataree can be built based on the path to the root and the data's fmts. Multiple fmts can be inputted. If no fmts are inputted, the program will try to detect them from the file name patterns, but this functionality still isn't perfect. Flag for if the data is L1 or L2 as the two trees are built slightly different" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 49, "id": "91c2e301", "metadata": {}, "outputs": [ @@ -33,16 +34,241 @@ "name": "stdout", "output_type": "stream", "text": [ - " ✓ all inventory files are present in the tree\n", - "tree keys: ['meta', 'drives', 'data']\n", - "[]\n" + "tree keys: ['meta', 'data']\n", + "AZ2Z.txt\n", + "Aa_tsyspeff_time_e18a24-b1.png\n", + "Aa_tsyspeff_time_e18a24-b2.png\n", + "Aa_tsyspeff_time_e18a24-b3.png\n", + "Aa_tsyspeff_time_e18a24-b4.png\n", + "Aa_tsyspeff_time_e18c21-b1.png\n", + "Aa_tsyspeff_time_e18c21-b2.png\n", + "Aa_tsyspeff_time_e18c21-b3.png\n", + "Aa_tsyspeff_time_e18c21-b4.png\n", + "Aa_tsyspeff_time_e18c25-b1.png\n", + "Aa_tsyspeff_time_e18c25-b2.png\n", + "Aa_tsyspeff_time_e18c25-b3.png\n", + "Aa_tsyspeff_time_e18c25-b4.png\n", + "Aa_tsyspeff_time_e18d28-b1.png\n", + "Aa_tsyspeff_time_e18d28-b2.png\n", + "Aa_tsyspeff_time_e18d28-b3.png\n", + "Aa_tsyspeff_time_e18d28-b4.png\n", + "Aa_tsyspeff_time_e18e22-b1.png\n", + "Aa_tsyspeff_time_e18e22-b2.png\n", + "Aa_tsyspeff_time_e18e22-b3.png\n", + "Aa_tsyspeff_time_e18e22-b4.png\n", + "Aa_tsyspeff_time_e18g27-b1.png\n", + "Aa_tsyspeff_time_e18g27-b2.png\n", + "Aa_tsyspeff_time_e18g27-b3.png\n", + "Aa_tsyspeff_time_e18g27-b4.png\n", + "Ax_TsysL_230ghz2018.html\n", + "Ax_TsysR_230ghz2018.html\n", + "Ax_tsys_comp.png\n", + "Ax_tsys_elev_comp.png\n", + "EHT2018_tau_time_230ghz.png\n", + "EHT2018_tsys_elev_230ghz.png\n", + "EHT2018_tsys_time_230ghz.png\n", + "Gl_TsysL_230ghz2018.html\n", + "Gl_TsysR_230ghz2018.html\n", + "Gl_tsys_comp.png\n", + "Gl_tsys_elev_comp.png\n", + "INVENTORY.txt\n", + "LICENSE.txt\n", + "Lm_tsys_comp.png\n", + "Mg_TsysL_230ghz2018.html\n", + "Mg_TsysR_230ghz2018.html\n", + "Mg_tsys_elev_comp.png\n", + "Mm_TsysR_230ghz2018.html\n", + "Mm_tsys_comp.png\n", + "Mm_tsys_elev_comp.png\n", + "Pv_TsysL_230ghz2018.html\n", + "Pv_TsysR_230ghz2018.html\n", + "Pv_tsys_comp.png\n", + "Pv_tsys_elev_comp.png\n", + "README\n", + "README\n", + "README\n", + "README.pdf\n", + "SMT2Z.txt\n", + "Sw_EHT2018_SEFD_elev_230ghz.png\n", + "Sw_SEFDL_230ghz2018.html\n", + "Sw_SEFDR_230ghz2018.html\n", + "Sw_pheff_e18a24.png\n", + "Sw_pheff_e18c21.png\n", + "Sw_pheff_e18c25.png\n", + "Sw_pheff_e18d28.png\n", + "Sw_pheff_e18e22.png\n", + "Sw_pheff_e18g27.png\n", + "Sw_sefd_time_e18a24.png\n", + "Sw_sefd_time_e18c21.png\n", + "Sw_sefd_time_e18c25.png\n", + "Sw_sefd_time_e18d28.png\n", + "Sw_sefd_time_e18e22.png\n", + "Sw_sefd_time_e18g27.png\n", + "Sw_tsys_elev_e18a24.png\n", + "Sw_tsys_elev_e18c21.png\n", + "Sw_tsys_elev_e18c25.png\n", + "Sw_tsys_elev_e18d28.png\n", + "Sw_tsys_elev_e18e22.png\n", + "Sw_tsys_elev_e18g27.png\n", + "Sw_tsys_time_e18a24.png\n", + "Sw_tsys_time_e18c21.png\n", + "Sw_tsys_time_e18c25.png\n", + "Sw_tsys_time_e18d28.png\n", + "Sw_tsys_time_e18e22.png\n", + "Sw_tsys_time_e18g27.png\n", + "Sz_TsysL_230ghz2018.html\n", + "Sz_TsysR_230ghz2018.html\n", + "Sz_tsys_comp.png\n", + "Sz_tsys_elev_comp.png\n", + "Z2AZ.txt\n", + "Z2SMT.txt\n", + "antL.txt\n", + "ant_locat.txt\n", + "deleted_df.tab\n", + "e18a24_AA.flag\n", + "e18a24_AA.pheff\n", + "e18a24_b1_AA.tsys\n", + "e18a24_b1_AA_PC.ANTAB\n", + "e18a24_b1_proc.AN\n", + "e18a24_b1_raw.AN\n", + "e18a24_b2_AA.tsys\n", + "e18a24_b2_AA_PC.ANTAB\n", + "e18a24_b2_proc.AN\n", + "e18a24_b2_raw.AN\n", + "e18a24_b3_AA.tsys\n", + "e18a24_b3_AA_PC.ANTAB\n", + "e18a24_b3_proc.AN\n", + "e18a24_b3_raw.AN\n", + "e18a24_b4_AA.tsys\n", + "e18a24_b4_AA_PC.ANTAB\n", + "e18a24_b4_proc.AN\n", + "e18a24_b4_raw.AN\n", + "e18c21_AA.flag\n", + "e18c21_AA.pheff\n", + "e18c21_b1_AA.tsys\n", + "e18c21_b1_AA_PC.ANTAB\n", + "e18c21_b1_proc.AN\n", + "e18c21_b1_raw.AN\n", + "e18c21_b2_AA.tsys\n", + "e18c21_b2_AA_PC.ANTAB\n", + "e18c21_b2_proc.AN\n", + "e18c21_b2_raw.AN\n", + "e18c21_b3_AA.tsys\n", + "e18c21_b3_AA_PC.ANTAB\n", + "e18c21_b3_proc.AN\n", + "e18c21_b3_raw.AN\n", + "e18c21_b4_AA.tsys\n", + "e18c21_b4_AA_PC.ANTAB\n", + "e18c21_b4_proc.AN\n", + "e18c21_b4_raw.AN\n", + "e18c25_AA.flag\n", + "e18c25_AA.pheff\n", + "e18c25_b1_AA.tsys\n", + "e18c25_b1_AA_PC.ANTAB\n", + "e18c25_b1_proc.AN\n", + "e18c25_b1_proc_fixAX.AN\n", + "e18c25_b1_raw.AN\n", + "e18c25_b2_AA.tsys\n", + "e18c25_b2_AA_PC.ANTAB\n", + "e18c25_b2_proc.AN\n", + "e18c25_b2_proc_fixAX.AN\n", + "e18c25_b2_raw.AN\n", + "e18c25_b3_AA.tsys\n", + "e18c25_b3_AA_PC.ANTAB\n", + "e18c25_b3_proc.AN\n", + "e18c25_b3_proc_fixAX.AN\n", + "e18c25_b3_raw.AN\n", + "e18c25_b4_AA.tsys\n", + "e18c25_b4_AA_PC.ANTAB\n", + "e18c25_b4_proc.AN\n", + "e18c25_b4_proc_fixAX.AN\n", + "e18c25_b4_raw.AN\n", + "e18d28_AA.flag\n", + "e18d28_AA.pheff\n", + "e18d28_b1_AA.tsys\n", + "e18d28_b1_AA_PC.ANTAB\n", + "e18d28_b1_proc.AN\n", + "e18d28_b1_raw.AN\n", + "e18d28_b2_AA.tsys\n", + "e18d28_b2_AA_PC.ANTAB\n", + "e18d28_b2_proc.AN\n", + "e18d28_b2_raw.AN\n", + "e18d28_b3_AA.tsys\n", + "e18d28_b3_AA_PC.ANTAB\n", + "e18d28_b3_proc.AN\n", + "e18d28_b3_raw.AN\n", + "e18d28_b4_AA.tsys\n", + "e18d28_b4_AA_PC.ANTAB\n", + "e18d28_b4_proc.AN\n", + "e18d28_b4_raw.AN\n", + "e18e22_AA.flag\n", + "e18e22_AA.pheff\n", + "e18e22_b1_AA.tsys\n", + "e18e22_b1_AA_PC.ANTAB\n", + "e18e22_b1_proc.AN\n", + "e18e22_b1_raw.AN\n", + "e18e22_b2_AA.tsys\n", + "e18e22_b2_AA_PC.ANTAB\n", + "e18e22_b2_proc.AN\n", + "e18e22_b2_raw.AN\n", + "e18e22_b3_AA.tsys\n", + "e18e22_b3_AA_PC.ANTAB\n", + "e18e22_b3_proc.AN\n", + "e18e22_b3_raw.AN\n", + "e18e22_b4_AA.tsys\n", + "e18e22_b4_AA_PC.ANTAB\n", + "e18e22_b4_proc.AN\n", + "e18e22_b4_raw.AN\n", + "e18g27_AA.flag\n", + "e18g27_AA.pheff\n", + "e18g27_b1_AA.tsys\n", + "e18g27_b1_AA_PC.ANTAB\n", + "e18g27_b1_proc.AN\n", + "e18g27_b1_raw.AN\n", + "e18g27_b2_AA.tsys\n", + "e18g27_b2_AA_PC.ANTAB\n", + "e18g27_b2_proc.AN\n", + "e18g27_b2_raw.AN\n", + "e18g27_b3_AA.tsys\n", + "e18g27_b3_AA_PC.ANTAB\n", + "e18g27_b3_proc.AN\n", + "e18g27_b3_raw.AN\n", + "e18g27_b4_AA.tsys\n", + "e18g27_b4_AA_PC.ANTAB\n", + "e18g27_b4_proc.AN\n", + "e18g27_b4_raw.AN\n", + "elev_diff.tab\n", + "expt2track.txt\n", + "exptL0.txt\n", + "modify_df.tab\n", + "opt_thick_scans.tab\n", + "sefd_proc.tab\n", + "sourL.txt\n", + "station_dic.txt\n", + "track2expt.txt\n", + "tsa.dict\n", + "tsys_proc.tab\n", + "tsystime_dupl.tab\n", + "vexflag_info.tab\n", + "vtf_elev_check.tab\n", + "vtf_raw.tab\n" ] } ], "source": [ - "root = Path(\"~/EHTC_FirstSgrAResults_May2022\").expanduser()\n", - "tree = build_tree(root)\n", - "print(\"tree keys:\", list(tree.keys()))" + "datasets = [\"EHTC_CenA2017_Jul2021\", \"EHTC_M87-2018_Mar2024\",\"EHTC_SgrAmwl2017_May2022\",\n", + " \"EHTC_First3C279Results_May2020\", \"EHTC_M87mwl2017_Apr2021\", \n", + " \"EHTC_metadata2018_Dec2023\", \"EHTC_FirstM87Results_Apr2019\",\n", + " \"EHTC_M87mwl2018_Dec2024\", \"EHTC_FirstSgrAPol_Mar2024\",\n", + " \"EHTC_M87pol2017_Nov2023\", \"EHTC_FirstSgrAResults_May2022\",\n", + " \"EHTC_MonitoringM87_Sep2020\"]\n", + "root = Path(f\"~/{datasets[5]}\").expanduser()\n", + "tree = build_tree(root, None, data_type=\"L2\")\n", + "print(\"tree keys:\", list(tree.keys()))\n", + "files = scan_inventory(root)\n", + "names = sorted(Path(f).name for f in files)\n", + "for name in names:\n", + " print(name)" ] }, { @@ -55,7 +281,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 50, "id": "0cb26601", "metadata": {}, "outputs": [ @@ -64,20 +290,32 @@ "output_type": "stream", "text": [ "=== META ===\n", - " path ext\n", - "0 EHTC_FirstSgrAResults_May2022_csv/EHTC_FirstSg... csv\n", - "1 EHTC_FirstSgrAResults_May2022_csv/EHTC_FirstSg... csv\n", - "2 EHTC_FirstSgrAResults_May2022_csv/EHTC_FirstSg... csv\n", - "3 EHTC_FirstSgrAResults_May2022_csv/EHTC_FirstSg... csv\n", - "4 EHTC_FirstSgrAResults_May2022_csv/EHTC_FirstSg... csv\n", - ".. ... ...\n", - "68 EHTC_FirstSgrAResults_May2022_uvfits/EHTC_Firs... py\n", - "69 INVENTORY.txt txt\n", - "70 LICENSE.txt txt\n", - "71 README.md md\n", - "72 run.sh sh\n", - "\n", - "[73 rows x 2 columns]\n" + " path ext\n", + "0 INVENTORY.txt txt\n", + "1 LICENSE.txt txt\n", + "2 README.pdf pdf\n", + "3 antab/README \n", + "4 ax/AZ2Z.txt txt\n", + "5 ax/README \n", + "6 ax/SMT2Z.txt txt\n", + "7 ax/Z2AZ.txt txt\n", + "8 ax/Z2SMT.txt txt\n", + "9 ax/antL.txt txt\n", + "10 ax/ant_locat.txt txt\n", + "11 ax/elev_diff.tab tab\n", + "12 ax/expt2track.txt txt\n", + "13 ax/exptL0.txt txt\n", + "14 ax/opt_thick_scans.tab tab\n", + "15 ax/sourL.txt txt\n", + "16 ax/station_dic.txt txt\n", + "17 ax/track2expt.txt txt\n", + "18 ax/tsa.dict dict\n", + "19 ax/tsystime_dupl.tab tab\n", + "20 ax/vexflag_info.tab tab\n", + "21 ax/vtf_elev_check.tab tab\n", + "22 ax/vtf_raw.tab tab\n", + "23 plots/README \n", + "24 plots/Sw_EHT2018_SEFD_elev_230ghz.png png\n" ] } ], @@ -88,13 +326,14 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 51, "id": "c1b7eb90", "metadata": {}, "outputs": [], "source": [ - "print(\"=== DRIVES ===\")\n", - "print(tree[\"drives\"])" + "if \"drives\" in tree:\n", + " print(\"=== DRIVES ===\")\n", + " print(tree[\"drives\"])" ] }, { @@ -107,13 +346,33 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 52, "id": "dbc50055", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "=== DATA FMTS ===\n", + "{p0}_tsys_{p1}_{p2}\n", + "{p0}_{p1}_230ghz2018\n", + "{p0}_{p1}_time_{p2}-{p3}\n", + "{p0}_df\n", + "{p0}_{p1}_proc\n", + "{p0}_b2_{p1}_{p2}\n", + "{p0}_b4_{p1}_{p2}\n", + "{p0}_b3_{p1}_{p2}\n", + "{p0}_b1_{p1}_{p2}\n", + "{p0}_{p1}_AA\n", + "Sw_pheff_{p0}\n" + ] + } + ], "source": [ "print(\"=== DATA FMTS ===\")\n", - "for stem in tree[\"data\"].keys():\n", + "fmt_keys = list(tree[\"data\"].keys())\n", + "for stem in fmt_keys:\n", " print(f\"{stem}\")" ] }, @@ -127,27 +386,92 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 53, "id": "d3747445", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "=== {p0}_tsys_{p1}_{p2} STEMS ===\n", + "Mm_elev_comp\n", + "Ax_comp_png\n", + "Gl_comp_png\n", + "Sz_elev_comp\n", + "Mm_comp_png\n", + "Lm_comp_png\n", + "EHT2018_elev_230ghz\n", + "Mg_elev_comp\n", + "Pv_elev_comp\n", + "Ax_elev_comp\n", + "Sz_comp_png\n", + "Gl_elev_comp\n", + "Pv_comp_png\n", + "proc_tab\n", + "Sw_elev_e18g27\n", + "Sw_elev_e18c21\n", + "Sw_elev_e18a24\n", + "Sw_elev_e18d28\n", + "Sw_elev_e18e22\n", + "Sw_elev_e18c25\n", + "all\n" + ] + } + ], "source": [ - "print(f\"=== {{VARIANT}} FMT STEMS ===\")\n", - "for key in tree[\"data\"]\\\n", - " [\"ER6_SGRA_2017_{date}_{band}_{pipeline}_netcal-LMTcal-{variant}_StokesI\"].keys():\n", + "print(f\"=== {fmt_keys[0]} STEMS ===\")\n", + "stem_keys = list(tree[\"data\"][fmt_keys[0]].keys())\n", + "for key in tree[\"data\"][fmt_keys[0]].keys():\n", " print(key)" ] }, + { + "cell_type": "markdown", + "id": "41e091d0", + "metadata": {}, + "source": [ + "files that share fmts besides a missing parameter will also be grouped with that fmt, despite technically being parsed with a seperate fmt." + ] + }, { "cell_type": "code", - "execution_count": null, + "execution_count": 54, "id": "9a270ad4", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "=== ALL PARAFRAME ===\n", + " path ext p0 p1 p2\n", + "0 plots/Mm_tsys_elev_comp.png png Mm elev comp\n", + "1 plots/Ax_tsys_comp.png png Ax comp png\n", + "2 plots/Gl_tsys_comp.png png Gl comp png\n", + "3 plots/Sz_tsys_elev_comp.png png Sz elev comp\n", + "4 plots/Mm_tsys_comp.png png Mm comp png\n", + "5 plots/Lm_tsys_comp.png png Lm comp png\n", + "6 plots/EHT2018_tsys_elev_230ghz.png png EHT2018 elev 230ghz\n", + "7 plots/Mg_tsys_elev_comp.png png Mg elev comp\n", + "8 plots/Pv_tsys_elev_comp.png png Pv elev comp\n", + "9 plots/Ax_tsys_elev_comp.png png Ax elev comp\n", + "10 plots/Sz_tsys_comp.png png Sz comp png\n", + "11 plots/Gl_tsys_elev_comp.png png Gl elev comp\n", + "12 plots/Pv_tsys_comp.png png Pv comp png\n", + "13 ax/tsys_proc.tab tab NaN proc tab\n", + "14 plots/plots_Sw/Sw_tsys_elev_e18g27.png png Sw elev e18g27\n", + "15 plots/plots_Sw/Sw_tsys_elev_e18c21.png png Sw elev e18c21\n", + "16 plots/plots_Sw/Sw_tsys_elev_e18a24.png png Sw elev e18a24\n", + "17 plots/plots_Sw/Sw_tsys_elev_e18d28.png png Sw elev e18d28\n", + "18 plots/plots_Sw/Sw_tsys_elev_e18e22.png png Sw elev e18e22\n", + "19 plots/plots_Sw/Sw_tsys_elev_e18c25.png png Sw elev e18c25\n" + ] + } + ], "source": [ - "print(\"=== PARAFRAMES MERGED ===\")\n", - "print(tree[\"data\"]\\\n", - " [\"ER6_SGRA_2017_{date}_{band}_{pipeline}_netcal-LMTcal-{variant}_StokesI\"][\"all\"])" + "print(f\"=== ALL PARAFRAME ===\")\n", + "print(tree[\"data\"][fmt_keys[0]][\"all\"])" ] }, { @@ -160,15 +484,24 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 55, "id": "866848c8", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "=== Mm_elev_comp STEM ===\n", + " path ext p0 p1 p2\n", + "0 plots/Mm_tsys_elev_comp.png png Mm elev comp\n" + ] + } + ], "source": [ - "print(\"=== 097_hi_hops_besttime STEM ===\")\n", + "print(f\"=== {stem_keys[0]} STEM ===\")\n", "print(\n", - "tree[\"data\"][\"ER6_SGRA_2017_{date}_{band}_{pipeline}_netcal-LMTcal-{variant}_StokesI\"]\\\n", - " [\"097_hi_hops_besttime\"])" + "tree[\"data\"][fmt_keys[0]][stem_keys[0]])" ] }, { @@ -181,15 +514,418 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 56, "id": "5c0ee919", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "=== MERGED PARAFRAME FILTERED ===\n", + " path ext p0 p1 p2\n", + "0 plots/Mm_tsys_elev_comp.png png Mm elev comp\n" + ] + } + ], "source": [ "print(\"=== MERGED PARAFRAME FILTERED ===\")\n", - "pf = tree[\"data\"]\\\n", - " [\"ER6_SGRA_2017_{date}_{band}_{pipeline}_netcal-LMTcal-{variant}_StokesI\"][\"all\"]\n", - "print(pf(date=\"097\")(band=\"hi\")(pipeline=\"hops\")(variant=\"besttime\"))" + "pf = tree[\"data\"][fmt_keys[0]][\"all\"]\n", + "\n", + "# get cols and params to ease testing and avoid hardcoding values\n", + "param_columns = [column for column in pf.columns if column not in (\"path\", \"ext\")]\n", + "filter_kwargs = {col: pf.iloc[0][col] for col in param_columns}\n", + "for column, value in filter_kwargs.items():\n", + " pf = pf(**{column: value})\n", + " \n", + "# can also print by doing print(pf(column0=\"value0\")(column1=\"value1\"),...)\n", + "print(pf)" + ] + }, + { + "cell_type": "markdown", + "id": "1d58a4af", + "metadata": {}, + "source": [ + "An L1 datatree is built similarly to the L2 case. Each zip drive is treated as its own L2 dataset, and prior to that will rely on recursive build_tree calls creating new branches for each folder until the zip drives are hit" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "19e53ab2", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "=== ROOT KEYS ===\n", + "['meta', '2016.1.01114.V', '2016.1.01154.V']\n" + ] + } + ], + "source": [ + "root = Path(\"~/eht_local_copy\").expanduser()\n", + "drive_fmts = [\"e17a10-7-hi{{p0}}\", \"\"]\n", + "tree = build_tree(root, None, data_type=\"L1\")\n", + "root_keys = list(tree.keys())\n", + "print(\"=== ROOT KEYS ===\")\n", + "print(root_keys)" + ] + }, + { + "cell_type": "markdown", + "id": "a217d6bc", + "metadata": {}, + "source": [ + "No \"data\" branches will be made until we reach a folder that has compressed drives" + ] + }, + { + "cell_type": "code", + "execution_count": 58, + "id": "0f76048a", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "=== META PARAFRAME ===\n", + " path ext\n", + "0 2016.1.01114.V.md5sums md5sums\n", + "1 2016.1.01114.V.sha1sums sha1sums\n", + "2 2016.1.01114.V.sha256sums sha256sums\n", + "3 2016.1.01114.V.sha512sums sha512sums\n", + "4 2016.1.01154.V.md5sums md5sums\n", + "5 2016.1.01154.V.sha1sums sha1sums\n", + "6 2016.1.01154.V.sha256sums sha256sums\n", + "7 2016.1.01154.V.sha512sums sha512sums\n", + "8 INVENTORY.txt txt\n", + "9 LICENSE.txt txt\n", + "10 README.md md\n", + "\n", + "=== 2016.1.01114.V KEYS ===\n", + "['meta', 'drives', 'group.uid___A001_X87c_X245.ec_jlgomez.e17a10-7-hi-oj287-1055-018-4fit', 'group.uid___A001_X87c_X245.ec_jlgomez.e17a10-7-hi-oj287-1055-018-dxin']\n", + "\n", + "=== 2016.1.01154.V KEYS ===\n", + "['meta', 'drives', 'group.uid___A001_X11a7_X37.ec_eht.e17b06-7-hi-m87-3C273-4fit', 'group.uid___A001_X11a7_X37.ec_eht.e17b06-7-hi-m87-3C273-dxin']\n", + "\n" + ] + } + ], + "source": [ + "all_project_keys = {}\n", + "for key in root_keys:\n", + " if key == \"meta\":\n", + " print(\"=== META PARAFRAME ===\")\n", + " print(tree[key])\n", + " print()\n", + " elif key == \"drives\":\n", + " print(\"=== DRIVES PARAFRAME ===\")\n", + " print(tree[key])\n", + " print()\n", + " else:\n", + " project_keys = list(tree[key].keys())\n", + " all_project_keys[key] = project_keys\n", + " print(f\"=== {key} KEYS ===\")\n", + " print(project_keys)\n", + " print()" + ] + }, + { + "cell_type": "code", + "execution_count": 59, + "id": "14ee04ae", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "=== 2016.1.01114.V PROJECT ===\n", + "=== meta PARAFRAME ===\n", + " path ext\n", + "0 group.uid___A001_X87c_X245.ec_jlgomez.README.txt txt\n", + "1 group.uid___A001_X87c_X245.ec_jlgomez.descript... pdf\n", + "2 group.uid___A001_X87c_X245.ec_jlgomez.ehtc-pro... txt\n", + "\n", + "=== drives PARAFRAME ===\n", + " path ext\n", + "0 group.uid___A001_X87c_X245.ec_jlgomez.e17a10-7... tgz\n", + "1 group.uid___A001_X87c_X245.ec_jlgomez.e17a10-7... tgz\n", + "\n", + "=== group.uid___A001_X87c_X245.ec_jlgomez.e17a10-7-hi-oj287-1055-018-4fit KEYS ===\n", + "['meta', 'data']\n", + "\n", + "=== group.uid___A001_X87c_X245.ec_jlgomez.e17a10-7-hi-oj287-1055-018-dxin KEYS ===\n", + "['meta', 'data']\n", + "\n", + "\n", + "=== 2016.1.01154.V PROJECT ===\n", + "=== meta PARAFRAME ===\n", + " path ext\n", + "0 group.uid___A001_X11a7_X37.ec_eht.README.txt txt\n", + "1 group.uid___A001_X11a7_X37.ec_eht.description.pdf pdf\n", + "2 group.uid___A001_X11a7_X37.ec_eht.ehtc-process... txt\n", + "\n", + "=== drives PARAFRAME ===\n", + " path ext\n", + "0 group.uid___A001_X11a7_X37.ec_eht.e17b06-7-hi-... tgz\n", + "1 group.uid___A001_X11a7_X37.ec_eht.e17b06-7-hi-... tgz\n", + "\n", + "=== group.uid___A001_X11a7_X37.ec_eht.e17b06-7-hi-m87-3C273-4fit KEYS ===\n", + "['meta', 'data']\n", + "\n", + "=== group.uid___A001_X11a7_X37.ec_eht.e17b06-7-hi-m87-3C273-dxin KEYS ===\n", + "['meta', 'data']\n", + "\n", + "\n" + ] + } + ], + "source": [ + "all_extract_keys = {}\n", + "for project_name in all_project_keys:\n", + " print(f\"=== {project_name} PROJECT ===\")\n", + " for extract_name in all_project_keys[project_name]:\n", + " if extract_name == \"meta\":\n", + " print(f\"=== {extract_name} PARAFRAME ===\")\n", + " print(tree[project_name][extract_name])\n", + " print()\n", + " elif extract_name == \"drives\":\n", + " print(f\"=== {extract_name} PARAFRAME ===\")\n", + " print(tree[project_name][extract_name])\n", + " print()\n", + " else:\n", + " extract_keys = list(tree[project_name][extract_name].keys())\n", + " all_extract_keys[f\"{project_name}/{extract_name}\"] = extract_keys\n", + " print(f\"=== {extract_name} KEYS ===\")\n", + " print(extract_keys)\n", + " print()\n", + " print()" + ] + }, + { + "cell_type": "code", + "execution_count": 60, + "id": "366eb8f2", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "=== 2016.1.01114.V PROJECT ===\n", + "\n", + "=== meta PARAFRAME ===\n", + " path ext\n", + "0 group.uid___A001_X87c_X245.ec_jlgomez.README.txt txt\n", + "1 group.uid___A001_X87c_X245.ec_jlgomez.descript... pdf\n", + "2 group.uid___A001_X87c_X245.ec_jlgomez.ehtc-pro... txt\n", + "\n", + "=== group.uid___A001_X87c_X245.ec_jlgomez.e17a10-7-hi-oj287-1055-018-4fit FMTS ===\n", + "['ff-oj287-1055+018-{p0}-{p1}-1055+018.{p2}', '{p0}.B.{p1}']\n", + "\n", + "=== group.uid___A001_X87c_X245.ec_jlgomez.e17a10-7-hi-oj287-1055-018-dxin FMTS ===\n", + "['e17a10-7-hi.{p0}']\n", + "\n", + "\n", + "=== 2016.1.01154.V PROJECT ===\n", + "\n", + "=== meta PARAFRAME ===\n", + " path ext\n", + "0 group.uid___A001_X11a7_X37.ec_eht.README.txt txt\n", + "1 group.uid___A001_X11a7_X37.ec_eht.description.pdf pdf\n", + "2 group.uid___A001_X11a7_X37.ec_eht.ehtc-process... txt\n", + "\n", + "=== group.uid___A001_X11a7_X37.ec_eht.e17b06-7-hi-m87-3C273-4fit FMTS ===\n", + "['ff-m87-3C273-096-{p0}-3C273.{p1}', '{p0}.B.{p1}']\n", + "\n", + "=== group.uid___A001_X11a7_X37.ec_eht.e17b06-7-hi-m87-3C273-dxin FMTS ===\n", + "['e17b06-7-hi.{p0}']\n", + "\n", + "\n" + ] + } + ], + "source": [ + "for project_name in all_project_keys:\n", + " print(f\"=== {project_name} PROJECT ===\")\n", + " print()\n", + " if \"meta\" in tree[project_name]:\n", + " print(\"=== meta PARAFRAME ===\")\n", + " print(tree[project_name][\"meta\"])\n", + " print()\n", + " elif \"drives\" in tree[project_name]:\n", + " print(\"=== drives PARAFRAME ===\")\n", + " print(tree[project_name][\"drives\"])\n", + " print()\n", + " for extract_name in all_project_keys[project_name]:\n", + " extract_dict = tree[project_name][extract_name]\n", + " if \"data\" in extract_dict:\n", + " print(f\"=== {extract_name} FMTS ===\")\n", + " print(list(extract_dict[\"data\"].keys()))\n", + " print()\n", + " print()" + ] + }, + { + "cell_type": "code", + "execution_count": 61, + "id": "e810a4f4", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "=== 2016.1.01114.V PROJECT ===\n", + "\n", + "=== group.uid___A001_X87c_X245.ec_jlgomez.e17a10-7-hi-oj287-1055-018-4fit EXTRACT ===\n", + "\n", + "=== ff-oj287-1055+018-{p0}-{p1}-1055+018.{p2} FMT PARAFRAME ===\n", + " path ext p0 p1 p2\n", + "0 3600/ff-oj287-1055+018-099-2317-1055+018.0AXG5... log 099 2317 0AXG5H\n", + "1 3600/ff-oj287-1055+018-100-0035-1055+018.0AXG5... log 100 0035 0AXG5L\n", + "2 3600/ff-oj287-1055+018-100-0123-1055+018.0AXG5... log 100 0123 0AXG5N\n", + "3 3600/ff-oj287-1055+018-100-0221-1055+018.0AXG5... log 100 0221 0AXG5U\n", + "4 3600/ff-oj287-1055+018-099-2348-1055+018.0AXG5... log 099 2348 0AXG5J\n", + "\n", + "=== {p0}.B.{p1} FMT PARAFRAME ===\n", + " path ext p0 p1\n", + "0 3600/100-0123/AP.B.8.0AXG5N 0AXG5N AP 8\n", + "1 3600/100-0123/ZP.B.45.0AXG5N 0AXG5N ZP 45\n", + "2 3600/100-0123/XL.B.29.0AXG5N 0AXG5N XL 29\n", + "3 3600/100-0123/XZ.B.37.0AXG5N 0AXG5N XZ 37\n", + "4 3600/100-0123/AP.B.7.0AXG5N 0AXG5N AP 7\n", + ".. ... ... .. ..\n", + "131 3600/100-0221/AZ.B.12.0AXG5U 0AXG5U AZ 12\n", + "132 3600/100-0221/XP.B.18.0AXG5U 0AXG5U XP 18\n", + "133 3600/100-0221/XZ.B.25.0AXG5U 0AXG5U XZ 25\n", + "134 3600/100-0221/XX.B.21.0AXG5U 0AXG5U XX 21\n", + "135 3600/100-0221/ZZ.B.32.0AXG5U 0AXG5U ZZ 32\n", + "\n", + "[136 rows x 4 columns]\n", + "\n", + "=== group.uid___A001_X87c_X245.ec_jlgomez.e17a10-7-hi-oj287-1055-018-dxin EXTRACT ===\n", + "\n", + "=== e17a10-7-hi.{p0} FMT PARAFRAME ===\n", + " path ext p0\n", + "0 e17a10-7-hi_1007.input input 1007\n", + "1 e17a10-7-hi_1007.machines machines 1007\n", + "2 e17a10-7-hi_1007.im im 1007\n", + "3 e17a10-7-hi_1007.calc calc 1007\n", + "4 e17a10-7-hi_1007.difxlog difxlog 1007\n", + "5 e17a10-7-hi_1007.threads threads 1007\n", + "6 e17a10-7-hi_1007.flag flag 1007\n", + "7 e17a10-7-hi_1011.im im 1011\n", + "8 e17a10-7-hi_1011.flag flag 1011\n", + "9 e17a10-7-hi_1011.calc calc 1011\n", + "10 e17a10-7-hi_1011.input input 1011\n", + "11 e17a10-7-hi_1011.difxlog difxlog 1011\n", + "12 e17a10-7-hi_1011.machines machines 1011\n", + "13 e17a10-7-hi_1011.threads threads 1011\n", + "14 e17a10-7-hi_1003.machines machines 1003\n", + "15 e17a10-7-hi_1003.input input 1003\n", + "16 e17a10-7-hi_1003.flag flag 1003\n", + "17 e17a10-7-hi_1003.difxlog difxlog 1003\n", + "18 e17a10-7-hi_1003.calc calc 1003\n", + "19 e17a10-7-hi_1003.im im 1003\n", + "20 e17a10-7-hi_1003.threads threads 1003\n", + "21 e17a10-7-hi_1016.flag flag 1016\n", + "22 e17a10-7-hi_1016.im im 1016\n", + "23 e17a10-7-hi_1016.machines machines 1016\n", + "24 e17a10-7-hi_1016.input input 1016\n", + "25 e17a10-7-hi_1016.difxlog difxlog 1016\n", + "26 e17a10-7-hi_1016.threads threads 1016\n", + "27 e17a10-7-hi_1016.calc calc 1016\n", + "28 e17a10-7-hi_1000.flag flag 1000\n", + "29 e17a10-7-hi_1000.threads threads 1000\n", + "30 e17a10-7-hi_1000.calc calc 1000\n", + "31 e17a10-7-hi_1000.machines machines 1000\n", + "32 e17a10-7-hi_1000.im im 1000\n", + "33 e17a10-7-hi_1000.difxlog difxlog 1000\n", + "34 e17a10-7-hi_1000.input input 1000\n", + "35 e17a10-7-hi.vex.obs obs vex\n", + "36 e17a10-7-hi.joblist joblist joblist\n", + "37 e17a10-7-hi.v2d v2d v2d\n", + "\n", + "\n", + "=== 2016.1.01154.V PROJECT ===\n", + "\n", + "=== group.uid___A001_X11a7_X37.ec_eht.e17b06-7-hi-m87-3C273-4fit EXTRACT ===\n", + "\n", + "=== ff-m87-3C273-096-{p0}-3C273.{p1} FMT PARAFRAME ===\n", + " path ext p0 p1\n", + "0 3598/ff-m87-3C273-096-0254-3C273.0AMTB3.log log 0254 0AMTB3\n", + "1 3598/ff-m87-3C273-096-0518-3C273.0AMTBC.log log 0518 0AMTBC\n", + "2 3598/ff-m87-3C273-096-0122-3C273.0AMTAW.log log 0122 0AMTAW\n", + "3 3598/ff-m87-3C273-096-0554-3C273.0AMTBI.log log 0554 0AMTBI\n", + "4 3598/ff-m87-3C273-096-0158-3C273.0AMTAZ.log log 0158 0AMTAZ\n", + "5 3598/ff-m87-3C273-096-0408-3C273.0AMTB6.log log 0408 0AMTB6\n", + "6 3598/ff-m87-3C273-096-0332-3C273.0AMTB4.log log 0332 0AMTB4\n", + "7 3598/ff-m87-3C273-096-0702-3C273.0AMTBR.log log 0702 0AMTBR\n", + "8 3598/ff-m87-3C273-096-0218-3C273.0AMTB1.log log 0218 0AMTB1\n", + "9 3598/ff-m87-3C273-096-0046-3C273.0AMTAV.log log 0046 0AMTAV\n", + "10 3598/ff-m87-3C273-096-0626-3C273.0AMTBN.log log 0626 0AMTBN\n", + "11 3598/ff-m87-3C273-096-0734-3C273.0AMTBX.log log 0734 0AMTBX\n", + "12 3598/ff-m87-3C273-096-0446-3C273.0AMTBA.log log 0446 0AMTBA\n", + "\n", + "=== {p0}.B.{p1} FMT PARAFRAME ===\n", + " path ext p0 p1\n", + "0 3598/096-0218/XX.B.17.0AMTB1 0AMTB1 XX 17\n", + "1 3598/096-0046/XX.B.17.0AMTAV 0AMTAV XX 17\n", + "2 3598/096-0254/XX.B.17.0AMTB3 0AMTB3 XX 17\n", + "3 3598/096-0218/LP.B.3.0AMTB1 0AMTB1 LP 3\n", + "4 3598/096-0218/XP.B.14.0AMTB1 0AMTB1 XP 14\n", + ".. ... ... .. ..\n", + "570 3598/096-0254/XL.B.14.0AMTB3 0AMTB3 XL 14\n", + "571 3598/096-0254/XL.B.15.0AMTB3 0AMTB3 XL 15\n", + "572 3598/096-0254/XL.B.16.0AMTB3 0AMTB3 XL 16\n", + "573 3598/096-0254/LL.B.11.0AMTB3 0AMTB3 LL 11\n", + "574 3598/096-0254/XL.B.13.0AMTB3 0AMTB3 XL 13\n", + "\n", + "[575 rows x 4 columns]\n", + "\n", + "=== group.uid___A001_X11a7_X37.ec_eht.e17b06-7-hi-m87-3C273-dxin EXTRACT ===\n", + "\n", + "=== e17b06-7-hi.{p0} FMT PARAFRAME ===\n", + " path ext p0\n", + "0 e17b06-7-hi_1034.input input 1034\n", + "1 e17b06-7-hi_1034.machines machines 1034\n", + "2 e17b06-7-hi_1034.threads threads 1034\n", + "3 e17b06-7-hi_1034.flag flag 1034\n", + "4 e17b06-7-hi_1034.calc calc 1034\n", + ".. ... ... ...\n", + "89 e17b06-7-hi_1046.difxlog difxlog 1046\n", + "90 e17b06-7-hi_1046.calc calc 1046\n", + "91 e17b06-7-hi_1046.flag flag 1046\n", + "92 e17b06-7-hi.v2d v2d v2d\n", + "93 e17b06-7-hi.joblist joblist joblist\n", + "\n", + "[94 rows x 3 columns]\n", + "\n", + "\n" + ] + } + ], + "source": [ + "for project_name in all_project_keys:\n", + " print(f\"=== {project_name} PROJECT ===\")\n", + " print()\n", + " for extract_name in all_project_keys[project_name]:\n", + " if extract_name != \"meta\" and extract_name != \"drives\":\n", + " print(f\"=== {extract_name} EXTRACT ===\")\n", + " print()\n", + " extract_dict = tree[project_name][extract_name]\n", + " if \"data\" in extract_dict:\n", + " for fmt_str, stems in extract_dict[\"data\"].items():\n", + " print(f\"=== {fmt_str} FMT PARAFRAME ===\")\n", + " print(stems[\"all\"])\n", + " print()\n", + " print()" ] } ], diff --git a/mod/hallmark/eht_datatree.py b/mod/hallmark/eht_datatree.py index dac57c9..33d4c38 100755 --- a/mod/hallmark/eht_datatree.py +++ b/mod/hallmark/eht_datatree.py @@ -3,7 +3,6 @@ import shutil from hallmark import ParaFrame from .fmt_detection import detect_fmt, DRIVE_EXTENSIONS -import pandas as pd import parse import re diff --git a/mod/hallmark/fmt_detection.py b/mod/hallmark/fmt_detection.py index da043b8..a20c7da 100644 --- a/mod/hallmark/fmt_detection.py +++ b/mod/hallmark/fmt_detection.py @@ -64,7 +64,7 @@ def _stems_to_fmts(stems: list[str]) -> list[str]: remaining_stems.remove(anchor) continue - # find the cluster with the most members that share positions with the anchor + # find the cluster with most members that share positions with the anchor _, best_members = max(stems_by_shared_positions.items(), \ key=lambda kv: len(kv[1])) cluster = [anchor] + best_members From e62c9c07d287f1860b2e549f2ecac79c7f72682a Mon Sep 17 00:00:00 2001 From: Sam Jacobs Date: Sat, 27 Jun 2026 15:50:57 -0700 Subject: [PATCH 07/10] cleaned up demo --- demo/demo_datatree.ipynb | 693 +++------------------------------------ 1 file changed, 50 insertions(+), 643 deletions(-) diff --git a/demo/demo_datatree.ipynb b/demo/demo_datatree.ipynb index 78446a6..7d0f90e 100755 --- a/demo/demo_datatree.ipynb +++ b/demo/demo_datatree.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 48, + "execution_count": null, "id": "77d8504a", "metadata": {}, "outputs": [], @@ -26,235 +26,10 @@ }, { "cell_type": "code", - "execution_count": 49, + "execution_count": null, "id": "91c2e301", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "tree keys: ['meta', 'data']\n", - "AZ2Z.txt\n", - "Aa_tsyspeff_time_e18a24-b1.png\n", - "Aa_tsyspeff_time_e18a24-b2.png\n", - "Aa_tsyspeff_time_e18a24-b3.png\n", - "Aa_tsyspeff_time_e18a24-b4.png\n", - "Aa_tsyspeff_time_e18c21-b1.png\n", - "Aa_tsyspeff_time_e18c21-b2.png\n", - "Aa_tsyspeff_time_e18c21-b3.png\n", - "Aa_tsyspeff_time_e18c21-b4.png\n", - "Aa_tsyspeff_time_e18c25-b1.png\n", - "Aa_tsyspeff_time_e18c25-b2.png\n", - "Aa_tsyspeff_time_e18c25-b3.png\n", - "Aa_tsyspeff_time_e18c25-b4.png\n", - "Aa_tsyspeff_time_e18d28-b1.png\n", - "Aa_tsyspeff_time_e18d28-b2.png\n", - "Aa_tsyspeff_time_e18d28-b3.png\n", - "Aa_tsyspeff_time_e18d28-b4.png\n", - "Aa_tsyspeff_time_e18e22-b1.png\n", - "Aa_tsyspeff_time_e18e22-b2.png\n", - "Aa_tsyspeff_time_e18e22-b3.png\n", - "Aa_tsyspeff_time_e18e22-b4.png\n", - "Aa_tsyspeff_time_e18g27-b1.png\n", - "Aa_tsyspeff_time_e18g27-b2.png\n", - "Aa_tsyspeff_time_e18g27-b3.png\n", - "Aa_tsyspeff_time_e18g27-b4.png\n", - "Ax_TsysL_230ghz2018.html\n", - "Ax_TsysR_230ghz2018.html\n", - "Ax_tsys_comp.png\n", - "Ax_tsys_elev_comp.png\n", - "EHT2018_tau_time_230ghz.png\n", - "EHT2018_tsys_elev_230ghz.png\n", - "EHT2018_tsys_time_230ghz.png\n", - "Gl_TsysL_230ghz2018.html\n", - "Gl_TsysR_230ghz2018.html\n", - "Gl_tsys_comp.png\n", - "Gl_tsys_elev_comp.png\n", - "INVENTORY.txt\n", - "LICENSE.txt\n", - "Lm_tsys_comp.png\n", - "Mg_TsysL_230ghz2018.html\n", - "Mg_TsysR_230ghz2018.html\n", - "Mg_tsys_elev_comp.png\n", - "Mm_TsysR_230ghz2018.html\n", - "Mm_tsys_comp.png\n", - "Mm_tsys_elev_comp.png\n", - "Pv_TsysL_230ghz2018.html\n", - "Pv_TsysR_230ghz2018.html\n", - "Pv_tsys_comp.png\n", - "Pv_tsys_elev_comp.png\n", - "README\n", - "README\n", - "README\n", - "README.pdf\n", - "SMT2Z.txt\n", - "Sw_EHT2018_SEFD_elev_230ghz.png\n", - "Sw_SEFDL_230ghz2018.html\n", - "Sw_SEFDR_230ghz2018.html\n", - "Sw_pheff_e18a24.png\n", - "Sw_pheff_e18c21.png\n", - "Sw_pheff_e18c25.png\n", - "Sw_pheff_e18d28.png\n", - "Sw_pheff_e18e22.png\n", - "Sw_pheff_e18g27.png\n", - "Sw_sefd_time_e18a24.png\n", - "Sw_sefd_time_e18c21.png\n", - "Sw_sefd_time_e18c25.png\n", - "Sw_sefd_time_e18d28.png\n", - "Sw_sefd_time_e18e22.png\n", - "Sw_sefd_time_e18g27.png\n", - "Sw_tsys_elev_e18a24.png\n", - "Sw_tsys_elev_e18c21.png\n", - "Sw_tsys_elev_e18c25.png\n", - "Sw_tsys_elev_e18d28.png\n", - "Sw_tsys_elev_e18e22.png\n", - "Sw_tsys_elev_e18g27.png\n", - "Sw_tsys_time_e18a24.png\n", - "Sw_tsys_time_e18c21.png\n", - "Sw_tsys_time_e18c25.png\n", - "Sw_tsys_time_e18d28.png\n", - "Sw_tsys_time_e18e22.png\n", - "Sw_tsys_time_e18g27.png\n", - "Sz_TsysL_230ghz2018.html\n", - "Sz_TsysR_230ghz2018.html\n", - "Sz_tsys_comp.png\n", - "Sz_tsys_elev_comp.png\n", - "Z2AZ.txt\n", - "Z2SMT.txt\n", - "antL.txt\n", - "ant_locat.txt\n", - "deleted_df.tab\n", - "e18a24_AA.flag\n", - "e18a24_AA.pheff\n", - "e18a24_b1_AA.tsys\n", - "e18a24_b1_AA_PC.ANTAB\n", - "e18a24_b1_proc.AN\n", - "e18a24_b1_raw.AN\n", - "e18a24_b2_AA.tsys\n", - "e18a24_b2_AA_PC.ANTAB\n", - "e18a24_b2_proc.AN\n", - "e18a24_b2_raw.AN\n", - "e18a24_b3_AA.tsys\n", - "e18a24_b3_AA_PC.ANTAB\n", - "e18a24_b3_proc.AN\n", - "e18a24_b3_raw.AN\n", - "e18a24_b4_AA.tsys\n", - "e18a24_b4_AA_PC.ANTAB\n", - "e18a24_b4_proc.AN\n", - "e18a24_b4_raw.AN\n", - "e18c21_AA.flag\n", - "e18c21_AA.pheff\n", - "e18c21_b1_AA.tsys\n", - "e18c21_b1_AA_PC.ANTAB\n", - "e18c21_b1_proc.AN\n", - "e18c21_b1_raw.AN\n", - "e18c21_b2_AA.tsys\n", - "e18c21_b2_AA_PC.ANTAB\n", - "e18c21_b2_proc.AN\n", - "e18c21_b2_raw.AN\n", - "e18c21_b3_AA.tsys\n", - "e18c21_b3_AA_PC.ANTAB\n", - "e18c21_b3_proc.AN\n", - "e18c21_b3_raw.AN\n", - "e18c21_b4_AA.tsys\n", - "e18c21_b4_AA_PC.ANTAB\n", - "e18c21_b4_proc.AN\n", - "e18c21_b4_raw.AN\n", - "e18c25_AA.flag\n", - "e18c25_AA.pheff\n", - "e18c25_b1_AA.tsys\n", - "e18c25_b1_AA_PC.ANTAB\n", - "e18c25_b1_proc.AN\n", - "e18c25_b1_proc_fixAX.AN\n", - "e18c25_b1_raw.AN\n", - "e18c25_b2_AA.tsys\n", - "e18c25_b2_AA_PC.ANTAB\n", - "e18c25_b2_proc.AN\n", - "e18c25_b2_proc_fixAX.AN\n", - "e18c25_b2_raw.AN\n", - "e18c25_b3_AA.tsys\n", - "e18c25_b3_AA_PC.ANTAB\n", - "e18c25_b3_proc.AN\n", - "e18c25_b3_proc_fixAX.AN\n", - "e18c25_b3_raw.AN\n", - "e18c25_b4_AA.tsys\n", - "e18c25_b4_AA_PC.ANTAB\n", - "e18c25_b4_proc.AN\n", - "e18c25_b4_proc_fixAX.AN\n", - "e18c25_b4_raw.AN\n", - "e18d28_AA.flag\n", - "e18d28_AA.pheff\n", - "e18d28_b1_AA.tsys\n", - "e18d28_b1_AA_PC.ANTAB\n", - "e18d28_b1_proc.AN\n", - "e18d28_b1_raw.AN\n", - "e18d28_b2_AA.tsys\n", - "e18d28_b2_AA_PC.ANTAB\n", - "e18d28_b2_proc.AN\n", - "e18d28_b2_raw.AN\n", - "e18d28_b3_AA.tsys\n", - "e18d28_b3_AA_PC.ANTAB\n", - "e18d28_b3_proc.AN\n", - "e18d28_b3_raw.AN\n", - "e18d28_b4_AA.tsys\n", - "e18d28_b4_AA_PC.ANTAB\n", - "e18d28_b4_proc.AN\n", - "e18d28_b4_raw.AN\n", - "e18e22_AA.flag\n", - "e18e22_AA.pheff\n", - "e18e22_b1_AA.tsys\n", - "e18e22_b1_AA_PC.ANTAB\n", - "e18e22_b1_proc.AN\n", - "e18e22_b1_raw.AN\n", - "e18e22_b2_AA.tsys\n", - "e18e22_b2_AA_PC.ANTAB\n", - "e18e22_b2_proc.AN\n", - "e18e22_b2_raw.AN\n", - "e18e22_b3_AA.tsys\n", - "e18e22_b3_AA_PC.ANTAB\n", - "e18e22_b3_proc.AN\n", - "e18e22_b3_raw.AN\n", - "e18e22_b4_AA.tsys\n", - "e18e22_b4_AA_PC.ANTAB\n", - "e18e22_b4_proc.AN\n", - "e18e22_b4_raw.AN\n", - "e18g27_AA.flag\n", - "e18g27_AA.pheff\n", - "e18g27_b1_AA.tsys\n", - "e18g27_b1_AA_PC.ANTAB\n", - "e18g27_b1_proc.AN\n", - "e18g27_b1_raw.AN\n", - "e18g27_b2_AA.tsys\n", - "e18g27_b2_AA_PC.ANTAB\n", - "e18g27_b2_proc.AN\n", - "e18g27_b2_raw.AN\n", - "e18g27_b3_AA.tsys\n", - "e18g27_b3_AA_PC.ANTAB\n", - "e18g27_b3_proc.AN\n", - "e18g27_b3_raw.AN\n", - "e18g27_b4_AA.tsys\n", - "e18g27_b4_AA_PC.ANTAB\n", - "e18g27_b4_proc.AN\n", - "e18g27_b4_raw.AN\n", - "elev_diff.tab\n", - "expt2track.txt\n", - "exptL0.txt\n", - "modify_df.tab\n", - "opt_thick_scans.tab\n", - "sefd_proc.tab\n", - "sourL.txt\n", - "station_dic.txt\n", - "track2expt.txt\n", - "tsa.dict\n", - "tsys_proc.tab\n", - "tsystime_dupl.tab\n", - "vexflag_info.tab\n", - "vtf_elev_check.tab\n", - "vtf_raw.tab\n" - ] - } - ], + "outputs": [], "source": [ "datasets = [\"EHTC_CenA2017_Jul2021\", \"EHTC_M87-2018_Mar2024\",\"EHTC_SgrAmwl2017_May2022\",\n", " \"EHTC_First3C279Results_May2020\", \"EHTC_M87mwl2017_Apr2021\", \n", @@ -262,13 +37,13 @@ " \"EHTC_M87mwl2018_Dec2024\", \"EHTC_FirstSgrAPol_Mar2024\",\n", " \"EHTC_M87pol2017_Nov2023\", \"EHTC_FirstSgrAResults_May2022\",\n", " \"EHTC_MonitoringM87_Sep2020\"]\n", - "root = Path(f\"~/{datasets[5]}\").expanduser()\n", + "root = Path(f\"~/{datasets[10]}\").expanduser()\n", "tree = build_tree(root, None, data_type=\"L2\")\n", "print(\"tree keys:\", list(tree.keys()))\n", - "files = scan_inventory(root)\n", - "names = sorted(Path(f).name for f in files)\n", - "for name in names:\n", - " print(name)" + "#files = scan_inventory(root)\n", + "#ames = sorted(Path(f).name for f in files)\n", + "#for name in names:\n", + "# print(name)" ] }, { @@ -281,44 +56,10 @@ }, { "cell_type": "code", - "execution_count": 50, + "execution_count": null, "id": "0cb26601", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "=== META ===\n", - " path ext\n", - "0 INVENTORY.txt txt\n", - "1 LICENSE.txt txt\n", - "2 README.pdf pdf\n", - "3 antab/README \n", - "4 ax/AZ2Z.txt txt\n", - "5 ax/README \n", - "6 ax/SMT2Z.txt txt\n", - "7 ax/Z2AZ.txt txt\n", - "8 ax/Z2SMT.txt txt\n", - "9 ax/antL.txt txt\n", - "10 ax/ant_locat.txt txt\n", - "11 ax/elev_diff.tab tab\n", - "12 ax/expt2track.txt txt\n", - "13 ax/exptL0.txt txt\n", - "14 ax/opt_thick_scans.tab tab\n", - "15 ax/sourL.txt txt\n", - "16 ax/station_dic.txt txt\n", - "17 ax/track2expt.txt txt\n", - "18 ax/tsa.dict dict\n", - "19 ax/tsystime_dupl.tab tab\n", - "20 ax/vexflag_info.tab tab\n", - "21 ax/vtf_elev_check.tab tab\n", - "22 ax/vtf_raw.tab tab\n", - "23 plots/README \n", - "24 plots/Sw_EHT2018_SEFD_elev_230ghz.png png\n" - ] - } - ], + "outputs": [], "source": [ "print(\"=== META ===\")\n", "print(tree[\"meta\"])" @@ -326,7 +67,7 @@ }, { "cell_type": "code", - "execution_count": 51, + "execution_count": null, "id": "c1b7eb90", "metadata": {}, "outputs": [], @@ -346,29 +87,10 @@ }, { "cell_type": "code", - "execution_count": 52, + "execution_count": null, "id": "dbc50055", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "=== DATA FMTS ===\n", - "{p0}_tsys_{p1}_{p2}\n", - "{p0}_{p1}_230ghz2018\n", - "{p0}_{p1}_time_{p2}-{p3}\n", - "{p0}_df\n", - "{p0}_{p1}_proc\n", - "{p0}_b2_{p1}_{p2}\n", - "{p0}_b4_{p1}_{p2}\n", - "{p0}_b3_{p1}_{p2}\n", - "{p0}_b1_{p1}_{p2}\n", - "{p0}_{p1}_AA\n", - "Sw_pheff_{p0}\n" - ] - } - ], + "outputs": [], "source": [ "print(\"=== DATA FMTS ===\")\n", "fmt_keys = list(tree[\"data\"].keys())\n", @@ -386,39 +108,10 @@ }, { "cell_type": "code", - "execution_count": 53, + "execution_count": null, "id": "d3747445", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "=== {p0}_tsys_{p1}_{p2} STEMS ===\n", - "Mm_elev_comp\n", - "Ax_comp_png\n", - "Gl_comp_png\n", - "Sz_elev_comp\n", - "Mm_comp_png\n", - "Lm_comp_png\n", - "EHT2018_elev_230ghz\n", - "Mg_elev_comp\n", - "Pv_elev_comp\n", - "Ax_elev_comp\n", - "Sz_comp_png\n", - "Gl_elev_comp\n", - "Pv_comp_png\n", - "proc_tab\n", - "Sw_elev_e18g27\n", - "Sw_elev_e18c21\n", - "Sw_elev_e18a24\n", - "Sw_elev_e18d28\n", - "Sw_elev_e18e22\n", - "Sw_elev_e18c25\n", - "all\n" - ] - } - ], + "outputs": [], "source": [ "print(f\"=== {fmt_keys[0]} STEMS ===\")\n", "stem_keys = list(tree[\"data\"][fmt_keys[0]].keys())\n", @@ -436,39 +129,10 @@ }, { "cell_type": "code", - "execution_count": 54, + "execution_count": null, "id": "9a270ad4", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "=== ALL PARAFRAME ===\n", - " path ext p0 p1 p2\n", - "0 plots/Mm_tsys_elev_comp.png png Mm elev comp\n", - "1 plots/Ax_tsys_comp.png png Ax comp png\n", - "2 plots/Gl_tsys_comp.png png Gl comp png\n", - "3 plots/Sz_tsys_elev_comp.png png Sz elev comp\n", - "4 plots/Mm_tsys_comp.png png Mm comp png\n", - "5 plots/Lm_tsys_comp.png png Lm comp png\n", - "6 plots/EHT2018_tsys_elev_230ghz.png png EHT2018 elev 230ghz\n", - "7 plots/Mg_tsys_elev_comp.png png Mg elev comp\n", - "8 plots/Pv_tsys_elev_comp.png png Pv elev comp\n", - "9 plots/Ax_tsys_elev_comp.png png Ax elev comp\n", - "10 plots/Sz_tsys_comp.png png Sz comp png\n", - "11 plots/Gl_tsys_elev_comp.png png Gl elev comp\n", - "12 plots/Pv_tsys_comp.png png Pv comp png\n", - "13 ax/tsys_proc.tab tab NaN proc tab\n", - "14 plots/plots_Sw/Sw_tsys_elev_e18g27.png png Sw elev e18g27\n", - "15 plots/plots_Sw/Sw_tsys_elev_e18c21.png png Sw elev e18c21\n", - "16 plots/plots_Sw/Sw_tsys_elev_e18a24.png png Sw elev e18a24\n", - "17 plots/plots_Sw/Sw_tsys_elev_e18d28.png png Sw elev e18d28\n", - "18 plots/plots_Sw/Sw_tsys_elev_e18e22.png png Sw elev e18e22\n", - "19 plots/plots_Sw/Sw_tsys_elev_e18c25.png png Sw elev e18c25\n" - ] - } - ], + "outputs": [], "source": [ "print(f\"=== ALL PARAFRAME ===\")\n", "print(tree[\"data\"][fmt_keys[0]][\"all\"])" @@ -484,20 +148,10 @@ }, { "cell_type": "code", - "execution_count": 55, + "execution_count": null, "id": "866848c8", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "=== Mm_elev_comp STEM ===\n", - " path ext p0 p1 p2\n", - "0 plots/Mm_tsys_elev_comp.png png Mm elev comp\n" - ] - } - ], + "outputs": [], "source": [ "print(f\"=== {stem_keys[0]} STEM ===\")\n", "print(\n", @@ -514,20 +168,10 @@ }, { "cell_type": "code", - "execution_count": 56, + "execution_count": null, "id": "5c0ee919", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "=== MERGED PARAFRAME FILTERED ===\n", - " path ext p0 p1 p2\n", - "0 plots/Mm_tsys_elev_comp.png png Mm elev comp\n" - ] - } - ], + "outputs": [], "source": [ "print(\"=== MERGED PARAFRAME FILTERED ===\")\n", "pf = tree[\"data\"][fmt_keys[0]][\"all\"]\n", @@ -555,20 +199,15 @@ "execution_count": null, "id": "19e53ab2", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "=== ROOT KEYS ===\n", - "['meta', '2016.1.01114.V', '2016.1.01154.V']\n" - ] - } - ], + "outputs": [], "source": [ "root = Path(\"~/eht_local_copy\").expanduser()\n", - "drive_fmts = [\"e17a10-7-hi{{p0}}\", \"\"]\n", - "tree = build_tree(root, None, data_type=\"L1\")\n", + "drive_fmts = [\n", + " \"e17a10-7-hi-{p0}.{p1}\",\n", + " \"e17b06-7-hi_{p2}.{p3}\",\n", + " \"ff-{p0}-{p1}-{p2}-{p3}-{p4}.{p5}\",\n", + " \"{p0}.B.{p1}.{p2}\",]\n", + "tree = build_tree(root, drive_fmts, data_type=\"L1\")\n", "root_keys = list(tree.keys())\n", "print(\"=== ROOT KEYS ===\")\n", "print(root_keys)" @@ -584,37 +223,10 @@ }, { "cell_type": "code", - "execution_count": 58, + "execution_count": null, "id": "0f76048a", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "=== META PARAFRAME ===\n", - " path ext\n", - "0 2016.1.01114.V.md5sums md5sums\n", - "1 2016.1.01114.V.sha1sums sha1sums\n", - "2 2016.1.01114.V.sha256sums sha256sums\n", - "3 2016.1.01114.V.sha512sums sha512sums\n", - "4 2016.1.01154.V.md5sums md5sums\n", - "5 2016.1.01154.V.sha1sums sha1sums\n", - "6 2016.1.01154.V.sha256sums sha256sums\n", - "7 2016.1.01154.V.sha512sums sha512sums\n", - "8 INVENTORY.txt txt\n", - "9 LICENSE.txt txt\n", - "10 README.md md\n", - "\n", - "=== 2016.1.01114.V KEYS ===\n", - "['meta', 'drives', 'group.uid___A001_X87c_X245.ec_jlgomez.e17a10-7-hi-oj287-1055-018-4fit', 'group.uid___A001_X87c_X245.ec_jlgomez.e17a10-7-hi-oj287-1055-018-dxin']\n", - "\n", - "=== 2016.1.01154.V KEYS ===\n", - "['meta', 'drives', 'group.uid___A001_X11a7_X37.ec_eht.e17b06-7-hi-m87-3C273-4fit', 'group.uid___A001_X11a7_X37.ec_eht.e17b06-7-hi-m87-3C273-dxin']\n", - "\n" - ] - } - ], + "outputs": [], "source": [ "all_project_keys = {}\n", "for key in root_keys:\n", @@ -630,61 +242,17 @@ " project_keys = list(tree[key].keys())\n", " all_project_keys[key] = project_keys\n", " print(f\"=== {key} KEYS ===\")\n", - " print(project_keys)\n", + " for key in project_keys:\n", + " print(key)\n", " print()" ] }, { "cell_type": "code", - "execution_count": 59, + "execution_count": null, "id": "14ee04ae", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "=== 2016.1.01114.V PROJECT ===\n", - "=== meta PARAFRAME ===\n", - " path ext\n", - "0 group.uid___A001_X87c_X245.ec_jlgomez.README.txt txt\n", - "1 group.uid___A001_X87c_X245.ec_jlgomez.descript... pdf\n", - "2 group.uid___A001_X87c_X245.ec_jlgomez.ehtc-pro... txt\n", - "\n", - "=== drives PARAFRAME ===\n", - " path ext\n", - "0 group.uid___A001_X87c_X245.ec_jlgomez.e17a10-7... tgz\n", - "1 group.uid___A001_X87c_X245.ec_jlgomez.e17a10-7... tgz\n", - "\n", - "=== group.uid___A001_X87c_X245.ec_jlgomez.e17a10-7-hi-oj287-1055-018-4fit KEYS ===\n", - "['meta', 'data']\n", - "\n", - "=== group.uid___A001_X87c_X245.ec_jlgomez.e17a10-7-hi-oj287-1055-018-dxin KEYS ===\n", - "['meta', 'data']\n", - "\n", - "\n", - "=== 2016.1.01154.V PROJECT ===\n", - "=== meta PARAFRAME ===\n", - " path ext\n", - "0 group.uid___A001_X11a7_X37.ec_eht.README.txt txt\n", - "1 group.uid___A001_X11a7_X37.ec_eht.description.pdf pdf\n", - "2 group.uid___A001_X11a7_X37.ec_eht.ehtc-process... txt\n", - "\n", - "=== drives PARAFRAME ===\n", - " path ext\n", - "0 group.uid___A001_X11a7_X37.ec_eht.e17b06-7-hi-... tgz\n", - "1 group.uid___A001_X11a7_X37.ec_eht.e17b06-7-hi-... tgz\n", - "\n", - "=== group.uid___A001_X11a7_X37.ec_eht.e17b06-7-hi-m87-3C273-4fit KEYS ===\n", - "['meta', 'data']\n", - "\n", - "=== group.uid___A001_X11a7_X37.ec_eht.e17b06-7-hi-m87-3C273-dxin KEYS ===\n", - "['meta', 'data']\n", - "\n", - "\n" - ] - } - ], + "outputs": [], "source": [ "all_extract_keys = {}\n", "for project_name in all_project_keys:\n", @@ -702,54 +270,18 @@ " extract_keys = list(tree[project_name][extract_name].keys())\n", " all_extract_keys[f\"{project_name}/{extract_name}\"] = extract_keys\n", " print(f\"=== {extract_name} KEYS ===\")\n", - " print(extract_keys)\n", + " for key in extract_keys:\n", + " print(key)\n", " print()\n", " print()" ] }, { "cell_type": "code", - "execution_count": 60, + "execution_count": null, "id": "366eb8f2", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "=== 2016.1.01114.V PROJECT ===\n", - "\n", - "=== meta PARAFRAME ===\n", - " path ext\n", - "0 group.uid___A001_X87c_X245.ec_jlgomez.README.txt txt\n", - "1 group.uid___A001_X87c_X245.ec_jlgomez.descript... pdf\n", - "2 group.uid___A001_X87c_X245.ec_jlgomez.ehtc-pro... txt\n", - "\n", - "=== group.uid___A001_X87c_X245.ec_jlgomez.e17a10-7-hi-oj287-1055-018-4fit FMTS ===\n", - "['ff-oj287-1055+018-{p0}-{p1}-1055+018.{p2}', '{p0}.B.{p1}']\n", - "\n", - "=== group.uid___A001_X87c_X245.ec_jlgomez.e17a10-7-hi-oj287-1055-018-dxin FMTS ===\n", - "['e17a10-7-hi.{p0}']\n", - "\n", - "\n", - "=== 2016.1.01154.V PROJECT ===\n", - "\n", - "=== meta PARAFRAME ===\n", - " path ext\n", - "0 group.uid___A001_X11a7_X37.ec_eht.README.txt txt\n", - "1 group.uid___A001_X11a7_X37.ec_eht.description.pdf pdf\n", - "2 group.uid___A001_X11a7_X37.ec_eht.ehtc-process... txt\n", - "\n", - "=== group.uid___A001_X11a7_X37.ec_eht.e17b06-7-hi-m87-3C273-4fit FMTS ===\n", - "['ff-m87-3C273-096-{p0}-3C273.{p1}', '{p0}.B.{p1}']\n", - "\n", - "=== group.uid___A001_X11a7_X37.ec_eht.e17b06-7-hi-m87-3C273-dxin FMTS ===\n", - "['e17b06-7-hi.{p0}']\n", - "\n", - "\n" - ] - } - ], + "outputs": [], "source": [ "for project_name in all_project_keys:\n", " print(f\"=== {project_name} PROJECT ===\")\n", @@ -766,151 +298,26 @@ " extract_dict = tree[project_name][extract_name]\n", " if \"data\" in extract_dict:\n", " print(f\"=== {extract_name} FMTS ===\")\n", - " print(list(extract_dict[\"data\"].keys()))\n", + " for fmt in extract_dict[\"data\"]:\n", + " print(f\"{fmt}\")\n", " print()\n", " print()" ] }, + { + "cell_type": "markdown", + "id": "557196c6", + "metadata": {}, + "source": [ + "inconsistant naming conventions makes it difficult to fair all files to the correct fmt" + ] + }, { "cell_type": "code", - "execution_count": 61, + "execution_count": null, "id": "e810a4f4", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "=== 2016.1.01114.V PROJECT ===\n", - "\n", - "=== group.uid___A001_X87c_X245.ec_jlgomez.e17a10-7-hi-oj287-1055-018-4fit EXTRACT ===\n", - "\n", - "=== ff-oj287-1055+018-{p0}-{p1}-1055+018.{p2} FMT PARAFRAME ===\n", - " path ext p0 p1 p2\n", - "0 3600/ff-oj287-1055+018-099-2317-1055+018.0AXG5... log 099 2317 0AXG5H\n", - "1 3600/ff-oj287-1055+018-100-0035-1055+018.0AXG5... log 100 0035 0AXG5L\n", - "2 3600/ff-oj287-1055+018-100-0123-1055+018.0AXG5... log 100 0123 0AXG5N\n", - "3 3600/ff-oj287-1055+018-100-0221-1055+018.0AXG5... log 100 0221 0AXG5U\n", - "4 3600/ff-oj287-1055+018-099-2348-1055+018.0AXG5... log 099 2348 0AXG5J\n", - "\n", - "=== {p0}.B.{p1} FMT PARAFRAME ===\n", - " path ext p0 p1\n", - "0 3600/100-0123/AP.B.8.0AXG5N 0AXG5N AP 8\n", - "1 3600/100-0123/ZP.B.45.0AXG5N 0AXG5N ZP 45\n", - "2 3600/100-0123/XL.B.29.0AXG5N 0AXG5N XL 29\n", - "3 3600/100-0123/XZ.B.37.0AXG5N 0AXG5N XZ 37\n", - "4 3600/100-0123/AP.B.7.0AXG5N 0AXG5N AP 7\n", - ".. ... ... .. ..\n", - "131 3600/100-0221/AZ.B.12.0AXG5U 0AXG5U AZ 12\n", - "132 3600/100-0221/XP.B.18.0AXG5U 0AXG5U XP 18\n", - "133 3600/100-0221/XZ.B.25.0AXG5U 0AXG5U XZ 25\n", - "134 3600/100-0221/XX.B.21.0AXG5U 0AXG5U XX 21\n", - "135 3600/100-0221/ZZ.B.32.0AXG5U 0AXG5U ZZ 32\n", - "\n", - "[136 rows x 4 columns]\n", - "\n", - "=== group.uid___A001_X87c_X245.ec_jlgomez.e17a10-7-hi-oj287-1055-018-dxin EXTRACT ===\n", - "\n", - "=== e17a10-7-hi.{p0} FMT PARAFRAME ===\n", - " path ext p0\n", - "0 e17a10-7-hi_1007.input input 1007\n", - "1 e17a10-7-hi_1007.machines machines 1007\n", - "2 e17a10-7-hi_1007.im im 1007\n", - "3 e17a10-7-hi_1007.calc calc 1007\n", - "4 e17a10-7-hi_1007.difxlog difxlog 1007\n", - "5 e17a10-7-hi_1007.threads threads 1007\n", - "6 e17a10-7-hi_1007.flag flag 1007\n", - "7 e17a10-7-hi_1011.im im 1011\n", - "8 e17a10-7-hi_1011.flag flag 1011\n", - "9 e17a10-7-hi_1011.calc calc 1011\n", - "10 e17a10-7-hi_1011.input input 1011\n", - "11 e17a10-7-hi_1011.difxlog difxlog 1011\n", - "12 e17a10-7-hi_1011.machines machines 1011\n", - "13 e17a10-7-hi_1011.threads threads 1011\n", - "14 e17a10-7-hi_1003.machines machines 1003\n", - "15 e17a10-7-hi_1003.input input 1003\n", - "16 e17a10-7-hi_1003.flag flag 1003\n", - "17 e17a10-7-hi_1003.difxlog difxlog 1003\n", - "18 e17a10-7-hi_1003.calc calc 1003\n", - "19 e17a10-7-hi_1003.im im 1003\n", - "20 e17a10-7-hi_1003.threads threads 1003\n", - "21 e17a10-7-hi_1016.flag flag 1016\n", - "22 e17a10-7-hi_1016.im im 1016\n", - "23 e17a10-7-hi_1016.machines machines 1016\n", - "24 e17a10-7-hi_1016.input input 1016\n", - "25 e17a10-7-hi_1016.difxlog difxlog 1016\n", - "26 e17a10-7-hi_1016.threads threads 1016\n", - "27 e17a10-7-hi_1016.calc calc 1016\n", - "28 e17a10-7-hi_1000.flag flag 1000\n", - "29 e17a10-7-hi_1000.threads threads 1000\n", - "30 e17a10-7-hi_1000.calc calc 1000\n", - "31 e17a10-7-hi_1000.machines machines 1000\n", - "32 e17a10-7-hi_1000.im im 1000\n", - "33 e17a10-7-hi_1000.difxlog difxlog 1000\n", - "34 e17a10-7-hi_1000.input input 1000\n", - "35 e17a10-7-hi.vex.obs obs vex\n", - "36 e17a10-7-hi.joblist joblist joblist\n", - "37 e17a10-7-hi.v2d v2d v2d\n", - "\n", - "\n", - "=== 2016.1.01154.V PROJECT ===\n", - "\n", - "=== group.uid___A001_X11a7_X37.ec_eht.e17b06-7-hi-m87-3C273-4fit EXTRACT ===\n", - "\n", - "=== ff-m87-3C273-096-{p0}-3C273.{p1} FMT PARAFRAME ===\n", - " path ext p0 p1\n", - "0 3598/ff-m87-3C273-096-0254-3C273.0AMTB3.log log 0254 0AMTB3\n", - "1 3598/ff-m87-3C273-096-0518-3C273.0AMTBC.log log 0518 0AMTBC\n", - "2 3598/ff-m87-3C273-096-0122-3C273.0AMTAW.log log 0122 0AMTAW\n", - "3 3598/ff-m87-3C273-096-0554-3C273.0AMTBI.log log 0554 0AMTBI\n", - "4 3598/ff-m87-3C273-096-0158-3C273.0AMTAZ.log log 0158 0AMTAZ\n", - "5 3598/ff-m87-3C273-096-0408-3C273.0AMTB6.log log 0408 0AMTB6\n", - "6 3598/ff-m87-3C273-096-0332-3C273.0AMTB4.log log 0332 0AMTB4\n", - "7 3598/ff-m87-3C273-096-0702-3C273.0AMTBR.log log 0702 0AMTBR\n", - "8 3598/ff-m87-3C273-096-0218-3C273.0AMTB1.log log 0218 0AMTB1\n", - "9 3598/ff-m87-3C273-096-0046-3C273.0AMTAV.log log 0046 0AMTAV\n", - "10 3598/ff-m87-3C273-096-0626-3C273.0AMTBN.log log 0626 0AMTBN\n", - "11 3598/ff-m87-3C273-096-0734-3C273.0AMTBX.log log 0734 0AMTBX\n", - "12 3598/ff-m87-3C273-096-0446-3C273.0AMTBA.log log 0446 0AMTBA\n", - "\n", - "=== {p0}.B.{p1} FMT PARAFRAME ===\n", - " path ext p0 p1\n", - "0 3598/096-0218/XX.B.17.0AMTB1 0AMTB1 XX 17\n", - "1 3598/096-0046/XX.B.17.0AMTAV 0AMTAV XX 17\n", - "2 3598/096-0254/XX.B.17.0AMTB3 0AMTB3 XX 17\n", - "3 3598/096-0218/LP.B.3.0AMTB1 0AMTB1 LP 3\n", - "4 3598/096-0218/XP.B.14.0AMTB1 0AMTB1 XP 14\n", - ".. ... ... .. ..\n", - "570 3598/096-0254/XL.B.14.0AMTB3 0AMTB3 XL 14\n", - "571 3598/096-0254/XL.B.15.0AMTB3 0AMTB3 XL 15\n", - "572 3598/096-0254/XL.B.16.0AMTB3 0AMTB3 XL 16\n", - "573 3598/096-0254/LL.B.11.0AMTB3 0AMTB3 LL 11\n", - "574 3598/096-0254/XL.B.13.0AMTB3 0AMTB3 XL 13\n", - "\n", - "[575 rows x 4 columns]\n", - "\n", - "=== group.uid___A001_X11a7_X37.ec_eht.e17b06-7-hi-m87-3C273-dxin EXTRACT ===\n", - "\n", - "=== e17b06-7-hi.{p0} FMT PARAFRAME ===\n", - " path ext p0\n", - "0 e17b06-7-hi_1034.input input 1034\n", - "1 e17b06-7-hi_1034.machines machines 1034\n", - "2 e17b06-7-hi_1034.threads threads 1034\n", - "3 e17b06-7-hi_1034.flag flag 1034\n", - "4 e17b06-7-hi_1034.calc calc 1034\n", - ".. ... ... ...\n", - "89 e17b06-7-hi_1046.difxlog difxlog 1046\n", - "90 e17b06-7-hi_1046.calc calc 1046\n", - "91 e17b06-7-hi_1046.flag flag 1046\n", - "92 e17b06-7-hi.v2d v2d v2d\n", - "93 e17b06-7-hi.joblist joblist joblist\n", - "\n", - "[94 rows x 3 columns]\n", - "\n", - "\n" - ] - } - ], + "outputs": [], "source": [ "for project_name in all_project_keys:\n", " print(f\"=== {project_name} PROJECT ===\")\n", From bb25cf5f579c0f5e34bc5bffb14f0092c23e5b68 Mon Sep 17 00:00:00 2001 From: YaqingSu Date: Mon, 29 Jun 2026 17:05:33 +0800 Subject: [PATCH 08/10] Uodated documentation for 5 files in mod and created API file in doc. --- doc/api.rst | 22 ++++ doc/conf.py | 4 + doc/index.rst | 1 + mod/hallmark/objects.py | 60 +++++++++++ mod/hallmark/paraframe.py | 198 +++++++++++++++++++++++----------- mod/hallmark/repo_config.py | 116 ++++++++++++++++++++ mod/hallmark/repo_manifest.py | 39 +++++++ mod/hallmark/state.py | 38 ++++--- 8 files changed, 398 insertions(+), 80 deletions(-) create mode 100644 doc/api.rst diff --git a/doc/api.rst b/doc/api.rst new file mode 100644 index 0000000..a3392fa --- /dev/null +++ b/doc/api.rst @@ -0,0 +1,22 @@ +API Reference +============= + +Repository State +---------------- +# Example: This tells Sphinx to import hallmark.repo_state and extract the module +# and function docstrings. + +.. automodule:: hallmark.repo + :members: + +.. automodule:: hallmark.repo_state + :members: + +.. automodule:: hallmark.state + :members: + +.. automodule:: hallmark.downloader + :members: + +.. automodule:: hallmark.paraframe + :members: \ No newline at end of file diff --git a/doc/conf.py b/doc/conf.py index 93b72c4..b15b3b1 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -5,6 +5,10 @@ # -- Project information ----------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information +import os +import sys + +sys.path.insert(0, os.path.abspath("../mod")) project = 'hallmark' copyright = '2025, the Hallmark Authors' diff --git a/doc/index.rst b/doc/index.rst index 3f0a695..d1b0f53 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -11,6 +11,7 @@ design usecase + api Indices and tables diff --git a/mod/hallmark/objects.py b/mod/hallmark/objects.py index 0c5dd30..8e518ea 100644 --- a/mod/hallmark/objects.py +++ b/mod/hallmark/objects.py @@ -1,3 +1,11 @@ +""" +Utilities for storing and restoring content-addressed objects. + +This module defines the ``Objects`` class, which manages the Hallmark +object store. Files are stored using their SHA-1 checksum and can later +be restored to a specified location. +""" + import os import shutil from pathlib import Path @@ -5,13 +13,52 @@ class Objects: + """ + Manage the Hallmark object store. + + Files are stored in a content-addressed directory structure based on + their SHA-1 checksum, allowing them to be efficiently retrieved and + restored. + """ def __init__(self, path: Union[Path, str]): + """ + Initialize the object store. + + Args: + path (Path or str): Path to the Hallmark repository. The object + store is located in the ``objects`` subdirectory. + """ self.root = Path(path) / "objects" def _split_checksum(self, sha1: str) -> Path: + """ + Convert a SHA-1 checksum into its storage path. + + The first two characters of the checksum form the directory name, + while the remaining characters form the filename. + + Args: + sha1 (str): SHA-1 checksum. + + Returns: + Path: Path where the object is stored. + """ return self.root / sha1[:2] / sha1[2:] def store(self, src: Path, sha1: str) -> Path: + """ + Store a file in the object store. + + The file is copied only if an object with the same checksum does not + already exist. + + Args: + src (Path): Source file to store. + sha1 (str): SHA-1 checksum of the file. + + Returns: + Path: Path to the stored object. + """ stored_checksum = self._split_checksum(sha1) if not stored_checksum.exists(): stored_checksum.parent.mkdir(parents=True, exist_ok=True) @@ -19,6 +66,19 @@ def store(self, src: Path, sha1: str) -> Path: return stored_checksum def restore(self, sha1: str, dest: Path) -> Path: + """ + Restore an object from the object store. Raises FileNotFoundError if + the requested object does not exist in the object store. + + The stored object is hard-linked to the destination path. + + Args: + sha1 (str): SHA-1 checksum of the stored object. + dest (Path): Destination path for the restored file. + + Returns: + Path: Path to the restored file. + """ stored = self._split_checksum(sha1) if not stored.exists(): raise FileNotFoundError(f"object {sha1} not found in objects store") diff --git a/mod/hallmark/paraframe.py b/mod/hallmark/paraframe.py index ef50874..0990306 100644 --- a/mod/hallmark/paraframe.py +++ b/mod/hallmark/paraframe.py @@ -12,6 +12,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +""" +Utilities for discovering and parsing parameterized file collections. + +This module defines :class:`ParaFrame`, a subclass of +:class:`pandas.DataFrame` that provides convenient methods for locating, +parsing, and filtering files whose names follow parameterized naming +conventions. +""" from glob import glob from pathlib import Path @@ -26,32 +34,51 @@ class ParaFrame(pd.DataFrame): """ - A subclass of :class:`pandas.DataFrame` with added methods for + A subclass of :class:`pandas.DataFrame` with additional methods for parameterized file discovery and filtering. - ``ParaFrame`` instances behave like ordinary DataFrames but add: - - - * ``__init__``: Initialises the class and stores ``encodings`` and - ``base_path`` as metadata - to the ParaFrame. - * ``_constructor``: Returns the subclassed ParaFrame with repo_path as a - default keyword argument. - * ``glob_search'': Returns files found in the directory using format string. - * ``parse``: a classmethod that builds a table of file paths and parsed - parameters from a format pattern (using ``glob`` + ``parse``). - * ``__call__``/``filter``: convenience filtering by column values. + ``ParaFrame`` behaves like a standard ``pandas.DataFrame`` while + providing the following additional functionality: + + * ``__init__``: Initializes the class and stores ``encodings`` and + ``base_path`` as metadata. + * ``_constructor``: Returns a new ``ParaFrame`` while preserving + ``encodings`` and ``base_path``. + * ``glob_search``: Finds files matching a parameterized format string. + * ``parse``: Builds a ``ParaFrame`` by parsing file paths that match + a format pattern. + * ``__call__`` and ``filter``: Convenience methods for filtering rows + by column values. """ _metadata = ["encodings", "base_path"] def __init__(self, data=None, encodings=None, base_path=None, **kwargs): + """ + Initialize a ``ParaFrame``. + + Args: + data: Data used to initialize the ``ParaFrame``. + encodings (dict, optional): Filename encoding specifications. + Defaults to ``{}``. + base_path (Path or str, optional): Root directory associated with + the ``ParaFrame``. Defaults to the current working directory. + **kwargs: Additional arguments passed to + :class:`pandas.DataFrame`. + """ super().__init__(data, **kwargs) self.encodings = encodings or {} self.base_path = Path(base_path) if base_path is not None else Path.cwd() @property def _constructor(self): + """ + Return the constructor used by pandas operations. + + Returns: + callable: A constructor that preserves ``encodings`` and + ``base_path`` metadata. + """ def _c(*args, **kwargs): kwargs.setdefault("encodings", self.encodings) kwargs.setdefault("base_path", self.base_path) @@ -59,28 +86,40 @@ def _c(*args, **kwargs): return _c def __call__(self, **kwds): + """ + Filter the ``ParaFrame``. + + Equivalent to calling :meth:`filter`. + + Args: + **kwds: Column names and values used for filtering. + + Returns: + pandas.DataFrame: The filtered ``ParaFrame``. + """ return self.filter(**kwds) def filter(self, **kwargs): """ - Filter a pandas ``DataFrame`` by matching column values. + Filter a ``ParaFrame`` by matching column values. - This function utlizes provided **kwargs to filter an existing - ``ParaFrame`` by masking based on column values. Filtering supports - single- and multi-conditioned queries, returning rows that satisfy - any of the provided conditions. + This method filters the current ``ParaFrame`` by applying one or more + conditions on its columns. Rows satisfying any of the specified + conditions are returned. Args: - **kwargs: Arbitrary keyword arguments specifying column names - and values to filter by. - * If the value is a tuple or list, rows where the column - matches any of those values are selected. - * If the value is a scalar, rows where the column equals - the value are selected. + **kwargs: Keyword arguments specifying column names and values to + filter by. + + - If a value is a tuple or list, rows matching any of those + values are selected. + - If a value is a scalar, rows whose column equals that value + are selected. Returns: - pandas.DataFrame: A filtered DataFrame containing only rows - that match the given conditions. + pandas.DataFrame: + A filtered DataFrame containing only rows that satisfy the + requested conditions. """ mask = np.zeros(len(self), dtype=bool) for k, v in kwargs.items(): @@ -103,38 +142,53 @@ def glob_search( **kwargs, ): """ - Find all the files specified in a directory using the format string specified. + Find files matching a parameterized format string. - This function utilizes the provided format string to find the specified - files. This function also looks through .yaml files if encoding = True - and evaluates conditionals. + This method searches for files using the supplied format string. + When ``encoding=True``, regular-expression encodings defined in the + YAML configuration are also applied. Args: - fmt (str): A format string specifying the expected file naming - pattern. - Fields wrapped in ``{}`` will be extracted into columns. - *args: Positional arguments used to fill the format string. - encodings (dict): The ``encodings`` list from ``State`` - (contents of ``config.yml``). - Defaults to ``{}``. - base_path (Path): Root directory to search from. - Defaults to ``Path.cwd()``. - debug (bool, optional): If True, prints debugging information - about the matching process. - Defaults to False. - return_pattern (bool, optional): Returns the pattern and globbed files. - Defaults to False. - encodings (bool,optional): If True, looks for the .yaml file and - extracts user specified format information. - Defaults to False. - **kwargs: Keyword arguments used to fill the format string. - If missing keys are encountered, they will be replaced by - a wildcard ``*`` for globbing. + fmt (str): + Format string describing the expected filename pattern. + Fields enclosed in ``{}`` are extracted as parameters. + + *args: + Positional arguments used to fill the format string. + + encodings (dict): + Encoding specifications from ``config.yml``. + Defaults to ``{}``. + + base_path (Path): + Root directory to search. + Defaults to ``Path.cwd()``. + + debug (bool): + If ``True``, prints debugging information. + Defaults to ``False``. + + return_pattern (bool): + If ``True``, returns both the glob pattern and the matched + files. + Defaults to ``False``. + + encoding (bool): + If ``True``, applies regex encodings defined in the YAML + configuration. + Defaults to ``False``. + + **kwargs: + Keyword arguments used to fill the format string. + Missing values are replaced with the wildcard ``*``. Returns: - if return_pattern = True, it returns the pattern and the globbed files. - Else, it returns the globbed files, the format string with the wildcards - and the user specification in the .yaml file (None, if encoding = False). + tuple: + If ``return_pattern`` is ``True``, returns + ``(globbed_files, pattern)``. + + Otherwise returns + ``(yaml_encodings, fmt_g, globbed_files)``. """ encodings = encodings or {} base_path = Path(base_path) if base_path is not None else Path.cwd() @@ -221,27 +275,41 @@ def parse( encoding=False, **kwargs, ): - """Build a ``ParaFrame`` by parsing file paths that match a pattern. + """ + Build a ``ParaFrame`` by parsing file paths that match a format + string. Args: - fmt (str): Format string with ``{param}`` fields. - encodings (dict): The ``encodings`` dict from ``State`` - (contents of ``hallmark.yml``). - Defaults to ``{}``. - base_path (Path): Root directory to search from. - Defaults to ``Path.cwd()``. - debug (bool): Print debug info. Defaults to ``False``. - encoding (bool): Apply regex encoding. Defaults to ``False``. + fmt (str): + Format string containing ``{parameter}`` fields. + + encodings (dict): + Encoding specifications from ``config.yml``. + Defaults to ``{}``. + + base_path (Path): + Root directory to search. + Defaults to ``Path.cwd()``. + + debug (bool): + If ``True``, prints debugging information. + Defaults to ``False``. + + encoding (bool): + If ``True``, applies regex encoding. + Defaults to ``False``. Returns: - ``ParaFrame`` where each row is a matched file with parsed - parameters as columns, plus a ``path`` column. + ParaFrame: + A ``ParaFrame`` whose rows correspond to matched files. + Parsed parameters are stored as columns together with a + ``path`` column. Example: >>> from hallmark import ParaFrame >>> pf = ParaFrame.parse( ... "/custom_parameter{custom_parameter}_p{parameter}.h5", - ... encoding=True + ... encoding=True, ... ) """ base_path = Path(base_path) if base_path is not None else Path.cwd() diff --git a/mod/hallmark/repo_config.py b/mod/hallmark/repo_config.py index dea4757..a94981e 100644 --- a/mod/hallmark/repo_config.py +++ b/mod/hallmark/repo_config.py @@ -1,3 +1,10 @@ +""" +Utilities for managing Hallmark repository configuration. + +This module provides helper functions for reading, validating, and +updating repository configuration values stored in ``config.yml``. +""" + from __future__ import annotations from string import Formatter @@ -6,6 +13,18 @@ #test def ensure_branch_data_spec(config: dict) -> dict: + """ + Ensure the configuration contains a valid data specification. + + If the ``data`` entry is missing or malformed, it is initialized with + a single empty dictionary. + + Args: + config (dict): Repository configuration. + + Returns: + dict: The branch data specification. + """ data = config.get("data") if not isinstance(data, list) or len(data) != 1 or not isinstance(data[0], dict): config["data"] = [{}] @@ -13,6 +32,16 @@ def ensure_branch_data_spec(config: dict) -> dict: def branch_data_spec(repo) -> dict: + """ + Return the branch data specification. Raises RuntimeError if + the configuration does not define exactly one entry under ``data``. + + Args: + repo: Repository object. + + Returns: + dict: The branch data specification. + """ data = repo.state.config.get("data") if not isinstance(data, list) or len(data) != 1 or not isinstance(data[0], dict): raise RuntimeError('branch config must define exactly ' \ @@ -21,6 +50,16 @@ def branch_data_spec(repo) -> dict: def branch_fmt(repo) -> str: + """ + Return the configured filename format. Raises RuntimeError if + no valid format string is defined. + + Args: + repo: Repository object. + + Returns: + str: The format string stored in ``data[0].fmt``. + """ fmt = branch_data_spec(repo).get("fmt") if not isinstance(fmt, str) or not fmt.strip(): raise RuntimeError('branch config must define one ' \ @@ -29,6 +68,13 @@ def branch_fmt(repo) -> str: def set_branch_fmt(repo, fmt: str) -> None: + """ + Set the branch filename format. + + Args: + repo: Repository object. + fmt (str): New filename format. + """ set_config(repo, fmt=fmt) @@ -40,6 +86,23 @@ def set_config( remote_url: Optional[str] = None, encoding_updates: Optional[Dict[str, str]] = None, ) -> dict: + """ + Update the repository configuration. + + Existing configuration values are preserved unless explicitly + replaced. + + Args: + repo: Repository object. + fmt (str, optional): Filename format. + remote_name (str, optional): Remote repository name. + remote_url (str, optional): Remote repository URL. + encoding_updates (dict, optional): Encoding values to merge into + the existing configuration. + + Returns: + dict: The updated configuration. + """ config = repo.state.config spec = ensure_branch_data_spec(config) @@ -88,11 +151,30 @@ def set_config( def branch_encodings(repo) -> list[dict]: + """ + Return the configured filename encodings. + + Args: + repo: Repository object. + + Returns: + list[dict]: A list containing the encoding specification, or an + empty list if no encodings are defined. + """ spec = branch_data_spec(repo) return [spec] if isinstance(spec.get("encoding"), dict) else [] def fmt_fields(fmt: str) -> list[str]: + """ + Extract field names from a format string. + + Args: + fmt (str): Format string containing replacement fields. + + Returns: + list[str]: Unique field names in the order they appear. + """ fields: list[str] = [] for _, field_name, _, _ in Formatter().parse(fmt): if field_name and field_name not in fields: @@ -101,6 +183,16 @@ def fmt_fields(fmt: str) -> list[str]: def coerce_fmt_value(value: str, spec: str): + """ + Convert a value according to a format specification. + + Args: + value (str): Value to convert. + spec (str): Format specification. + + Returns: + The converted value. + """ if not spec: return value if spec.endswith("d"): @@ -111,6 +203,16 @@ def coerce_fmt_value(value: str, spec: str): def row_to_path(row, fmt: str) -> Path: + """ + Construct a file path from a table row. + + Args: + row: Table row containing field values. + fmt (str): Format string used to build the path. + + Returns: + Path: Path generated from the row values. + """ values = {} for _, field_name, format_spec, _ in Formatter().parse(fmt): if field_name: @@ -119,4 +221,18 @@ def row_to_path(row, fmt: str) -> Path: def path_from_row(repo, row, fmt: Optional[str] = None) -> Path: + """ + Construct a file path from a table row. + + If no format string is provided, the repository's configured format + is used. + + Args: + repo: Repository object. + row: Table row containing field values. + fmt (str, optional): Format string to use. + + Returns: + Path: Path generated from the row values. + """ return row_to_path(row, fmt or branch_fmt(repo)) diff --git a/mod/hallmark/repo_manifest.py b/mod/hallmark/repo_manifest.py index 72914e7..1edde50 100644 --- a/mod/hallmark/repo_manifest.py +++ b/mod/hallmark/repo_manifest.py @@ -9,6 +9,21 @@ def manifest_frame_from_pf(pf, fmt: str) -> pd.DataFrame: + """ + Build a manifest table from a ``ParaFrame``. Raises RuntimeError + if a file path cannot be parsed using ``fmt``. + + The returned table contains a ``sha1`` column together with the + fields extracted from the configured filename format. + + Args: + pf: ``ParaFrame`` containing indexed file paths. + fmt (str): Filename format used to parse the paths. + + Returns: + pandas.DataFrame: Manifest table containing ``sha1`` values and + parsed filename fields. + """ if pf.empty: return pd.DataFrame(columns=["sha1", *fmt_fields(fmt)]) @@ -25,6 +40,16 @@ def manifest_frame_from_pf(pf, fmt: str) -> pd.DataFrame: def manifest_map(state) -> dict[str, str]: + """ + Create a mapping from file paths to SHA-1 checksums. + + Args: + state: Repository state. + + Returns: + dict[str, str]: Dictionary mapping relative file paths to their + corresponding SHA-1 checksums. + """ if state.data.empty: return {} data = state.config.get("data") @@ -40,6 +65,20 @@ def manifest_map(state) -> dict[str, str]: def frame_from_paths(repo, rel_paths: list[Path]): + """ + Build a ``ParaFrame`` from repository paths. + + Each path is converted into a row containing the relative path and + its SHA-1 checksum. + + Args: + repo: Repository object. + rel_paths (list[Path]): Relative paths within the repository. + + Returns: + ParaFrame: ``ParaFrame`` containing the indexed paths and their + checksums. + """ records = [{"path": str(path)} for path in rel_paths] pf = repo.paraframe_cls(records, base_path=repo.worktree) if not pf.empty: diff --git a/mod/hallmark/state.py b/mod/hallmark/state.py index 2884a83..73e23bc 100644 --- a/mod/hallmark/state.py +++ b/mod/hallmark/state.py @@ -23,13 +23,14 @@ @dataclass class State: - """In-memory hallmark state database. + """ + In-memory Hallmark state database. Attributes: config: Repository configuration values. - meta: Metadata. - data: Tabular file index; each row is keyed by path/change and - stores the indexed object checksum (``sha1``). + meta: Repository metadata. + data: Tabular file index containing indexed object checksums + (``sha1``) and associated metadata. """ config: dict = field(default_factory=dict) @@ -39,14 +40,18 @@ class State: ) def update(self, pf): - ''' - Marge paraframe rows into the state database. + """ + Merge ``ParaFrame`` rows into the state database. + + Existing rows with matching keys are updated, while new rows are + appended. Args: - pf (paraframe): paraframe containing rows to add or update + pf (ParaFrame): ``ParaFrame`` containing rows to add or update. + Returns: - none. - ''' + None. + """ if pf.empty: incoming = pd.DataFrame(columns=self.data.columns if len(self.data.columns) else COLUMNS) @@ -59,11 +64,12 @@ def update(self, pf): for column in incoming.columns: if column != "sha1": incoming[column] = incoming[column].astype(str) - + # Merge the incoming rows with the existing state. merged = pd.concat([self.data, incoming], ignore_index=True, sort=False) key_columns = [column for column in merged.columns if column != "sha1"] if key_columns: + # Remove duplicate entries while keeping the most recent row. deduped = merged.drop_duplicates(subset=key_columns, keep="last") else: deduped = merged @@ -71,16 +77,18 @@ def update(self, pf): self.data = deduped.loc[:, ["sha1", *key_columns]] def replace(self, pf): - ''' - Merge paraframe rows into the tsate database. - Existing rows with matching keys are replaced by the incoming rows. + """ + Replace the contents of the state database. + + Existing rows are discarded and replaced with the rows from the + provided ``ParaFrame``. Args: - pf (paraframe): Paraframe containing rows to add or update. + pf (ParaFrame): ``ParaFrame`` containing the replacement rows. Returns: None. - ''' + """ if pf.empty: self.data = pd.DataFrame(columns=self.data.columns if len(self.data.columns) else COLUMNS) From b819ef6912367fabbf7e095b410196b9c2f10dd2 Mon Sep 17 00:00:00 2001 From: YaqingSu Date: Mon, 29 Jun 2026 17:42:29 +0800 Subject: [PATCH 09/10] eliminated sphinx warnings --- mod/hallmark/paraframe.py | 38 +++++++++--------------------------- mod/hallmark/repo.py | 41 ++++++++++++++++++++++----------------- mod/hallmark/state.py | 4 ++-- 3 files changed, 34 insertions(+), 49 deletions(-) diff --git a/mod/hallmark/paraframe.py b/mod/hallmark/paraframe.py index ed67f83..915c397 100644 --- a/mod/hallmark/paraframe.py +++ b/mod/hallmark/paraframe.py @@ -108,18 +108,13 @@ def filter(self, **kwargs): conditions are returned. Args: - **kwargs: Keyword arguments specifying column names and values to - filter by. - - - If a value is a tuple or list, rows matching any of those - values are selected. - - If a value is a scalar, rows whose column equals that value - are selected. + ``**kwargs``: Keyword arguments specifying column names and values to + filter by. Values may be scalars or sequences (lists or tuples). + Sequence values match any element in the sequence. Returns: - pandas.DataFrame: - A filtered DataFrame containing only rows that satisfy the - requested conditions. + pandas.DataFrame: A filtered DataFrame containing only the rows + that satisfy the requested conditions. """ mask = np.zeros(len(self), dtype=bool) for k, v in kwargs.items(): @@ -151,34 +146,28 @@ def glob_search( Args: fmt (str): Format string describing the expected filename pattern. - Fields enclosed in ``{}`` are extracted as parameters. - *args: + ``*args``: Positional arguments used to fill the format string. encodings (dict): Encoding specifications from ``config.yml``. - Defaults to ``{}``. base_path (Path): Root directory to search. - Defaults to ``Path.cwd()``. debug (bool): If ``True``, prints debugging information. - Defaults to ``False``. return_pattern (bool): If ``True``, returns both the glob pattern and the matched files. - Defaults to ``False``. encoding (bool): If ``True``, applies regex encodings defined in the YAML configuration. - Defaults to ``False``. - **kwargs: + ``**kwargs``: Keyword arguments used to fill the format string. Missing values are replaced with the wildcard ``*``. @@ -300,17 +289,8 @@ def parse( Defaults to ``False``. Returns: - ParaFrame: - A ``ParaFrame`` whose rows correspond to matched files. - Parsed parameters are stored as columns together with a - ``path`` column. - - Example: - >>> from hallmark import ParaFrame - >>> pf = ParaFrame.parse( - ... "/custom_parameter{custom_parameter}_p{parameter}.h5", - ... encoding=True, - ... ) + ParaFrame: A ``ParaFrame`` whose rows correspond to matched files. + Parsed parameters are stored as columns together with a ``path`` column. """ base_path = Path(base_path) if base_path is not None else Path.cwd() diff --git a/mod/hallmark/repo.py b/mod/hallmark/repo.py index 4358924..48c4c7d 100644 --- a/mod/hallmark/repo.py +++ b/mod/hallmark/repo.py @@ -56,7 +56,8 @@ def chdir(path): @dataclass(init=False) class Repo: - """Hallmark repository. + """ + Hallmark repository. This is the Python API boundary. It loads the in-memory ``State`` from repository ``Dothm``, and @@ -148,8 +149,7 @@ def clone( url(string): remote repository URL. path(path|string): destination path for clone. fetch_data (boolean): if true, downloads associated data files. - max_workers (integer): Number of parallel workers for - downloading data. + max_workers (integer): Number of parallel workers for downloading data. show_progress (boolean): wether to display download progress. Returns: Repo: Cloned Repository instance @@ -199,8 +199,9 @@ def clone( def checksum(path: Path, chunk_size: int = 1024 * 1024) -> str: ''' Computes the SHA1 checksum of a file. + Args: - path (path):path to the file to hash. + path (path): path to the file to hash. chunk_size (integer): size of chunks used for streaming reads. Returns: String: Hexadecimal SHA1 digest of the file contents. @@ -228,16 +229,18 @@ def set_config( remote_url: Optional[str] = None, encoding_updates: Optional[Dict[str, str]] = None, ) -> dict: - ''' + """ Update repository configuration values. - Args: - fmt (string|None): Data dormat specification. - remote_name (string|None): Name of the remote repository. - remote_url (string|None): URL of the remote repository. - encoding_updates (dictioary[str,str] | None): updates to encoding rules. + + Args: + fmt (str, optional): Data format specification. + remote_name (str, optional): Name of the remote repository. + remote_url (str, optional): URL of the remote repository. + encoding_updates (dict[str, str], optional): Updates to encoding rules. + Returns: - dictionary: Updated configuration dictionary - ''' + dict: Updated configuration dictionary. + """ repo_set_config( self, fmt=fmt, @@ -253,7 +256,8 @@ def status(self) -> dict[str, object]: Return repository status information. Includes staged changes, orktree modifications, deletions, and untracked files. - Args: none? + Args: + none? Returns: dict[str, object]: Status summary including: @@ -384,8 +388,8 @@ def commit(self, msg: str, allow_empty: bool = False) -> bool: message is empty or invalid. Args: - msg (string): commit message. - allow_empty (boolean): Allow comitting even if no changes exists. + msg (string): commit message. + allow_empty (boolean): Allow comitting even if no changes exists. Returns: boolean: True if a commit was created, false otherwise. ''' @@ -405,7 +409,7 @@ def log(self) -> str: Return commit history log. Returns: - string: Git log output, or an empty string if no valid HEAD exists. + string: Git log output, or an empty string if no valid HEAD exists. ''' if not self.dothm.head.is_valid(): return "" @@ -414,10 +418,11 @@ def log(self) -> str: def branches(self) -> dict[str, object]: ''' List repository branches. + Returns: dictionary[string, object]: Dictionary containing: - - current (string): Active branch name - - names (list[string]): All branch names + - ``current`` (string): Active branch name + - ``names``: All branch names ''' current = self.dothm.active_branch.name names = sorted(head.name for head in self.dothm.heads) diff --git a/mod/hallmark/state.py b/mod/hallmark/state.py index 73e23bc..8701115 100644 --- a/mod/hallmark/state.py +++ b/mod/hallmark/state.py @@ -4,7 +4,7 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -30,7 +30,7 @@ class State: config: Repository configuration values. meta: Repository metadata. data: Tabular file index containing indexed object checksums - (``sha1``) and associated metadata. + (``sha1``) and associated metadata. """ config: dict = field(default_factory=dict) From e4dc5971a47223cf3cd098bd07db542d545947e5 Mon Sep 17 00:00:00 2001 From: YaqingSu Date: Wed, 1 Jul 2026 13:48:25 +0800 Subject: [PATCH 10/10] fixed length error --- doc/api.rst | 2 +- mod/hallmark/cli.py | 3 ++- mod/hallmark/dothm.py | 9 ++++++--- mod/hallmark/downloader.py | 3 ++- mod/hallmark/repo.py | 3 ++- 5 files changed, 13 insertions(+), 7 deletions(-) diff --git a/doc/api.rst b/doc/api.rst index a3392fa..1e09ba6 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -3,7 +3,7 @@ API Reference Repository State ---------------- -# Example: This tells Sphinx to import hallmark.repo_state and extract the module +# This tells Sphinx to import hallmark.filename and extract the module # and function docstrings. .. automodule:: hallmark.repo diff --git a/mod/hallmark/cli.py b/mod/hallmark/cli.py index 02c3b54..74b0433 100644 --- a/mod/hallmark/cli.py +++ b/mod/hallmark/cli.py @@ -238,7 +238,8 @@ def checkout(repo, target_branch): try: if repo.checkout(target_branch): click.echo(f'Switched to branch "{target_branch}".') - except (GitError, RuntimeError, ValueError, FileNotFoundError, CheckoutError) as e: + except (GitError, RuntimeError, ValueError, FileNotFoundError, + CheckoutError) as e: raise ClickException(str(e)) diff --git a/mod/hallmark/dothm.py b/mod/hallmark/dothm.py index 277acbb..cb9d7e0 100644 --- a/mod/hallmark/dothm.py +++ b/mod/hallmark/dothm.py @@ -42,12 +42,14 @@ def path(self) -> Path: def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) if self.working_tree_dir is None: - raise DothmError('The ".hm" directory must be a valid git worktree.') + raise DothmError('The ".hm" directory must be a valid git ' \ + 'worktree.') @classmethod def init(cls, *args, **kwargs) -> "Dothm": if kwargs.get('bare', False): - raise DothmError('A ".hm" directory must not be a bare git repository') + raise DothmError('A ".hm" directory must not be a bare git ' \ + 'repository') kwargs.setdefault("initial_branch", "main") dothm = super().init(*args, **kwargs) @@ -139,7 +141,8 @@ def dump_yml(self, data: dict, stem: Union[Path, str]) -> None: yaml.dump(data, f, sort_keys=False) def load_tsv(self, stem: Union[Path, str]) -> pd.DataFrame: - return pd.read_csv((self.path/stem).with_suffix(".tsv"), sep="\t", dtype=str) + return pd.read_csv((self.path/stem).with_suffix(".tsv"), sep="\t", + dtype=str) def dump_tsv(self, data: pd.DataFrame, stem: Union[Path, str]) -> None: data.to_csv((self.path/stem).with_suffix(".tsv"), sep="\t", index=False) diff --git a/mod/hallmark/downloader.py b/mod/hallmark/downloader.py index a61b9f9..3149d2a 100644 --- a/mod/hallmark/downloader.py +++ b/mod/hallmark/downloader.py @@ -146,7 +146,8 @@ def download_remote_data( try: with ThreadPoolExecutor(max_workers=max_workers) as executor: futures = { - executor.submit(_download_file, url, destination, sha1): destination + executor.submit(_download_file, + url, destination, sha1): destination for url, destination, sha1 in files_to_download } for future in as_completed(futures): diff --git a/mod/hallmark/repo.py b/mod/hallmark/repo.py index 48c4c7d..f333932 100644 --- a/mod/hallmark/repo.py +++ b/mod/hallmark/repo.py @@ -500,7 +500,8 @@ def checkout(self, target_branch: str) -> bool: def add_worktree(self, target_branch: str) -> bool: ''' - Create or link a new worktree for a branch. Raises ValueError if branch name is invalid. + Create or link a new worktree for a branch. Raises ValueError if branch name + is invalid. Rasies Runtime error if called in a bare repository or worktree creation fails. Args: