OSF backed data

README.md

@@ -8,14 +8,19 @@ This repository contains benchmarks for comparing models of object learning agai
   <img src="site/readme_images/human_learning_curves.svg" alt="Alt text" >
 </div>
 
+If you just want to download the raw data and images without using the `hobj` library, check out the [OSF repository](https://osf.io/pj6wm/files/osfstorage) for this project.
 
 ## Quickstart
 
 ### Install
 
 The `hobj` package works for Python >=3.12. After cloning this repository on your machine, navigate to this directory in your shell and run:
 
-# todo
+On first use, the packaged dataset is downloaded automatically from the OSF
+repository into a versioned cache directory under `~/.hobj_cache`. The default
+path is `~/.hobj_cache/pj6wm-v1/data`. In total, `hobj` takes up around ~1 GB
+of space on your computer.
+
 
 ### Using `hobj` to comparing a linear learner against human learning data 
 
@@ -55,19 +60,23 @@ print(result.msen, result.msen_CI95)
 # print(result.model_statistics)
 ```
 
-Note that on first use, the packaged dataset is downloaded automatically into `./data`.
+
+For more details (e.g., how to load the raw behavioral data or images in Python), check out the Jupyter notebooks in `examples/`.
+
 To use a different location, pass `cachedir=...` to a data loader or benchmark
 constructor, or prefetch manually with `hobj-download-data --cachedir /path/to/data`.
 
-For more details (e.g., how to load the raw behavioral data or images), check out the Jupyter notebooks in `examples/`.
+## Contact 
+If you have any questions, need help, or experience a bug, please don't hesitate to email me ([mil@mit.edu](mailto:name@example.com)), or open an issue on this repo!
 
-### Need help or have questions?
 
-Please don't hesitate to email me ([mil@mit.edu](mailto:name@example.com)), or open an issue on this repo!
 
-## Citation
+## Changes to codebase since publication
+This codebase was overhauled in 2026 to improve its accessibility, performance, and quality. Along the way, minor changes to the statistical analysis procedure were introduced, along with changes to the names of the original filenames (see [changelist](site/changelist.md)). To see the codebase at the time of publication, check out the repo with the `v1` tag [here](https://github.com/himjl/hobj/releases/tag/v1).
 
 
+## Citation
+
 ```
 @article{lee2023well,
   title={How well do rudimentary plasticity rules predict adult visual object learning?},
@@ -80,7 +89,3 @@ Please don't hesitate to email me ([mil@mit.edu](mailto:name@example.com)), or o
   publisher={Public Library of Science San Francisco, CA USA}
 }
 ```
-
-
-## Changes to codebase since publication
-This codebase was refactored in 2026 to improve the accessibility, performance, and quality of the code. Along the way, minor changes to the statistical analysis of the original codebase were introduced (see [changelist](site/changelist.md)). To see the codebase at the time of publication, check out the repo with the `v1` tag [here](https://github.com/himjl/hobj/releases/tag/v1).

hobj/data/download.py

@@ -5,6 +5,7 @@
 import argparse
 import shutil
 import tarfile
+import zipfile
 from pathlib import Path
 from tempfile import TemporaryDirectory
 from urllib.parse import urlparse
@@ -13,9 +14,12 @@
 from tqdm import tqdm
 
 
+# Public OSF node that hosts the packaged HOBJ dataset files.
+OSF_NODE_ID = "pj6wm"
+# Bump this to invalidate the local cache when the packaged OSF dataset changes.
+DATASET_CACHE_VERSION = "v2"
 DATA_ARCHIVE_URL = (
-    "https://hlbdatasets.s3.us-east-1.amazonaws.com/"
-    "lee-dicarlo-2023-learning-data.tar.gz"
+    f"https://files.osf.io/v1/resources/{OSF_NODE_ID}/providers/osfstorage/?zip="
 )
 EXPECTED_DATA_RELATIVE_PATHS = (
     Path("meta-MutatorHighVarImageset.csv"),
@@ -28,17 +32,23 @@
 _DOWNLOAD_CHUNK_SIZE_BYTES = 1024 * 1024
 
 
-def _get_repo_root() -> Path:
-    """Return the repository root for this package."""
-    return Path(__file__).resolve().parents[2]
+def _get_default_data_root(repo_root: Path | None = None) -> Path:
+    """Return the default packaged data directory.
 
+    Args:
+        repo_root: Unused legacy parameter kept for compatibility with the
+            public function signature.
 
-def _get_default_data_root(repo_root: Path | None = None) -> Path:
-    """Return the default packaged data directory."""
-    resolved_repo_root = (
-        repo_root if repo_root is not None else _get_repo_root()
-    ).resolve()
-    return resolved_repo_root / "data"
+    Returns:
+        A persistent per-user cache directory outside the installed package.
+    """
+    del repo_root
+    return (
+        Path.home().resolve()
+        / ".hobj_cache"
+        / f"{OSF_NODE_ID}-{DATASET_CACHE_VERSION}"
+        / "data"
+    )
 
 
 def _get_missing_expected_paths(data_root: Path) -> list[Path]:
@@ -52,7 +62,16 @@ def _get_missing_expected_paths(data_root: Path) -> list[Path]:
 
 def _derive_archive_path(repo_root: Path, url: str) -> Path:
     """Return the on-disk path for the downloaded archive."""
-    archive_name = Path(urlparse(url).path).name
+    parsed_url = urlparse(url)
+    path_segments = [segment for segment in parsed_url.path.split("/") if segment]
+    if (
+        parsed_url.query == "zip="
+        and len(path_segments) >= 4
+        and path_segments[:3] == ["v1", "resources", path_segments[2]]
+    ):
+        archive_name = f"{path_segments[2]}.zip"
+    else:
+        archive_name = Path(parsed_url.path).name
     if not archive_name:
         raise ValueError(f"Could not derive archive filename from URL: {url}")
     return repo_root / archive_name
@@ -95,6 +114,25 @@ def _safe_extract_tarball(archive_path: Path, destination: Path) -> None:
         tar.extractall(destination, filter="data")
 
 
+def _safe_extract_zipfile(archive_path: Path, destination: Path) -> None:
+    """Extract a zipfile while preventing path traversal.
+
+    Args:
+        archive_path: Zipfile to extract.
+        destination: Destination directory for extracted contents.
+
+    Raises:
+        ValueError: If a zip entry would escape the extraction directory.
+    """
+    destination = destination.resolve()
+
+    with zipfile.ZipFile(archive_path) as archive:
+        for member in archive.infolist():
+            member_path = (destination / member.filename).resolve()
+            member_path.relative_to(destination)
+        archive.extractall(destination)
+
+
 def _find_extracted_data_root(extraction_root: Path) -> Path:
     """Return the packaged data directory from an extracted archive tree.
 
@@ -120,6 +158,26 @@ def _find_extracted_data_root(extraction_root: Path) -> Path:
     )
 
 
+def _extract_nested_images_archive(data_root: Path) -> None:
+    """Extract ``images.tar.gz`` into ``data/images`` when present.
+
+    Args:
+        data_root: Extracted packaged data directory.
+    """
+    images_archive_path = data_root / "images.tar.gz"
+    images_root = data_root / "images"
+    if images_root.exists():
+        if images_archive_path.exists():
+            images_archive_path.unlink()
+        return
+
+    if not images_archive_path.exists():
+        return
+
+    _safe_extract_tarball(archive_path=images_archive_path, destination=data_root)
+    images_archive_path.unlink()
+
+
 def resolve_data_root(
     *,
     cachedir: Path | None = None,
@@ -141,6 +199,7 @@ def resolve_data_root(
     ).resolve()
     if _get_missing_expected_paths(data_root):
         return download_data(cachedir=data_root, url=DATA_ARCHIVE_URL)
+    _extract_nested_images_archive(data_root)
     return data_root
 
 
@@ -171,7 +230,8 @@ def download_data(
 
     Raises:
         requests.HTTPError: If the archive download fails.
-        tarfile.TarError: If the archive cannot be unpacked.
+        tarfile.TarError: If a tar archive cannot be unpacked.
+        zipfile.BadZipFile: If the zip archive cannot be unpacked.
         ValueError: If the archive path is invalid or extraction would escape
             the extraction directory.
         FileNotFoundError: If extraction completes but the expected dataset
@@ -192,8 +252,9 @@ def download_data(
 
     with TemporaryDirectory(dir=archive_root) as extraction_root_str:
         extraction_root = Path(extraction_root_str)
-        _safe_extract_tarball(archive_path=archive_path, destination=extraction_root)
+        _safe_extract_zipfile(archive_path=archive_path, destination=extraction_root)
         extracted_data_root = _find_extracted_data_root(extraction_root)
+        _extract_nested_images_archive(extracted_data_root)
         shutil.copytree(extracted_data_root, data_root, dirs_exist_ok=True)
 
     missing_paths = _get_missing_expected_paths(data_root)

pyproject.toml

@@ -25,6 +25,9 @@ dependencies = [
     "scipy>=1.17.1",
 ]
 
+[project.scripts]
+hobj-download-data = "hobj.data.download:main"
+
 [dependency-groups]
 dev = [
     "enczoo==0.1.3.dev1",

tests/test_data_loader_cache.py

@@ -1,4 +1,5 @@
 import tarfile
+import zipfile
 from pathlib import Path
 
 import pandas as pd
@@ -125,23 +126,46 @@ def _write_minimal_packaged_dataset(data_root: Path) -> None:
     ).to_csv(behavior_root / "human-behavior-oneshot-subtasks.csv", index=False)
 
 
+def _write_packaged_images_archive(data_root: Path) -> None:
+    """Write the nested ``images.tar.gz`` archive expected by OSF downloads.
+
+    Args:
+        data_root: Packaged data directory containing metadata and behavior CSVs.
+    """
+    staging_root = data_root.parent / "images-staging"
+    images_root = staging_root / "images"
+    images_root.mkdir(parents=True, exist_ok=True)
+
+    Image.new("RGB", (2, 3), color=(255, 0, 0)).save(images_root / "highvar.png")
+    Image.new("RGB", (2, 3), color=(0, 255, 0)).save(images_root / "oneshot.png")
+    Image.new("RGB", (2, 3), color=(0, 0, 255)).save(images_root / "warmup.png")
+    Image.new("RGB", (2, 3), color=(255, 255, 0)).save(images_root / "catch.png")
+
+    with tarfile.open(data_root / "images.tar.gz", mode="w:gz") as archive:
+        archive.add(images_root, arcname="images")
+
+
 def test_download_data_extracts_into_custom_cachedir(tmp_path: Path) -> None:
     staging_root = tmp_path / "staging" / "data"
     _write_minimal_packaged_dataset(staging_root)
+    _write_packaged_images_archive(staging_root)
 
-    archive_path = tmp_path / "fixture.tar.gz"
-    with tarfile.open(archive_path, mode="w:gz") as archive:
-        archive.add(staging_root, arcname="data")
+    archive_path = tmp_path / "download"
+    with zipfile.ZipFile(archive_path, mode="w") as archive:
+        for path in sorted(staging_root.rglob("*")):
+            archive.write(path, arcname=Path("data") / path.relative_to(staging_root))
 
     custom_cache = tmp_path / "custom-cache"
     resolved_cache = download_module.download_data(
-        url="https://example.com/fixture.tar.gz",
+        url="https://example.com/download?zip=",
         cachedir=custom_cache,
     )
 
     assert resolved_cache == custom_cache.resolve()
     assert (custom_cache / "meta-MutatorHighVarImageset.csv").exists()
     assert (custom_cache / "behavior" / "human-behavior-highvar-subtasks.csv").exists()
+    assert (custom_cache / "images" / "highvar.png").exists()
+    assert not (custom_cache / "images.tar.gz").exists()
     assert not (tmp_path / "data").exists()
 
 
@@ -192,10 +216,7 @@ def test_load_highvar_behavior_uses_custom_cachedir(tmp_path: Path) -> None:
 def test_load_image_uses_custom_cachedir(tmp_path: Path) -> None:
     custom_cache = tmp_path / "image-cache"
     _write_minimal_packaged_dataset(custom_cache)
-
-    image_path = custom_cache / "images" / "highvar.png"
-    image_path.parent.mkdir(parents=True, exist_ok=True)
-    Image.new("RGB", (2, 3), color=(255, 0, 0)).save(image_path)
+    _write_packaged_images_archive(custom_cache)
 
     image = load_image("img-highvar", cachedir=custom_cache)
 
@@ -206,11 +227,36 @@ def test_load_image_uses_custom_cachedir(tmp_path: Path) -> None:
 def test_get_image_path_uses_custom_cachedir(tmp_path: Path) -> None:
     custom_cache = tmp_path / "image-path-cache"
     _write_minimal_packaged_dataset(custom_cache)
-
-    image_path = custom_cache / "images" / "highvar.png"
-    image_path.parent.mkdir(parents=True, exist_ok=True)
-    Image.new("RGB", (2, 3), color=(255, 0, 0)).save(image_path)
+    _write_packaged_images_archive(custom_cache)
 
     resolved_path = get_image_path("img-highvar", cachedir=custom_cache)
 
-    assert resolved_path == image_path
+    assert resolved_path == custom_cache / "images" / "highvar.png"
+
+
+def test_resolve_data_root_extracts_nested_images_archive_without_redownload(
+    tmp_path: Path,
+) -> None:
+    custom_cache = tmp_path / "resolved-cache"
+    _write_minimal_packaged_dataset(custom_cache)
+    _write_packaged_images_archive(custom_cache)
+
+    resolved_cache = download_module.resolve_data_root(cachedir=custom_cache)
+
+    assert resolved_cache == custom_cache.resolve()
+    assert (custom_cache / "images" / "highvar.png").exists()
+    assert not (custom_cache / "images.tar.gz").exists()
+
+
+def test_get_default_data_root_uses_versioned_home_cache(monkeypatch) -> None:
+    monkeypatch.setattr(download_module.Path, "home", lambda: Path("/tmp/fake-home"))
+
+    resolved_path = download_module._get_default_data_root()
+
+    assert (
+        resolved_path
+        == Path(
+            f"/tmp/fake-home/.hobj_cache/{download_module.OSF_NODE_ID}-"
+            f"{download_module.DATASET_CACHE_VERSION}/data"
+        ).resolve()
+    )