[1488] Support frequency nominal dim in consolidate functions

.ci_helpers/docker/docker-compose.yaml

@@ -10,12 +10,7 @@ services:
     volumes:
       - echopype_miniodata:/data
   httpserver:
-    # NOTE ====================================
-    # For httpserver test data,
-    # it needs to be copied to the docker container
-    # docker cp -L docker_httpserver_1:/usr/local/apache2/htdocs/data ./echopype/test_data
-    # =====================================
-    image: cormorack/http:latest
+    image: httpd:2.4
     ports:
       - 8080:80
     networks:

.ci_helpers/docker/setup-services.py

@@ -6,13 +6,14 @@
 
 import argparse
 import logging
-import shutil
+import os
 import subprocess
 import sys
 from pathlib import Path
 from typing import Dict, List
 
 import fsspec
+import pooch
 
 logger = logging.getLogger("setup-services")
 streamHandler = logging.StreamHandler(sys.stdout)
@@ -24,7 +25,18 @@
 HERE = Path(".").absolute()
 BASE = Path(__file__).parent.absolute()
 COMPOSE_FILE = BASE / "docker-compose.yaml"
-TEST_DATA_PATH = HERE / "echopype" / "test_data"
+
+
+def get_pooch_data_path() -> Path:
+    """Return path to the Pooch test data cache."""
+    ver = os.getenv("ECHOPYPE_DATA_VERSION", "v0.11.1a2")
+    cache_dir = Path(pooch.os_cache("echopype")) / ver
+    if not cache_dir.exists():
+        raise FileNotFoundError(
+            f"Pooch cache directory not found: {cache_dir}\n"
+            "Make sure test data was fetched via conftest.py"
+        )
+    return cache_dir
 
 
 def parse_args():
@@ -77,36 +89,70 @@ def run_commands(commands: List[Dict]) -> None:
 
 
 def load_s3(*args, **kwargs) -> None:
+    """Populate MinIO with test data from the Pooch cache (skip .zip files)."""
+    pooch_path = get_pooch_data_path()
     common_storage_options = dict(
         client_kwargs=dict(endpoint_url="http://localhost:9000/"),
         key="minioadmin",
         secret="minioadmin",
     )
-    bucket_name = "ooi-raw-data"
-    fs = fsspec.filesystem(
-        "s3",
-        **common_storage_options,
-    )
+    bucket_name = "echo-test-data"
+    fs = fsspec.filesystem("s3", **common_storage_options)
     test_data = "data"
+
     if not fs.exists(test_data):
         fs.mkdir(test_data)
-
     if not fs.exists(bucket_name):
         fs.mkdir(bucket_name)
 
-    # Load test data into bucket
-    for d in TEST_DATA_PATH.iterdir():
-        source_path = f"echopype/test_data/{d.name}"
-        fs.put(source_path, f"{test_data}/{d.name}", recursive=True)
+    for d in pooch_path.iterdir():
+        if d.suffix == ".zip":  # skip zip archives to cut redundant I/O
+            continue
+        source_path = str(d)
+        target_path = f"{test_data}/{d.name}"
+        logger.info(f"Copying {source_path} → {target_path}")
+        fs.put(source_path, target_path, recursive=True)
+
+
+def load_http_server(http_server_id) -> None:
+    """Copy test data from Pooch cache to HTTP server container."""
+    pooch_path = get_pooch_data_path()
+
+    # Create the data directory in the container
+    mkdir_result = subprocess.run(
+        ["docker", "exec", http_server_id, "sh", "-c", "mkdir -p /usr/local/apache2/htdocs/data"],
+        capture_output=True,
+        text=True,
+    )
+    if mkdir_result.returncode == 0:
+        logger.info("Created /usr/local/apache2/htdocs/data directory")
+    else:
+        logger.warning(f"mkdir warning (may be harmless): {mkdir_result.stderr}")
+
+    # Copy all dataset directories directly to /usr/local/apache2/htdocs/data/
+    for d in pooch_path.iterdir():
+        if d.suffix == ".zip":  # skip zip archives
+            continue
+        source_path = str(d)
+        dataset_name = d.name
+
+        # Copy directly to the data directory
+        target_path = f"/usr/local/apache2/htdocs/data/{dataset_name}"
+        cmd = ["docker", "cp", source_path, f"{http_server_id}:{target_path}"]
+        logger.info(f"Copying {source_path} → {target_path}")
+        result = subprocess.run(cmd, capture_output=True, text=True)
+
+        if result.returncode != 0:
+            logger.error(f"Failed to copy {dataset_name}: {result.stderr}")
 
 
 if __name__ == "__main__":
     args = parse_args()
     commands = []
+
     if all([args.deploy, args.tear_down]):
         print("Cannot have both --deploy and --tear-down. Exiting.")
         sys.exit(1)
-
     if not any([args.deploy, args.tear_down]):
         print("Please provide either --deploy or --tear-down flags. For more help use --help flag.")
         sys.exit(0)
@@ -136,33 +182,29 @@ def load_s3(*args, **kwargs) -> None:
                 }
             )
 
-        if TEST_DATA_PATH.exists():
-            commands.append(
-                {
-                    "msg": f"Deleting old test folder at {TEST_DATA_PATH} ...",
-                    "cmd": shutil.rmtree,
-                    "args": TEST_DATA_PATH,
-                }
-            )
+        pooch_path = get_pooch_data_path()
+        commands.append({"msg": f"Using Pooch test data at {pooch_path}", "cmd": None})
+
         commands.append(
             {
-                "msg": "Copying new test folder from http service ...",
-                "cmd": [
-                    "docker",
-                    "cp",
-                    "-L",
-                    f"{args.http_server}:/usr/local/apache2/htdocs/data",
-                    TEST_DATA_PATH,
-                ],
+                "msg": "Setting up MinIO S3 bucket with Pooch test data ...",
+                "cmd": load_s3,
             }
         )
 
-        commands.append({"msg": "Setting up minio s3 bucket ...", "cmd": load_s3})
+        commands.append(
+            {
+                "msg": f"Setting up HTTP server {args.http_server} with Pooch test data ...",
+                "cmd": load_http_server,
+                "args": args.http_server,
+            }
+        )
 
     if args.tear_down:
         command = ["docker-compose", "-f", COMPOSE_FILE, "down", "--remove-orphans", "--volumes"]
         if args.images:
-            command = command + ["--rmi", "all"]
+            command += ["--rmi", "all"]
         commands.append({"msg": "Stopping test services deployment ...", "cmd": command})
+
     commands.append({"msg": "Done.", "cmd": ["docker", "ps", "--last", "2"]})
     run_commands(commands)

.ci_helpers/py3.10.yaml → .ci_helpers/py3.13.yaml

@@ -2,7 +2,7 @@ name: echopype
 channels:
   - conda-forge
 dependencies:
-  - python=3.10
+  - python=3.13
   - pip
   - pip:
     - -r ../requirements.txt

.ci_helpers/setup-services-windows.py

@@ -0,0 +1,195 @@
+# .ci_helpers/setup-services-windows.py
+"""
+Spin up local services for Windows CI without Docker:
+- Start a MinIO server on localhost:9000
+- Seed S3 from the Pooch cache (unzipped test bundles)
+- Start a simple HTTP server on :8080 that exposes ./data (mirrors Linux job)
+- Write PID files for clean teardown
+
+Usage:
+  python .ci_helpers/setup-services-windows.py start [--no-http]
+  python .ci_helpers/setup-services-windows.py stop
+"""
+
+import argparse
+import os
+import pathlib
+import shutil
+import subprocess
+import sys
+import time
+from urllib.request import urlretrieve
+
+import fsspec
+import pooch
+
+MINIO_URL = "https://dl.min.io/server/minio/release/windows-amd64/minio.exe"
+MINIO_BIN = pathlib.Path(".ci_helpers") / "minio.exe"
+STATE_DIR = pathlib.Path(".ci_helpers") / ".state"
+STATE_DIR.mkdir(parents=True, exist_ok=True)
+MINIO_PID = STATE_DIR / "minio.pid"
+HTTP_PID = STATE_DIR / "http.pid"
+
+# Use localhost everywhere to match tests (which hit http://localhost:9000/...)
+MINIO_ENDPOINT = "http://localhost:9000/"
+MINIO_USER = "minioadmin"
+MINIO_PASS = "minioadmin"
+
+
+def get_pooch_cache() -> pathlib.Path:
+    """Return the Pooch cache dir for the configured dataset version."""
+    ver = os.getenv("ECHOPYPE_DATA_VERSION", "v0.11.1a2")
+    root = pathlib.Path(pooch.os_cache("echopype"))
+    path = root / ver
+    path.mkdir(parents=True, exist_ok=True)
+    return path
+
+
+def ensure_minio_downloaded() -> None:
+    if MINIO_BIN.exists():
+        return
+    MINIO_BIN.parent.mkdir(parents=True, exist_ok=True)
+    print(f"Downloading MinIO -> {MINIO_BIN}", flush=True)
+    urlretrieve(MINIO_URL, MINIO_BIN)
+    MINIO_BIN.chmod(0o755)
+
+
+def start_minio() -> None:
+    """Start MinIO on localhost:9000 and wait until ready."""
+    ensure_minio_downloaded()
+    data_dir = pathlib.Path(os.getenv("USERPROFILE", str(pathlib.Path.home()))) / "minio" / "data"
+    data_dir.mkdir(parents=True, exist_ok=True)
+
+    env = os.environ.copy()
+    env["MINIO_ROOT_USER"] = MINIO_USER
+    env["MINIO_ROOT_PASSWORD"] = MINIO_PASS
+
+    print(f"Starting MinIO on {MINIO_ENDPOINT} (data: {data_dir})", flush=True)
+    proc = subprocess.Popen(
+        [
+            str(MINIO_BIN),
+            "server",
+            str(data_dir),
+            "--address=localhost:9000",
+            "--console-address=localhost:9001",
+        ],
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL,
+        env=env,
+    )
+    MINIO_PID.write_text(str(proc.pid))
+
+    # Wait for readiness
+    import urllib.request
+
+    for _ in range(60):
+        try:
+            with urllib.request.urlopen(MINIO_ENDPOINT + "minio/health/ready", timeout=1):
+                break
+        except Exception:
+            time.sleep(1)
+    else:
+        raise RuntimeError("MinIO did not become ready on :9000")
+
+
+def seed_s3_from_pooch() -> None:
+    """Upload unzipped Pooch bundles into MinIO under the 'data/' prefix,
+    and also mirror a local ./data/ folder for the HTTP server."""
+    cache = get_pooch_cache()
+
+    # Seed S3 (MinIO)
+    fs = fsspec.filesystem(
+        "s3",
+        client_kwargs=dict(endpoint_url=MINIO_ENDPOINT),
+        key=MINIO_USER,
+        secret=MINIO_PASS,
+    )
+
+    for base in ("data", "ooi-raw-data"):
+        try:
+            fs.mkdir(base)
+        except Exception:
+            pass
+
+    for d in cache.iterdir():
+        if d.suffix == ".zip":
+            continue
+        tgt = f"data/{d.name}"
+        print(f"Uploading {d} -> {tgt}", flush=True)
+        fs.put(str(d), tgt, recursive=True)
+
+    # Build local ./data for HTTP tests that hit http://localhost:8080/data/...
+    local_data = pathlib.Path("data")
+    local_data.mkdir(exist_ok=True)
+    for d in cache.iterdir():
+        if d.suffix == ".zip":
+            continue
+        dst = local_data / d.name
+        if not dst.exists():
+            if d.is_dir():
+                shutil.copytree(d, dst)
+            else:
+                shutil.copy2(d, dst)
+
+
+def start_http_server() -> None:
+    """Serve the repository root so /data/... exists on :8080."""
+    root = pathlib.Path(".").absolute()
+    print(f"Starting local HTTP server on :8080 (root={root})", flush=True)
+    proc = subprocess.Popen(
+        [sys.executable, "-m", "http.server", "8080", "--directory", str(root)],
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL,
+    )
+    HTTP_PID.write_text(str(proc.pid))
+
+
+def stop_pid_file(path: pathlib.Path) -> None:
+    try:
+        pid = int(path.read_text().strip())
+    except Exception:
+        return
+    try:
+        if sys.platform.startswith("win"):
+            subprocess.run(["taskkill", "/PID", str(pid), "/F"], check=False)
+        else:
+            os.kill(pid, 9)
+    except Exception:
+        pass
+    try:
+        path.unlink(missing_ok=True)
+    except Exception:
+        pass
+
+
+def cmd_start(no_http: bool) -> None:
+    # Ensure cache exists (prefetch step should have populated it)
+    _ = get_pooch_cache()
+    start_minio()
+    seed_s3_from_pooch()
+    if not no_http:
+        start_http_server()
+    print("Services up.", flush=True)
+
+
+def cmd_stop() -> None:
+    stop_pid_file(HTTP_PID)
+    stop_pid_file(MINIO_PID)
+    print("Services stopped.", flush=True)
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    sub = ap.add_subparsers(dest="cmd", required=True)
+    s = sub.add_parser("start")
+    s.add_argument("--no-http", action="store_true", help="Do not start the local HTTP server")
+    sub.add_parser("stop")
+    args = ap.parse_args()
+    if args.cmd == "start":
+        cmd_start(no_http=args.no_http)
+    else:
+        cmd_stop()
+
+
+if __name__ == "__main__":
+    main()