Merge Cellpose dataset-source selection and BIA resolver improvements

.gitignore

@@ -173,4 +173,5 @@ data/
 bioimageio_test_reports/
 
 # MacOS
-.DS_STORE
+.DS_STORE
+demo-dataset/

bioengine_apps/bia_resolve_url_proxy/README.md

@@ -0,0 +1,5 @@
+# BIA Resolve URL Proxy
+
+Minimal ray-serve app that exposes a single `resolve_url` RPC method.
+
+It is used by `cellpose_finetuning` to fetch BioImage Archive assets through Hypha when direct fetches are blocked.

bioengine_apps/bia_resolve_url_proxy/main.py

@@ -0,0 +1,136 @@
+import base64
+import os
+from typing import Any
+from urllib.parse import urlparse
+
+import httpx
+from hypha_rpc.utils.schema import schema_method
+from ray import serve
+
+_DEFAULT_ALLOWED_HOSTS = "beta.bioimagearchive.org,www.ebi.ac.uk,ftp.ebi.ac.uk"
+
+
+def _allowed_hosts() -> set[str]:
+    raw_value = os.environ.get("RESOLVE_URL_ALLOWED_HOSTS", _DEFAULT_ALLOWED_HOSTS)
+    return {part.strip().lower() for part in raw_value.split(",") if part.strip()}
+
+
+def _normalize_headers(headers: dict[str, Any] | None) -> dict[str, str]:
+    normalized: dict[str, str] = {}
+    if not isinstance(headers, dict):
+        return normalized
+    for key, value in headers.items():
+        if not isinstance(key, str):
+            continue
+        if isinstance(value, str):
+            normalized[key] = value
+        elif value is not None:
+            normalized[key] = str(value)
+    return normalized
+
+
+@serve.deployment(
+    ray_actor_options={
+        "num_cpus": 0.5,
+        "num_gpus": 0,
+        "runtime_env": {
+            "pip": [
+                "httpx",
+            ],
+        },
+    }
+)
+class ResolveUrlProxy:
+    @schema_method
+    async def resolve_url(
+        self,
+        url: str,
+        method: str = "GET",
+        headers: dict[str, Any] | None = None,
+        timeout: float = 60.0,
+        body: str | dict[str, Any] | list[Any] | None = None,
+        context: dict[str, Any] | None = None,
+    ) -> dict[str, Any]:
+        try:
+            parsed = urlparse(str(url))
+        except Exception as exp:
+            return {
+                "ok": False,
+                "status_code": 400,
+                "url": str(url),
+                "error": f"Invalid URL: {exp}",
+            }
+
+        if parsed.scheme.lower() != "https":
+            return {
+                "ok": False,
+                "status_code": 400,
+                "url": str(url),
+                "error": "Only https URLs are allowed",
+            }
+
+        host = (parsed.hostname or "").lower()
+        if host not in _allowed_hosts():
+            return {
+                "ok": False,
+                "status_code": 403,
+                "url": str(url),
+                "error": f"Host '{host}' is not allowed",
+            }
+
+        method_value = str(method or "GET").upper()
+        if method_value not in {
+            "GET",
+            "POST",
+            "PUT",
+            "PATCH",
+            "DELETE",
+            "HEAD",
+            "OPTIONS",
+        }:
+            return {
+                "ok": False,
+                "status_code": 400,
+                "url": str(url),
+                "error": f"Unsupported method: {method_value}",
+            }
+
+        request_kwargs: dict[str, Any] = {
+            "method": method_value,
+            "url": str(url),
+            "headers": _normalize_headers(headers),
+        }
+        request_kwargs["headers"].setdefault(
+            "User-Agent", "bioengine-bia-resolve-url-proxy/1.0"
+        )
+
+        if body is not None:
+            if isinstance(body, (dict, list)):
+                request_kwargs["json"] = body
+            else:
+                request_kwargs["content"] = str(body)
+
+        try:
+            async with httpx.AsyncClient(
+                timeout=max(1.0, float(timeout)),
+                follow_redirects=True,
+            ) as client:
+                response = await client.request(**request_kwargs)
+        except Exception as exp:
+            return {
+                "ok": False,
+                "status_code": 502,
+                "url": str(url),
+                "error": str(exp),
+            }
+
+        status_code = int(response.status_code)
+        ok = 200 <= status_code < 300
+        return {
+            "ok": ok,
+            "status_code": status_code,
+            "url": str(response.url),
+            "headers": dict(response.headers),
+            "content_base64": base64.b64encode(response.content).decode("ascii"),
+            "error": "" if ok else f"Upstream returned HTTP {status_code}",
+        }

bioengine_apps/bia_resolve_url_proxy/manifest.yaml

@@ -0,0 +1,16 @@
+name: BIA Resolve URL Proxy
+id: bia-resolve-url-proxy
+id_emoji: '🌐'
+covers: []
+description: Resolve URL proxy for BioImage Archive downloads (resolve_url only).
+documentation: README.md
+format_version: 0.5.0
+git_repo: https://github.com/aicell-lab/bioengine-worker
+links: []
+tags: [bioimage-archive, proxy, resolve-url]
+type: ray-serve
+version: 0.0.1
+deployments:
+  - main:ResolveUrlProxy
+authorized_users:
+  - "*"

bioengine_apps/cellpose_finetuning/README.md

@@ -191,9 +191,11 @@ Start asynchronous model fine-tuning.
 - `metadata_dir` (str, optional): Folder containing metadata JSON files with image/annotation paths (e.g. `image_path` + `mask_path`). If provided, explicit `train_annotations` can be omitted.
 - `test_images` (str, optional): Optional test images path (same format as train_images). Providing test data enables per-epoch pixel-level validation metrics and end-of-training instance segmentation metrics (AP@0.5/0.75/0.9).
 - `test_annotations` (str, optional): Optional test annotations (label masks). Required together with `test_images`.
+- `split_mode` (str): Dataset split mode (`"manual"` or `"auto"`, default: `"manual"`). In `"auto"`, the service creates train/validation split from the selected sample pool.
+- `train_split_ratio` (float): Train ratio used when `split_mode="auto"` (default: `0.8`, i.e. 80% train / 20% validation).
 - `model` (str): Pretrained model to start from (default: "cpsam")
 - `n_epochs` (int): Number of training epochs (default: 10)
-- `n_samples` (int, optional): Limit number of samples to use
+- `n_samples` (int | float, optional): Limit sample usage. If `n_samples > 1`, treated as absolute count. If `0 < n_samples <= 1`, treated as decimal fraction of available samples (e.g. `0.05` = 5%).
 - `learning_rate` (float): Learning rate (default: 1e-6)
 - `weight_decay` (float): Weight decay (default: 0.0001)
 - `min_train_masks` (int): Minimum number of masks per training batch (default: 5). Lower values speed up training.
@@ -376,6 +378,31 @@ Notes:
 - Sessions with stale `waiting`/`preparing`/`running` states after a redeploy are normalized to `stopped` (interrupted).
 - Running-session indicators in the UI only track sessions that are currently active.
 
+### `delete_training_session()`
+
+Delete a non-running training session and its local artifacts.
+
+**Parameters:**
+- `session_id` (str): Session ID to delete
+- `force_stop_if_blocked` (bool, optional): If true, stop a blocked `running`/`preparing` session first, then delete
+
+**Returns:** Dict with `deleted` (bool) and `session_id` (str)
+
+### `preflight_training_dataset()`
+
+Run a lightweight backend preflight over training path configuration without starting training.
+
+**Parameters:**
+- `artifact` (str): Artifact ID or BioImage Archive URL
+- `train_images`, `train_annotations` (str, optional): Training path specs
+- `metadata_dir` (str, optional): Metadata mode directory
+- `test_images`, `test_annotations` (str, optional): Optional test path specs
+- `split_mode` (str): `manual` or `auto`
+- `n_samples` (int | float, optional): Optional sample limit/count
+- `max_candidates` (int): Cap for path matching candidates during preflight
+
+**Returns:** Dict including `ok`, pair counts (`train_pair_count`, `test_pair_count`), and an explanatory `message`.
+
 Validation metrics are only produced when both `test_images` and `test_annotations` are provided.
 
 ## Inference Parameters Guide
@@ -430,7 +457,7 @@ The recommended way to deploy (whether for the first time or to update) is to us
 source .env
 
 # Deploy to production
-python tests/cellpose_legacy_scripts/redeploy_cellpose.py \
+python tests/cellpose/redeploy_cellpose.py \
     --artifact-id ri-scale/cellpose-finetuning \
     --application-id cellpose-finetuning
 ```