From bdc0f9abaa7ea6dc77ae1f4ce9a4b03624e806cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Kuku=C4=8Dka?= Date: Thu, 29 Jan 2026 17:32:30 +0000 Subject: [PATCH 01/22] feat: tile masks --- configs/data/tiled/knl_patos/2_224.yaml | 9 +++ configs/preprocessing/tile_masks.yaml | 8 +++ preprocessing/tile_masks.py | 92 +++++++++++++++++++++++++ scripts/preprocessing/tile_masks.py | 16 +++++ 4 files changed, 125 insertions(+) create mode 100644 configs/data/tiled/knl_patos/2_224.yaml create mode 100644 configs/preprocessing/tile_masks.yaml create mode 100644 preprocessing/tile_masks.py create mode 100644 scripts/preprocessing/tile_masks.py diff --git a/configs/data/tiled/knl_patos/2_224.yaml b/configs/data/tiled/knl_patos/2_224.yaml new file mode 100644 index 0000000..4e45af4 --- /dev/null +++ b/configs/data/tiled/knl_patos/2_224.yaml @@ -0,0 +1,9 @@ +# @package _global_ + +dataset: + institution: knl_patos + mpp: 1.55 + tile_extent: 224 + uris: + - test_preliminary: ??? + - test_final: ??? diff --git a/configs/preprocessing/tile_masks.yaml b/configs/preprocessing/tile_masks.yaml new file mode 100644 index 0000000..a140812 --- /dev/null +++ b/configs/preprocessing/tile_masks.yaml @@ -0,0 +1,8 @@ +# @package _global_ + +max_concurrent: 64 +artifact_path: tile_masks + +metadata: + run_name: "🎭 Tile Masks: ${dataset.institution}" + description: Tile masks for ${dataset.institution} at level ${level} \ No newline at end of file diff --git a/preprocessing/tile_masks.py b/preprocessing/tile_masks.py new file mode 100644 index 0000000..4517b27 --- /dev/null +++ b/preprocessing/tile_masks.py @@ -0,0 +1,92 @@ +from collections.abc import Iterable +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import cast + +import hydra +import mlflow.artifacts +import numpy as np +import pandas as pd +import pyvips +import ray +from omegaconf import DictConfig +from rationai.masks import process_items, tile_mask, write_big_tiff +from rationai.mlkit import autolog, with_cli_args +from rationai.mlkit.lightning.loggers import MLFlowLogger + + +ray.init(runtime_env={"excludes": [".git", ".venv"]}) + + +def download_slide_tiles(uris: Iterable[str]) -> tuple[pd.DataFrame, pd.DataFrame]: + slidess, tiless = [], [] + for uri in uris: + path = mlflow.artifacts.download_artifacts(artifact_uri=uri) + slidess.append(pd.read_parquet(Path(path) / "slides.parquet")) + tiless.append(pd.read_parquet(Path(path) / "tiles.parquet")) + + slides = pd.concat(slidess).reset_index(drop=True) + tiles = pd.concat(tiless).reset_index(drop=True) + return slides, tiles + + +@ray.remote(memory=4 * 1024**3) +def process_slide( + slide: pd.Series, + tiles_ref: ray.ObjectRef, + output_folder: Path, +) -> None: + tiles = cast("pd.DataFrame", ray.get(tiles_ref)) + slide_tiles = tiles[tiles["slide_id"] == slide.id] + + blur_slide_tiles = slide_tiles[slide_tiles["blur"] > 0.25] + artifacts_slide_tiles = slide_tiles[slide_tiles["artifacts"] > 0.25] + slide_tiles = slide_tiles.drop(blur_slide_tiles.index) + slide_tiles = slide_tiles.drop(artifacts_slide_tiles.index) + + for folder, tiles_subset in [ + ("blur_tiles", blur_slide_tiles), + ("artifacts_tiles", artifacts_slide_tiles), + ("clean_tiles", slide_tiles), + ]: + if tiles_subset.empty: + continue + + mask = tile_mask( + tiles_subset, + tile_extent=(slide["tile_extent_x"], slide["tile_extent_y"]), + size=(slide["extent_x"], slide["extent_y"]), + ) + + mask_path = output_folder / folder / f"{Path(slide['path']).stem}.tiff" + mask_path.parent.mkdir(parents=True, exist_ok=True) + write_big_tiff( + pyvips.Image.new_from_array(np.array(mask)), + mask_path, + mpp_x=slide["mpp_x"], + mpp_y=slide["mpp_y"], + ) + + +@with_cli_args(["+preprocessing=tile_masks"]) +@hydra.main(config_path="../configs", config_name="preprocessing", version_base=None) +@autolog +def main(config: DictConfig, logger: MLFlowLogger) -> None: + slides, tiles = download_slide_tiles(config.dataset.uris.values()) + tiles_ref = ray.put(tiles) + + with TemporaryDirectory() as output_dir: + process_items( + (slide for _, slide in slides.iterrows()), + process_item=process_slide, + fn_kwargs={ + "tiles_ref": tiles_ref, + "output_folder": Path(output_dir), + }, + max_concurrent=config.max_concurrent, + ) + logger.log_artifacts(local_dir=output_dir, artifact_path=config.artifact_path) + + +if __name__ == "__main__": + main() diff --git a/scripts/preprocessing/tile_masks.py b/scripts/preprocessing/tile_masks.py new file mode 100644 index 0000000..53c7ec4 --- /dev/null +++ b/scripts/preprocessing/tile_masks.py @@ -0,0 +1,16 @@ +from kube_jobs import submit_job + + +submit_job( + job_name="ulcerative-colitis-tile-masks-...", + username=..., + public=False, + cpu=64, + memory="32Gi", + script=[ + "git clone https://gitlab.ics.muni.cz/rationai/digital-pathology/pathology/ulcerative-colitis.git workdir", + "cd workdir", + "uv sync --frozen", + "uv run --active -m preprocessing.tile_masks +data=tiled/.../...", + ], +) From b8242f489cc458f41a252daeba72f43b28912a4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Kuku=C4=8Dka?= Date: Sun, 1 Feb 2026 09:57:43 +0000 Subject: [PATCH 02/22] feat: add tiled dataset configurations --- configs/data/tiled/ftn/0_320.yaml | 10 ++++++++++ configs/data/tiled/ftn/0_430.yaml | 10 ++++++++++ configs/data/tiled/ftn/1_224.yaml | 10 ++++++++++ configs/data/tiled/ftn/2_224.yaml | 10 ++++++++++ configs/data/tiled/ikem/0_320.yaml | 10 ++++++++++ configs/data/tiled/ikem/0_430.yaml | 10 ++++++++++ configs/data/tiled/ikem/1_224.yaml | 10 ++++++++++ configs/data/tiled/ikem/2_224.yaml | 10 ++++++++++ configs/data/tiled/knl_patos/0_320.yaml | 9 +++++++++ configs/data/tiled/knl_patos/0_430.yaml | 9 +++++++++ configs/data/tiled/knl_patos/1_224.yaml | 9 +++++++++ configs/data/tiled/knl_patos/2_224.yaml | 4 ++-- 12 files changed, 109 insertions(+), 2 deletions(-) create mode 100644 configs/data/tiled/ftn/0_320.yaml create mode 100644 configs/data/tiled/ftn/0_430.yaml create mode 100644 configs/data/tiled/ftn/1_224.yaml create mode 100644 configs/data/tiled/ftn/2_224.yaml create mode 100644 configs/data/tiled/ikem/0_320.yaml create mode 100644 configs/data/tiled/ikem/0_430.yaml create mode 100644 configs/data/tiled/ikem/1_224.yaml create mode 100644 configs/data/tiled/ikem/2_224.yaml create mode 100644 configs/data/tiled/knl_patos/0_320.yaml create mode 100644 configs/data/tiled/knl_patos/0_430.yaml create mode 100644 configs/data/tiled/knl_patos/1_224.yaml diff --git a/configs/data/tiled/ftn/0_320.yaml b/configs/data/tiled/ftn/0_320.yaml new file mode 100644 index 0000000..0a5f821 --- /dev/null +++ b/configs/data/tiled/ftn/0_320.yaml @@ -0,0 +1,10 @@ +# @package _global_ + +dataset: + institution: ftn + mpp: 0.17 + tile_extent: 320 + uris: + - train: "mlflow-artifacts:/86/63f6ec12cda147ee96f1865abe7bca1f/artifacts/train - ftn" # TODO update URI + - test_preliminary: "mlflow-artifacts:/86/63f6ec12cda147ee96f1865abe7bca1f/artifacts/test preliminary - ftn" # TODO update URI + - test_final: "mlflow-artifacts:/86/63f6ec12cda147ee96f1865abe7bca1f/artifacts/test final - ftn" # TODO update URI \ No newline at end of file diff --git a/configs/data/tiled/ftn/0_430.yaml b/configs/data/tiled/ftn/0_430.yaml new file mode 100644 index 0000000..4fa56ec --- /dev/null +++ b/configs/data/tiled/ftn/0_430.yaml @@ -0,0 +1,10 @@ +# @package _global_ + +dataset: + institution: ftn + mpp: 0.17 + tile_extent: 430 + uris: + - train: "mlflow-artifacts:/86/a5e82f95be7f4e27a9893a5306449d4a/artifacts/train - ftn" # TODO update URI + - test_preliminary: "mlflow-artifacts:/86/a5e82f95be7f4e27a9893a5306449d4a/artifacts/test preliminary - ftn" # TODO update URI + - test_final: "mlflow-artifacts:/86/a5e82f95be7f4e27a9893a5306449d4a/artifacts/test final - ftn" # TODO update URI \ No newline at end of file diff --git a/configs/data/tiled/ftn/1_224.yaml b/configs/data/tiled/ftn/1_224.yaml new file mode 100644 index 0000000..fdff5e2 --- /dev/null +++ b/configs/data/tiled/ftn/1_224.yaml @@ -0,0 +1,10 @@ +# @package _global_ + +dataset: + institution: ftn + mpp: 0.52 + tile_extent: 224 + uris: + - train: "mlflow-artifacts:/86/caf93595030443cc8a968751b8462208/artifacts/train - ftn" # TODO update URI + - test_preliminary: "mlflow-artifacts:/86/caf93595030443cc8a968751b8462208/artifacts/test preliminary - ftn" # TODO update URI + - test_final: "mlflow-artifacts:/86/caf93595030443cc8a968751b8462208/artifacts/test final - ftn" # TODO update URI \ No newline at end of file diff --git a/configs/data/tiled/ftn/2_224.yaml b/configs/data/tiled/ftn/2_224.yaml new file mode 100644 index 0000000..9501b7c --- /dev/null +++ b/configs/data/tiled/ftn/2_224.yaml @@ -0,0 +1,10 @@ +# @package _global_ + +dataset: + institution: ftn + mpp: 1.55 + tile_extent: 224 + uris: + - train: "mlflow-artifacts:/86/5101b69d86ff4034bd36cfeb4a6ed448/artifacts/train - ftn" # TODO update URI + - test_preliminary: "mlflow-artifacts:/86/5101b69d86ff4034bd36cfeb4a6ed448/artifacts/test preliminary - ftn" # TODO update URI + - test_final: "mlflow-artifacts:/86/5101b69d86ff4034bd36cfeb4a6ed448/artifacts/test final - ftn" # TODO update URI \ No newline at end of file diff --git a/configs/data/tiled/ikem/0_320.yaml b/configs/data/tiled/ikem/0_320.yaml new file mode 100644 index 0000000..6f5a9aa --- /dev/null +++ b/configs/data/tiled/ikem/0_320.yaml @@ -0,0 +1,10 @@ +# @package _global_ + +dataset: + institution: ikem + mpp: 0.17 + tile_extent: 320 + uris: + - train: "mlflow-artifacts:/86/7a5266d3b02343ada4dbcc1510d6316d/artifacts/train - ikem" # TODO update URI + - test_preliminary: "mlflow-artifacts:/86/7a5266d3b02343ada4dbcc1510d6316d/artifacts/test preliminary - ikem" # TODO update URI + - test_final: "mlflow-artifacts:/86/7a5266d3b02343ada4dbcc1510d6316d/artifacts/test final - ikem" # TODO update URI \ No newline at end of file diff --git a/configs/data/tiled/ikem/0_430.yaml b/configs/data/tiled/ikem/0_430.yaml new file mode 100644 index 0000000..40026f2 --- /dev/null +++ b/configs/data/tiled/ikem/0_430.yaml @@ -0,0 +1,10 @@ +# @package _global_ + +dataset: + institution: ikem + mpp: 0.17 + tile_extent: 430 + uris: + - train: "mlflow-artifacts:/86/95c33c64f0704450809d9cd9e61d1e3e/artifacts/train - ikem" # TODO update URI + - test_preliminary: "mlflow-artifacts:/86/95c33c64f0704450809d9cd9e61d1e3e/artifacts/test preliminary - ikem" # TODO update URI + - test_final: "mlflow-artifacts:/86/95c33c64f0704450809d9cd9e61d1e3e/artifacts/test final - ikem" # TODO update URI \ No newline at end of file diff --git a/configs/data/tiled/ikem/1_224.yaml b/configs/data/tiled/ikem/1_224.yaml new file mode 100644 index 0000000..2d235c7 --- /dev/null +++ b/configs/data/tiled/ikem/1_224.yaml @@ -0,0 +1,10 @@ +# @package _global_ + +dataset: + institution: ikem + mpp: 0.52 + tile_extent: 224 + uris: + - train: "mlflow-artifacts:/86/ad2b3102f90f4999944b48f978317f92/artifacts/train - ikem" # TODO update URI + - test_preliminary: "mlflow-artifacts:/86/ad2b3102f90f4999944b48f978317f92/artifacts/test preliminary - ikem" # TODO update URI + - test_final: "mlflow-artifacts:/86/ad2b3102f90f4999944b48f978317f92/artifacts/test final - ikem" # TODO update URI \ No newline at end of file diff --git a/configs/data/tiled/ikem/2_224.yaml b/configs/data/tiled/ikem/2_224.yaml new file mode 100644 index 0000000..245e3cb --- /dev/null +++ b/configs/data/tiled/ikem/2_224.yaml @@ -0,0 +1,10 @@ +# @package _global_ + +dataset: + institution: ikem + mpp: 1.55 + tile_extent: 224 + uris: + - train: "mlflow-artifacts:/86/6cf22b59ff6e40f18f8f754cd475ecd7/artifacts/train - ikem" # TODO update URI + - test_preliminary: "mlflow-artifacts:/86/6cf22b59ff6e40f18f8f754cd475ecd7/artifacts/test preliminary - ikem" # TODO update URI + - test_final: "mlflow-artifacts:/86/6cf22b59ff6e40f18f8f754cd475ecd7/artifacts/test final - ikem" # TODO update URI \ No newline at end of file diff --git a/configs/data/tiled/knl_patos/0_320.yaml b/configs/data/tiled/knl_patos/0_320.yaml new file mode 100644 index 0000000..5f065c4 --- /dev/null +++ b/configs/data/tiled/knl_patos/0_320.yaml @@ -0,0 +1,9 @@ +# @package _global_ + +dataset: + institution: knl_patos + mpp: 0.17 + tile_extent: 320 + uris: + - test_preliminary: "mlflow-artifacts:/86/9dd07f04cd2a4414be39f67f448c120e/artifacts/test preliminary - knl_patos" # TODO update URI + - test_final: "mlflow-artifacts:/86/9dd07f04cd2a4414be39f67f448c120e/artifacts/test final - knl_patos" # TODO update URI diff --git a/configs/data/tiled/knl_patos/0_430.yaml b/configs/data/tiled/knl_patos/0_430.yaml new file mode 100644 index 0000000..faa4488 --- /dev/null +++ b/configs/data/tiled/knl_patos/0_430.yaml @@ -0,0 +1,9 @@ +# @package _global_ + +dataset: + institution: knl_patos + mpp: 0.17 + tile_extent: 430 + uris: + - test_preliminary: "mlflow-artifacts:/86/417a3e18427d4ddfa95f9b6b23e56201/artifacts/test preliminary - knl_patos" # TODO update URI + - test_final: "mlflow-artifacts:/86/417a3e18427d4ddfa95f9b6b23e56201/artifacts/test final - knl_patos" # TODO update URI \ No newline at end of file diff --git a/configs/data/tiled/knl_patos/1_224.yaml b/configs/data/tiled/knl_patos/1_224.yaml new file mode 100644 index 0000000..301ad0d --- /dev/null +++ b/configs/data/tiled/knl_patos/1_224.yaml @@ -0,0 +1,9 @@ +# @package _global_ + +dataset: + institution: knl_patos + mpp: 0.52 + tile_extent: 224 + uris: + - test_preliminary: "mlflow-artifacts:/86/066af7c14b0b4f7684e0c155229f68ba/artifacts/test preliminary - knl_patos" # TODO update URI + - test_final: "mlflow-artifacts:/86/066af7c14b0b4f7684e0c155229f68ba/artifacts/test final - knl_patos" # TODO update URI \ No newline at end of file diff --git a/configs/data/tiled/knl_patos/2_224.yaml b/configs/data/tiled/knl_patos/2_224.yaml index 4e45af4..a06baba 100644 --- a/configs/data/tiled/knl_patos/2_224.yaml +++ b/configs/data/tiled/knl_patos/2_224.yaml @@ -5,5 +5,5 @@ dataset: mpp: 1.55 tile_extent: 224 uris: - - test_preliminary: ??? - - test_final: ??? + - test_preliminary: "mlflow-artifacts:/86/4643080588b147b1a1b3cde12392209b/artifacts/test preliminary - knl_patos" # TODO update URI + - test_final: "mlflow-artifacts:/86/4643080588b147b1a1b3cde12392209b/artifacts/test final - knl_patos" # TODO update URI \ No newline at end of file From 2c02ee277a5e020e28c387980dfc6facd4473527 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Kuku=C4=8Dka?= Date: Sun, 1 Feb 2026 10:09:24 +0000 Subject: [PATCH 03/22] fix: add level to tiled datasets --- configs/data/tiled/ftn/0_320.yaml | 1 + configs/data/tiled/ftn/0_430.yaml | 1 + configs/data/tiled/ftn/1_224.yaml | 1 + configs/data/tiled/ftn/2_224.yaml | 1 + configs/data/tiled/ikem/0_320.yaml | 1 + configs/data/tiled/ikem/0_430.yaml | 1 + configs/data/tiled/ikem/1_224.yaml | 1 + configs/data/tiled/ikem/2_224.yaml | 1 + configs/data/tiled/knl_patos/0_320.yaml | 1 + configs/data/tiled/knl_patos/0_430.yaml | 1 + configs/data/tiled/knl_patos/1_224.yaml | 1 + configs/data/tiled/knl_patos/2_224.yaml | 1 + configs/preprocessing/tile_masks.yaml | 2 +- 13 files changed, 13 insertions(+), 1 deletion(-) diff --git a/configs/data/tiled/ftn/0_320.yaml b/configs/data/tiled/ftn/0_320.yaml index 0a5f821..5fe9d26 100644 --- a/configs/data/tiled/ftn/0_320.yaml +++ b/configs/data/tiled/ftn/0_320.yaml @@ -4,6 +4,7 @@ dataset: institution: ftn mpp: 0.17 tile_extent: 320 + level: 0 uris: - train: "mlflow-artifacts:/86/63f6ec12cda147ee96f1865abe7bca1f/artifacts/train - ftn" # TODO update URI - test_preliminary: "mlflow-artifacts:/86/63f6ec12cda147ee96f1865abe7bca1f/artifacts/test preliminary - ftn" # TODO update URI diff --git a/configs/data/tiled/ftn/0_430.yaml b/configs/data/tiled/ftn/0_430.yaml index 4fa56ec..0521b22 100644 --- a/configs/data/tiled/ftn/0_430.yaml +++ b/configs/data/tiled/ftn/0_430.yaml @@ -4,6 +4,7 @@ dataset: institution: ftn mpp: 0.17 tile_extent: 430 + level: 0 uris: - train: "mlflow-artifacts:/86/a5e82f95be7f4e27a9893a5306449d4a/artifacts/train - ftn" # TODO update URI - test_preliminary: "mlflow-artifacts:/86/a5e82f95be7f4e27a9893a5306449d4a/artifacts/test preliminary - ftn" # TODO update URI diff --git a/configs/data/tiled/ftn/1_224.yaml b/configs/data/tiled/ftn/1_224.yaml index fdff5e2..d9b3f45 100644 --- a/configs/data/tiled/ftn/1_224.yaml +++ b/configs/data/tiled/ftn/1_224.yaml @@ -4,6 +4,7 @@ dataset: institution: ftn mpp: 0.52 tile_extent: 224 + level: 1 uris: - train: "mlflow-artifacts:/86/caf93595030443cc8a968751b8462208/artifacts/train - ftn" # TODO update URI - test_preliminary: "mlflow-artifacts:/86/caf93595030443cc8a968751b8462208/artifacts/test preliminary - ftn" # TODO update URI diff --git a/configs/data/tiled/ftn/2_224.yaml b/configs/data/tiled/ftn/2_224.yaml index 9501b7c..d81843a 100644 --- a/configs/data/tiled/ftn/2_224.yaml +++ b/configs/data/tiled/ftn/2_224.yaml @@ -4,6 +4,7 @@ dataset: institution: ftn mpp: 1.55 tile_extent: 224 + level: 2 uris: - train: "mlflow-artifacts:/86/5101b69d86ff4034bd36cfeb4a6ed448/artifacts/train - ftn" # TODO update URI - test_preliminary: "mlflow-artifacts:/86/5101b69d86ff4034bd36cfeb4a6ed448/artifacts/test preliminary - ftn" # TODO update URI diff --git a/configs/data/tiled/ikem/0_320.yaml b/configs/data/tiled/ikem/0_320.yaml index 6f5a9aa..970abe8 100644 --- a/configs/data/tiled/ikem/0_320.yaml +++ b/configs/data/tiled/ikem/0_320.yaml @@ -4,6 +4,7 @@ dataset: institution: ikem mpp: 0.17 tile_extent: 320 + level: 0 uris: - train: "mlflow-artifacts:/86/7a5266d3b02343ada4dbcc1510d6316d/artifacts/train - ikem" # TODO update URI - test_preliminary: "mlflow-artifacts:/86/7a5266d3b02343ada4dbcc1510d6316d/artifacts/test preliminary - ikem" # TODO update URI diff --git a/configs/data/tiled/ikem/0_430.yaml b/configs/data/tiled/ikem/0_430.yaml index 40026f2..2e9d772 100644 --- a/configs/data/tiled/ikem/0_430.yaml +++ b/configs/data/tiled/ikem/0_430.yaml @@ -4,6 +4,7 @@ dataset: institution: ikem mpp: 0.17 tile_extent: 430 + level: 0 uris: - train: "mlflow-artifacts:/86/95c33c64f0704450809d9cd9e61d1e3e/artifacts/train - ikem" # TODO update URI - test_preliminary: "mlflow-artifacts:/86/95c33c64f0704450809d9cd9e61d1e3e/artifacts/test preliminary - ikem" # TODO update URI diff --git a/configs/data/tiled/ikem/1_224.yaml b/configs/data/tiled/ikem/1_224.yaml index 2d235c7..f5c59b1 100644 --- a/configs/data/tiled/ikem/1_224.yaml +++ b/configs/data/tiled/ikem/1_224.yaml @@ -4,6 +4,7 @@ dataset: institution: ikem mpp: 0.52 tile_extent: 224 + level: 1 uris: - train: "mlflow-artifacts:/86/ad2b3102f90f4999944b48f978317f92/artifacts/train - ikem" # TODO update URI - test_preliminary: "mlflow-artifacts:/86/ad2b3102f90f4999944b48f978317f92/artifacts/test preliminary - ikem" # TODO update URI diff --git a/configs/data/tiled/ikem/2_224.yaml b/configs/data/tiled/ikem/2_224.yaml index 245e3cb..4683805 100644 --- a/configs/data/tiled/ikem/2_224.yaml +++ b/configs/data/tiled/ikem/2_224.yaml @@ -4,6 +4,7 @@ dataset: institution: ikem mpp: 1.55 tile_extent: 224 + level: 2 uris: - train: "mlflow-artifacts:/86/6cf22b59ff6e40f18f8f754cd475ecd7/artifacts/train - ikem" # TODO update URI - test_preliminary: "mlflow-artifacts:/86/6cf22b59ff6e40f18f8f754cd475ecd7/artifacts/test preliminary - ikem" # TODO update URI diff --git a/configs/data/tiled/knl_patos/0_320.yaml b/configs/data/tiled/knl_patos/0_320.yaml index 5f065c4..5c2e9f5 100644 --- a/configs/data/tiled/knl_patos/0_320.yaml +++ b/configs/data/tiled/knl_patos/0_320.yaml @@ -4,6 +4,7 @@ dataset: institution: knl_patos mpp: 0.17 tile_extent: 320 + level: 0 uris: - test_preliminary: "mlflow-artifacts:/86/9dd07f04cd2a4414be39f67f448c120e/artifacts/test preliminary - knl_patos" # TODO update URI - test_final: "mlflow-artifacts:/86/9dd07f04cd2a4414be39f67f448c120e/artifacts/test final - knl_patos" # TODO update URI diff --git a/configs/data/tiled/knl_patos/0_430.yaml b/configs/data/tiled/knl_patos/0_430.yaml index faa4488..7e3de10 100644 --- a/configs/data/tiled/knl_patos/0_430.yaml +++ b/configs/data/tiled/knl_patos/0_430.yaml @@ -4,6 +4,7 @@ dataset: institution: knl_patos mpp: 0.17 tile_extent: 430 + level: 0 uris: - test_preliminary: "mlflow-artifacts:/86/417a3e18427d4ddfa95f9b6b23e56201/artifacts/test preliminary - knl_patos" # TODO update URI - test_final: "mlflow-artifacts:/86/417a3e18427d4ddfa95f9b6b23e56201/artifacts/test final - knl_patos" # TODO update URI \ No newline at end of file diff --git a/configs/data/tiled/knl_patos/1_224.yaml b/configs/data/tiled/knl_patos/1_224.yaml index 301ad0d..3f95611 100644 --- a/configs/data/tiled/knl_patos/1_224.yaml +++ b/configs/data/tiled/knl_patos/1_224.yaml @@ -4,6 +4,7 @@ dataset: institution: knl_patos mpp: 0.52 tile_extent: 224 + level: 1 uris: - test_preliminary: "mlflow-artifacts:/86/066af7c14b0b4f7684e0c155229f68ba/artifacts/test preliminary - knl_patos" # TODO update URI - test_final: "mlflow-artifacts:/86/066af7c14b0b4f7684e0c155229f68ba/artifacts/test final - knl_patos" # TODO update URI \ No newline at end of file diff --git a/configs/data/tiled/knl_patos/2_224.yaml b/configs/data/tiled/knl_patos/2_224.yaml index a06baba..d9c9d37 100644 --- a/configs/data/tiled/knl_patos/2_224.yaml +++ b/configs/data/tiled/knl_patos/2_224.yaml @@ -4,6 +4,7 @@ dataset: institution: knl_patos mpp: 1.55 tile_extent: 224 + level: 2 uris: - test_preliminary: "mlflow-artifacts:/86/4643080588b147b1a1b3cde12392209b/artifacts/test preliminary - knl_patos" # TODO update URI - test_final: "mlflow-artifacts:/86/4643080588b147b1a1b3cde12392209b/artifacts/test final - knl_patos" # TODO update URI \ No newline at end of file diff --git a/configs/preprocessing/tile_masks.yaml b/configs/preprocessing/tile_masks.yaml index a140812..e753055 100644 --- a/configs/preprocessing/tile_masks.yaml +++ b/configs/preprocessing/tile_masks.yaml @@ -5,4 +5,4 @@ artifact_path: tile_masks metadata: run_name: "🎭 Tile Masks: ${dataset.institution}" - description: Tile masks for ${dataset.institution} at level ${level} \ No newline at end of file + description: Tile masks for ${dataset.institution} at level ${dataset.level} \ No newline at end of file From 996dbf44d48115c109bfa66e1ee6041a165c7308 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Kuku=C4=8Dka?= Date: Sun, 1 Feb 2026 10:13:12 +0000 Subject: [PATCH 04/22] fix: listconf -> dictconf --- configs/data/tiled/ftn/0_320.yaml | 6 +++--- configs/data/tiled/ftn/0_430.yaml | 6 +++--- configs/data/tiled/ftn/1_224.yaml | 6 +++--- configs/data/tiled/ftn/2_224.yaml | 6 +++--- configs/data/tiled/ikem/0_320.yaml | 6 +++--- configs/data/tiled/ikem/0_430.yaml | 6 +++--- configs/data/tiled/ikem/1_224.yaml | 6 +++--- configs/data/tiled/ikem/2_224.yaml | 6 +++--- configs/data/tiled/knl_patos/0_320.yaml | 4 ++-- configs/data/tiled/knl_patos/0_430.yaml | 4 ++-- configs/data/tiled/knl_patos/1_224.yaml | 4 ++-- configs/data/tiled/knl_patos/2_224.yaml | 4 ++-- 12 files changed, 32 insertions(+), 32 deletions(-) diff --git a/configs/data/tiled/ftn/0_320.yaml b/configs/data/tiled/ftn/0_320.yaml index 5fe9d26..2821139 100644 --- a/configs/data/tiled/ftn/0_320.yaml +++ b/configs/data/tiled/ftn/0_320.yaml @@ -6,6 +6,6 @@ dataset: tile_extent: 320 level: 0 uris: - - train: "mlflow-artifacts:/86/63f6ec12cda147ee96f1865abe7bca1f/artifacts/train - ftn" # TODO update URI - - test_preliminary: "mlflow-artifacts:/86/63f6ec12cda147ee96f1865abe7bca1f/artifacts/test preliminary - ftn" # TODO update URI - - test_final: "mlflow-artifacts:/86/63f6ec12cda147ee96f1865abe7bca1f/artifacts/test final - ftn" # TODO update URI \ No newline at end of file + train: "mlflow-artifacts:/86/63f6ec12cda147ee96f1865abe7bca1f/artifacts/train - ftn" # TODO update URI + test_preliminary: "mlflow-artifacts:/86/63f6ec12cda147ee96f1865abe7bca1f/artifacts/test preliminary - ftn" # TODO update URI + test_final: "mlflow-artifacts:/86/63f6ec12cda147ee96f1865abe7bca1f/artifacts/test final - ftn" # TODO update URI \ No newline at end of file diff --git a/configs/data/tiled/ftn/0_430.yaml b/configs/data/tiled/ftn/0_430.yaml index 0521b22..748eb1b 100644 --- a/configs/data/tiled/ftn/0_430.yaml +++ b/configs/data/tiled/ftn/0_430.yaml @@ -6,6 +6,6 @@ dataset: tile_extent: 430 level: 0 uris: - - train: "mlflow-artifacts:/86/a5e82f95be7f4e27a9893a5306449d4a/artifacts/train - ftn" # TODO update URI - - test_preliminary: "mlflow-artifacts:/86/a5e82f95be7f4e27a9893a5306449d4a/artifacts/test preliminary - ftn" # TODO update URI - - test_final: "mlflow-artifacts:/86/a5e82f95be7f4e27a9893a5306449d4a/artifacts/test final - ftn" # TODO update URI \ No newline at end of file + train: "mlflow-artifacts:/86/a5e82f95be7f4e27a9893a5306449d4a/artifacts/train - ftn" # TODO update URI + test_preliminary: "mlflow-artifacts:/86/a5e82f95be7f4e27a9893a5306449d4a/artifacts/test preliminary - ftn" # TODO update URI + test_final: "mlflow-artifacts:/86/a5e82f95be7f4e27a9893a5306449d4a/artifacts/test final - ftn" # TODO update URI \ No newline at end of file diff --git a/configs/data/tiled/ftn/1_224.yaml b/configs/data/tiled/ftn/1_224.yaml index d9b3f45..cc65308 100644 --- a/configs/data/tiled/ftn/1_224.yaml +++ b/configs/data/tiled/ftn/1_224.yaml @@ -6,6 +6,6 @@ dataset: tile_extent: 224 level: 1 uris: - - train: "mlflow-artifacts:/86/caf93595030443cc8a968751b8462208/artifacts/train - ftn" # TODO update URI - - test_preliminary: "mlflow-artifacts:/86/caf93595030443cc8a968751b8462208/artifacts/test preliminary - ftn" # TODO update URI - - test_final: "mlflow-artifacts:/86/caf93595030443cc8a968751b8462208/artifacts/test final - ftn" # TODO update URI \ No newline at end of file + train: "mlflow-artifacts:/86/caf93595030443cc8a968751b8462208/artifacts/train - ftn" # TODO update URI + test_preliminary: "mlflow-artifacts:/86/caf93595030443cc8a968751b8462208/artifacts/test preliminary - ftn" # TODO update URI + test_final: "mlflow-artifacts:/86/caf93595030443cc8a968751b8462208/artifacts/test final - ftn" # TODO update URI \ No newline at end of file diff --git a/configs/data/tiled/ftn/2_224.yaml b/configs/data/tiled/ftn/2_224.yaml index d81843a..992ed61 100644 --- a/configs/data/tiled/ftn/2_224.yaml +++ b/configs/data/tiled/ftn/2_224.yaml @@ -6,6 +6,6 @@ dataset: tile_extent: 224 level: 2 uris: - - train: "mlflow-artifacts:/86/5101b69d86ff4034bd36cfeb4a6ed448/artifacts/train - ftn" # TODO update URI - - test_preliminary: "mlflow-artifacts:/86/5101b69d86ff4034bd36cfeb4a6ed448/artifacts/test preliminary - ftn" # TODO update URI - - test_final: "mlflow-artifacts:/86/5101b69d86ff4034bd36cfeb4a6ed448/artifacts/test final - ftn" # TODO update URI \ No newline at end of file + train: "mlflow-artifacts:/86/5101b69d86ff4034bd36cfeb4a6ed448/artifacts/train - ftn" # TODO update URI + test_preliminary: "mlflow-artifacts:/86/5101b69d86ff4034bd36cfeb4a6ed448/artifacts/test preliminary - ftn" # TODO update URI + test_final: "mlflow-artifacts:/86/5101b69d86ff4034bd36cfeb4a6ed448/artifacts/test final - ftn" # TODO update URI \ No newline at end of file diff --git a/configs/data/tiled/ikem/0_320.yaml b/configs/data/tiled/ikem/0_320.yaml index 970abe8..a38fb44 100644 --- a/configs/data/tiled/ikem/0_320.yaml +++ b/configs/data/tiled/ikem/0_320.yaml @@ -6,6 +6,6 @@ dataset: tile_extent: 320 level: 0 uris: - - train: "mlflow-artifacts:/86/7a5266d3b02343ada4dbcc1510d6316d/artifacts/train - ikem" # TODO update URI - - test_preliminary: "mlflow-artifacts:/86/7a5266d3b02343ada4dbcc1510d6316d/artifacts/test preliminary - ikem" # TODO update URI - - test_final: "mlflow-artifacts:/86/7a5266d3b02343ada4dbcc1510d6316d/artifacts/test final - ikem" # TODO update URI \ No newline at end of file + train: "mlflow-artifacts:/86/7a5266d3b02343ada4dbcc1510d6316d/artifacts/train - ikem" # TODO update URI + test_preliminary: "mlflow-artifacts:/86/7a5266d3b02343ada4dbcc1510d6316d/artifacts/test preliminary - ikem" # TODO update URI + test_final: "mlflow-artifacts:/86/7a5266d3b02343ada4dbcc1510d6316d/artifacts/test final - ikem" # TODO update URI \ No newline at end of file diff --git a/configs/data/tiled/ikem/0_430.yaml b/configs/data/tiled/ikem/0_430.yaml index 2e9d772..c57df1c 100644 --- a/configs/data/tiled/ikem/0_430.yaml +++ b/configs/data/tiled/ikem/0_430.yaml @@ -6,6 +6,6 @@ dataset: tile_extent: 430 level: 0 uris: - - train: "mlflow-artifacts:/86/95c33c64f0704450809d9cd9e61d1e3e/artifacts/train - ikem" # TODO update URI - - test_preliminary: "mlflow-artifacts:/86/95c33c64f0704450809d9cd9e61d1e3e/artifacts/test preliminary - ikem" # TODO update URI - - test_final: "mlflow-artifacts:/86/95c33c64f0704450809d9cd9e61d1e3e/artifacts/test final - ikem" # TODO update URI \ No newline at end of file + train: "mlflow-artifacts:/86/95c33c64f0704450809d9cd9e61d1e3e/artifacts/train - ikem" # TODO update URI + test_preliminary: "mlflow-artifacts:/86/95c33c64f0704450809d9cd9e61d1e3e/artifacts/test preliminary - ikem" # TODO update URI + test_final: "mlflow-artifacts:/86/95c33c64f0704450809d9cd9e61d1e3e/artifacts/test final - ikem" # TODO update URI \ No newline at end of file diff --git a/configs/data/tiled/ikem/1_224.yaml b/configs/data/tiled/ikem/1_224.yaml index f5c59b1..7a6ce84 100644 --- a/configs/data/tiled/ikem/1_224.yaml +++ b/configs/data/tiled/ikem/1_224.yaml @@ -6,6 +6,6 @@ dataset: tile_extent: 224 level: 1 uris: - - train: "mlflow-artifacts:/86/ad2b3102f90f4999944b48f978317f92/artifacts/train - ikem" # TODO update URI - - test_preliminary: "mlflow-artifacts:/86/ad2b3102f90f4999944b48f978317f92/artifacts/test preliminary - ikem" # TODO update URI - - test_final: "mlflow-artifacts:/86/ad2b3102f90f4999944b48f978317f92/artifacts/test final - ikem" # TODO update URI \ No newline at end of file + train: "mlflow-artifacts:/86/ad2b3102f90f4999944b48f978317f92/artifacts/train - ikem" # TODO update URI + test_preliminary: "mlflow-artifacts:/86/ad2b3102f90f4999944b48f978317f92/artifacts/test preliminary - ikem" # TODO update URI + test_final: "mlflow-artifacts:/86/ad2b3102f90f4999944b48f978317f92/artifacts/test final - ikem" # TODO update URI \ No newline at end of file diff --git a/configs/data/tiled/ikem/2_224.yaml b/configs/data/tiled/ikem/2_224.yaml index 4683805..260758e 100644 --- a/configs/data/tiled/ikem/2_224.yaml +++ b/configs/data/tiled/ikem/2_224.yaml @@ -6,6 +6,6 @@ dataset: tile_extent: 224 level: 2 uris: - - train: "mlflow-artifacts:/86/6cf22b59ff6e40f18f8f754cd475ecd7/artifacts/train - ikem" # TODO update URI - - test_preliminary: "mlflow-artifacts:/86/6cf22b59ff6e40f18f8f754cd475ecd7/artifacts/test preliminary - ikem" # TODO update URI - - test_final: "mlflow-artifacts:/86/6cf22b59ff6e40f18f8f754cd475ecd7/artifacts/test final - ikem" # TODO update URI \ No newline at end of file + train: "mlflow-artifacts:/86/6cf22b59ff6e40f18f8f754cd475ecd7/artifacts/train - ikem" # TODO update URI + test_preliminary: "mlflow-artifacts:/86/6cf22b59ff6e40f18f8f754cd475ecd7/artifacts/test preliminary - ikem" # TODO update URI + test_final: "mlflow-artifacts:/86/6cf22b59ff6e40f18f8f754cd475ecd7/artifacts/test final - ikem" # TODO update URI \ No newline at end of file diff --git a/configs/data/tiled/knl_patos/0_320.yaml b/configs/data/tiled/knl_patos/0_320.yaml index 5c2e9f5..63b8cd2 100644 --- a/configs/data/tiled/knl_patos/0_320.yaml +++ b/configs/data/tiled/knl_patos/0_320.yaml @@ -6,5 +6,5 @@ dataset: tile_extent: 320 level: 0 uris: - - test_preliminary: "mlflow-artifacts:/86/9dd07f04cd2a4414be39f67f448c120e/artifacts/test preliminary - knl_patos" # TODO update URI - - test_final: "mlflow-artifacts:/86/9dd07f04cd2a4414be39f67f448c120e/artifacts/test final - knl_patos" # TODO update URI + test_preliminary: "mlflow-artifacts:/86/9dd07f04cd2a4414be39f67f448c120e/artifacts/test preliminary - knl_patos" # TODO update URI + test_final: "mlflow-artifacts:/86/9dd07f04cd2a4414be39f67f448c120e/artifacts/test final - knl_patos" # TODO update URI diff --git a/configs/data/tiled/knl_patos/0_430.yaml b/configs/data/tiled/knl_patos/0_430.yaml index 7e3de10..ba8785f 100644 --- a/configs/data/tiled/knl_patos/0_430.yaml +++ b/configs/data/tiled/knl_patos/0_430.yaml @@ -6,5 +6,5 @@ dataset: tile_extent: 430 level: 0 uris: - - test_preliminary: "mlflow-artifacts:/86/417a3e18427d4ddfa95f9b6b23e56201/artifacts/test preliminary - knl_patos" # TODO update URI - - test_final: "mlflow-artifacts:/86/417a3e18427d4ddfa95f9b6b23e56201/artifacts/test final - knl_patos" # TODO update URI \ No newline at end of file + test_preliminary: "mlflow-artifacts:/86/417a3e18427d4ddfa95f9b6b23e56201/artifacts/test preliminary - knl_patos" # TODO update URI + test_final: "mlflow-artifacts:/86/417a3e18427d4ddfa95f9b6b23e56201/artifacts/test final - knl_patos" # TODO update URI \ No newline at end of file diff --git a/configs/data/tiled/knl_patos/1_224.yaml b/configs/data/tiled/knl_patos/1_224.yaml index 3f95611..62e74b3 100644 --- a/configs/data/tiled/knl_patos/1_224.yaml +++ b/configs/data/tiled/knl_patos/1_224.yaml @@ -6,5 +6,5 @@ dataset: tile_extent: 224 level: 1 uris: - - test_preliminary: "mlflow-artifacts:/86/066af7c14b0b4f7684e0c155229f68ba/artifacts/test preliminary - knl_patos" # TODO update URI - - test_final: "mlflow-artifacts:/86/066af7c14b0b4f7684e0c155229f68ba/artifacts/test final - knl_patos" # TODO update URI \ No newline at end of file + test_preliminary: "mlflow-artifacts:/86/066af7c14b0b4f7684e0c155229f68ba/artifacts/test preliminary - knl_patos" # TODO update URI + test_final: "mlflow-artifacts:/86/066af7c14b0b4f7684e0c155229f68ba/artifacts/test final - knl_patos" # TODO update URI \ No newline at end of file diff --git a/configs/data/tiled/knl_patos/2_224.yaml b/configs/data/tiled/knl_patos/2_224.yaml index d9c9d37..8de3c91 100644 --- a/configs/data/tiled/knl_patos/2_224.yaml +++ b/configs/data/tiled/knl_patos/2_224.yaml @@ -6,5 +6,5 @@ dataset: tile_extent: 224 level: 2 uris: - - test_preliminary: "mlflow-artifacts:/86/4643080588b147b1a1b3cde12392209b/artifacts/test preliminary - knl_patos" # TODO update URI - - test_final: "mlflow-artifacts:/86/4643080588b147b1a1b3cde12392209b/artifacts/test final - knl_patos" # TODO update URI \ No newline at end of file + test_preliminary: "mlflow-artifacts:/86/4643080588b147b1a1b3cde12392209b/artifacts/test preliminary - knl_patos" # TODO update URI + test_final: "mlflow-artifacts:/86/4643080588b147b1a1b3cde12392209b/artifacts/test final - knl_patos" # TODO update URI \ No newline at end of file From 752cdd76837d7e3e5656d8186db1328caefc3c13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Kuku=C4=8Dka?= Date: Sun, 1 Feb 2026 11:58:25 +0000 Subject: [PATCH 05/22] fix: update process_slide to use tiles DataFrame directly instead of tiles_ref --- preprocessing/tile_masks.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/preprocessing/tile_masks.py b/preprocessing/tile_masks.py index 4517b27..034d2cc 100644 --- a/preprocessing/tile_masks.py +++ b/preprocessing/tile_masks.py @@ -33,10 +33,9 @@ def download_slide_tiles(uris: Iterable[str]) -> tuple[pd.DataFrame, pd.DataFram @ray.remote(memory=4 * 1024**3) def process_slide( slide: pd.Series, - tiles_ref: ray.ObjectRef, + tiles: pd.DataFrame, # tiles are automatically serialized by Ray output_folder: Path, ) -> None: - tiles = cast("pd.DataFrame", ray.get(tiles_ref)) slide_tiles = tiles[tiles["slide_id"] == slide.id] blur_slide_tiles = slide_tiles[slide_tiles["blur"] > 0.25] @@ -57,11 +56,12 @@ def process_slide( tile_extent=(slide["tile_extent_x"], slide["tile_extent_y"]), size=(slide["extent_x"], slide["extent_y"]), ) + mask = cast("pyvips.Image", pyvips.Image.new_from_array(np.array(mask))) mask_path = output_folder / folder / f"{Path(slide['path']).stem}.tiff" mask_path.parent.mkdir(parents=True, exist_ok=True) write_big_tiff( - pyvips.Image.new_from_array(np.array(mask)), + mask, mask_path, mpp_x=slide["mpp_x"], mpp_y=slide["mpp_y"], @@ -80,7 +80,7 @@ def main(config: DictConfig, logger: MLFlowLogger) -> None: (slide for _, slide in slides.iterrows()), process_item=process_slide, fn_kwargs={ - "tiles_ref": tiles_ref, + "tiles": tiles_ref, "output_folder": Path(output_dir), }, max_concurrent=config.max_concurrent, From dc136c54ff5581b8957a79c4f6a8ad59573a775a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Kuku=C4=8Dka?= Date: Sun, 1 Feb 2026 12:03:57 +0000 Subject: [PATCH 06/22] fix: duplicite drop --- preprocessing/tile_masks.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/preprocessing/tile_masks.py b/preprocessing/tile_masks.py index 034d2cc..f78464d 100644 --- a/preprocessing/tile_masks.py +++ b/preprocessing/tile_masks.py @@ -40,8 +40,9 @@ def process_slide( blur_slide_tiles = slide_tiles[slide_tiles["blur"] > 0.25] artifacts_slide_tiles = slide_tiles[slide_tiles["artifacts"] > 0.25] - slide_tiles = slide_tiles.drop(blur_slide_tiles.index) - slide_tiles = slide_tiles.drop(artifacts_slide_tiles.index) + + drop_indices = blur_slide_tiles.index.union(artifacts_slide_tiles.index) + slide_tiles = slide_tiles.drop(drop_indices) for folder, tiles_subset in [ ("blur_tiles", blur_slide_tiles), From 282eb88d6c7560f27446c4c7a8a5be51808bcada Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Kuku=C4=8Dka?= Date: Sun, 1 Feb 2026 18:39:36 +0000 Subject: [PATCH 07/22] feat: update tiling uris --- configs/data/tiled/ftn/0_320.yaml | 6 +++--- configs/data/tiled/ftn/0_430.yaml | 6 +++--- configs/data/tiled/ftn/1_224.yaml | 6 +++--- configs/data/tiled/ftn/2_224.yaml | 6 +++--- configs/data/tiled/ikem/0_320.yaml | 6 +++--- configs/data/tiled/ikem/0_430.yaml | 6 +++--- configs/data/tiled/ikem/1_224.yaml | 6 +++--- configs/data/tiled/ikem/2_224.yaml | 6 +++--- configs/data/tiled/knl_patos/0_320.yaml | 4 ++-- configs/data/tiled/knl_patos/0_430.yaml | 4 ++-- configs/data/tiled/knl_patos/1_224.yaml | 4 ++-- configs/data/tiled/knl_patos/2_224.yaml | 4 ++-- 12 files changed, 32 insertions(+), 32 deletions(-) diff --git a/configs/data/tiled/ftn/0_320.yaml b/configs/data/tiled/ftn/0_320.yaml index 2821139..06afedd 100644 --- a/configs/data/tiled/ftn/0_320.yaml +++ b/configs/data/tiled/ftn/0_320.yaml @@ -6,6 +6,6 @@ dataset: tile_extent: 320 level: 0 uris: - train: "mlflow-artifacts:/86/63f6ec12cda147ee96f1865abe7bca1f/artifacts/train - ftn" # TODO update URI - test_preliminary: "mlflow-artifacts:/86/63f6ec12cda147ee96f1865abe7bca1f/artifacts/test preliminary - ftn" # TODO update URI - test_final: "mlflow-artifacts:/86/63f6ec12cda147ee96f1865abe7bca1f/artifacts/test final - ftn" # TODO update URI \ No newline at end of file + train: "mlflow-artifacts:/86/bbbe4603bc30495d85ac99093fc9269a/artifacts/train - ftn" # TODO update URI + test_preliminary: "mlflow-artifacts:/86/bbbe4603bc30495d85ac99093fc9269a/artifacts/test preliminary - ftn" # TODO update URI + test_final: "mlflow-artifacts:/86/bbbe4603bc30495d85ac99093fc9269a/artifacts/test final - ftn" # TODO update URI \ No newline at end of file diff --git a/configs/data/tiled/ftn/0_430.yaml b/configs/data/tiled/ftn/0_430.yaml index 748eb1b..2d1f3ff 100644 --- a/configs/data/tiled/ftn/0_430.yaml +++ b/configs/data/tiled/ftn/0_430.yaml @@ -6,6 +6,6 @@ dataset: tile_extent: 430 level: 0 uris: - train: "mlflow-artifacts:/86/a5e82f95be7f4e27a9893a5306449d4a/artifacts/train - ftn" # TODO update URI - test_preliminary: "mlflow-artifacts:/86/a5e82f95be7f4e27a9893a5306449d4a/artifacts/test preliminary - ftn" # TODO update URI - test_final: "mlflow-artifacts:/86/a5e82f95be7f4e27a9893a5306449d4a/artifacts/test final - ftn" # TODO update URI \ No newline at end of file + train: "mlflow-artifacts:/86/de450f835f0d4462a91b35f4a79a500f/artifacts/train - ftn" # TODO update URI + test_preliminary: "mlflow-artifacts:/86/de450f835f0d4462a91b35f4a79a500f/artifacts/test preliminary - ftn" # TODO update URI + test_final: "mlflow-artifacts:/86/de450f835f0d4462a91b35f4a79a500f/artifacts/test final - ftn" # TODO update URI \ No newline at end of file diff --git a/configs/data/tiled/ftn/1_224.yaml b/configs/data/tiled/ftn/1_224.yaml index cc65308..b8904a8 100644 --- a/configs/data/tiled/ftn/1_224.yaml +++ b/configs/data/tiled/ftn/1_224.yaml @@ -6,6 +6,6 @@ dataset: tile_extent: 224 level: 1 uris: - train: "mlflow-artifacts:/86/caf93595030443cc8a968751b8462208/artifacts/train - ftn" # TODO update URI - test_preliminary: "mlflow-artifacts:/86/caf93595030443cc8a968751b8462208/artifacts/test preliminary - ftn" # TODO update URI - test_final: "mlflow-artifacts:/86/caf93595030443cc8a968751b8462208/artifacts/test final - ftn" # TODO update URI \ No newline at end of file + train: "mlflow-artifacts:/86/f85b64a7f96c41e38f86d84956e2dbe9/artifacts/train - ftn" # TODO update URI + test_preliminary: "mlflow-artifacts:/86/f85b64a7f96c41e38f86d84956e2dbe9/artifacts/test preliminary - ftn" # TODO update URI + test_final: "mlflow-artifacts:/86/f85b64a7f96c41e38f86d84956e2dbe9/artifacts/test final - ftn" # TODO update URI \ No newline at end of file diff --git a/configs/data/tiled/ftn/2_224.yaml b/configs/data/tiled/ftn/2_224.yaml index 992ed61..8e89157 100644 --- a/configs/data/tiled/ftn/2_224.yaml +++ b/configs/data/tiled/ftn/2_224.yaml @@ -6,6 +6,6 @@ dataset: tile_extent: 224 level: 2 uris: - train: "mlflow-artifacts:/86/5101b69d86ff4034bd36cfeb4a6ed448/artifacts/train - ftn" # TODO update URI - test_preliminary: "mlflow-artifacts:/86/5101b69d86ff4034bd36cfeb4a6ed448/artifacts/test preliminary - ftn" # TODO update URI - test_final: "mlflow-artifacts:/86/5101b69d86ff4034bd36cfeb4a6ed448/artifacts/test final - ftn" # TODO update URI \ No newline at end of file + train: "mlflow-artifacts:/86/5814484b6cd7467e9d712889655479af/artifacts/train - ftn" # TODO update URI + test_preliminary: "mlflow-artifacts:/86/5814484b6cd7467e9d712889655479af/artifacts/test preliminary - ftn" # TODO update URI + test_final: "mlflow-artifacts:/86/5814484b6cd7467e9d712889655479af/artifacts/test final - ftn" # TODO update URI \ No newline at end of file diff --git a/configs/data/tiled/ikem/0_320.yaml b/configs/data/tiled/ikem/0_320.yaml index a38fb44..7951f25 100644 --- a/configs/data/tiled/ikem/0_320.yaml +++ b/configs/data/tiled/ikem/0_320.yaml @@ -6,6 +6,6 @@ dataset: tile_extent: 320 level: 0 uris: - train: "mlflow-artifacts:/86/7a5266d3b02343ada4dbcc1510d6316d/artifacts/train - ikem" # TODO update URI - test_preliminary: "mlflow-artifacts:/86/7a5266d3b02343ada4dbcc1510d6316d/artifacts/test preliminary - ikem" # TODO update URI - test_final: "mlflow-artifacts:/86/7a5266d3b02343ada4dbcc1510d6316d/artifacts/test final - ikem" # TODO update URI \ No newline at end of file + train: "mlflow-artifacts:/86/4486e598446d412d926ac66dadb35e51/artifacts/train - ikem" # TODO update URI + test_preliminary: "mlflow-artifacts:/86/4486e598446d412d926ac66dadb35e51/artifacts/test preliminary - ikem" # TODO update URI + test_final: "mlflow-artifacts:/86/4486e598446d412d926ac66dadb35e51/artifacts/test final - ikem" # TODO update URI \ No newline at end of file diff --git a/configs/data/tiled/ikem/0_430.yaml b/configs/data/tiled/ikem/0_430.yaml index c57df1c..d63dfe0 100644 --- a/configs/data/tiled/ikem/0_430.yaml +++ b/configs/data/tiled/ikem/0_430.yaml @@ -6,6 +6,6 @@ dataset: tile_extent: 430 level: 0 uris: - train: "mlflow-artifacts:/86/95c33c64f0704450809d9cd9e61d1e3e/artifacts/train - ikem" # TODO update URI - test_preliminary: "mlflow-artifacts:/86/95c33c64f0704450809d9cd9e61d1e3e/artifacts/test preliminary - ikem" # TODO update URI - test_final: "mlflow-artifacts:/86/95c33c64f0704450809d9cd9e61d1e3e/artifacts/test final - ikem" # TODO update URI \ No newline at end of file + train: "mlflow-artifacts:/86/fd112e63819c49d999502542b35bfce1/artifacts/train - ikem" # TODO update URI + test_preliminary: "mlflow-artifacts:/86/fd112e63819c49d999502542b35bfce1/artifacts/test preliminary - ikem" # TODO update URI + test_final: "mlflow-artifacts:/86/fd112e63819c49d999502542b35bfce1/artifacts/test final - ikem" # TODO update URI \ No newline at end of file diff --git a/configs/data/tiled/ikem/1_224.yaml b/configs/data/tiled/ikem/1_224.yaml index 7a6ce84..467d3a4 100644 --- a/configs/data/tiled/ikem/1_224.yaml +++ b/configs/data/tiled/ikem/1_224.yaml @@ -6,6 +6,6 @@ dataset: tile_extent: 224 level: 1 uris: - train: "mlflow-artifacts:/86/ad2b3102f90f4999944b48f978317f92/artifacts/train - ikem" # TODO update URI - test_preliminary: "mlflow-artifacts:/86/ad2b3102f90f4999944b48f978317f92/artifacts/test preliminary - ikem" # TODO update URI - test_final: "mlflow-artifacts:/86/ad2b3102f90f4999944b48f978317f92/artifacts/test final - ikem" # TODO update URI \ No newline at end of file + train: "mlflow-artifacts:/86/ece822e7c0e3416f97212267c773c8ac/artifacts/train - ikem" # TODO update URI + test_preliminary: "mlflow-artifacts:/86/ece822e7c0e3416f97212267c773c8ac/artifacts/test preliminary - ikem" # TODO update URI + test_final: "mlflow-artifacts:/86/ece822e7c0e3416f97212267c773c8ac/artifacts/test final - ikem" # TODO update URI \ No newline at end of file diff --git a/configs/data/tiled/ikem/2_224.yaml b/configs/data/tiled/ikem/2_224.yaml index 260758e..0ec13aa 100644 --- a/configs/data/tiled/ikem/2_224.yaml +++ b/configs/data/tiled/ikem/2_224.yaml @@ -6,6 +6,6 @@ dataset: tile_extent: 224 level: 2 uris: - train: "mlflow-artifacts:/86/6cf22b59ff6e40f18f8f754cd475ecd7/artifacts/train - ikem" # TODO update URI - test_preliminary: "mlflow-artifacts:/86/6cf22b59ff6e40f18f8f754cd475ecd7/artifacts/test preliminary - ikem" # TODO update URI - test_final: "mlflow-artifacts:/86/6cf22b59ff6e40f18f8f754cd475ecd7/artifacts/test final - ikem" # TODO update URI \ No newline at end of file + train: "mlflow-artifacts:/86/0c09e1c61d294fa3877b6b21703bab2f/artifacts/train - ikem" # TODO update URI + test_preliminary: "mlflow-artifacts:/86/0c09e1c61d294fa3877b6b21703bab2f/artifacts/test preliminary - ikem" # TODO update URI + test_final: "mlflow-artifacts:/86/0c09e1c61d294fa3877b6b21703bab2f/artifacts/test final - ikem" # TODO update URI \ No newline at end of file diff --git a/configs/data/tiled/knl_patos/0_320.yaml b/configs/data/tiled/knl_patos/0_320.yaml index 63b8cd2..e2c7d5b 100644 --- a/configs/data/tiled/knl_patos/0_320.yaml +++ b/configs/data/tiled/knl_patos/0_320.yaml @@ -6,5 +6,5 @@ dataset: tile_extent: 320 level: 0 uris: - test_preliminary: "mlflow-artifacts:/86/9dd07f04cd2a4414be39f67f448c120e/artifacts/test preliminary - knl_patos" # TODO update URI - test_final: "mlflow-artifacts:/86/9dd07f04cd2a4414be39f67f448c120e/artifacts/test final - knl_patos" # TODO update URI + test_preliminary: "mlflow-artifacts:/86/7b9a446145b14965981bbac88e8e2c8b/artifacts/test preliminary - knl_patos" # TODO update URI + test_final: "mlflow-artifacts:/86/7b9a446145b14965981bbac88e8e2c8b/artifacts/test final - knl_patos" # TODO update URI diff --git a/configs/data/tiled/knl_patos/0_430.yaml b/configs/data/tiled/knl_patos/0_430.yaml index ba8785f..dafd755 100644 --- a/configs/data/tiled/knl_patos/0_430.yaml +++ b/configs/data/tiled/knl_patos/0_430.yaml @@ -6,5 +6,5 @@ dataset: tile_extent: 430 level: 0 uris: - test_preliminary: "mlflow-artifacts:/86/417a3e18427d4ddfa95f9b6b23e56201/artifacts/test preliminary - knl_patos" # TODO update URI - test_final: "mlflow-artifacts:/86/417a3e18427d4ddfa95f9b6b23e56201/artifacts/test final - knl_patos" # TODO update URI \ No newline at end of file + test_preliminary: "mlflow-artifacts:/86/eb29255c944d4dad926160a7cb102ad9/artifacts/test preliminary - knl_patos" # TODO update URI + test_final: "mlflow-artifacts:/86/eb29255c944d4dad926160a7cb102ad9/artifacts/test final - knl_patos" # TODO update URI \ No newline at end of file diff --git a/configs/data/tiled/knl_patos/1_224.yaml b/configs/data/tiled/knl_patos/1_224.yaml index 62e74b3..3c86e8e 100644 --- a/configs/data/tiled/knl_patos/1_224.yaml +++ b/configs/data/tiled/knl_patos/1_224.yaml @@ -6,5 +6,5 @@ dataset: tile_extent: 224 level: 1 uris: - test_preliminary: "mlflow-artifacts:/86/066af7c14b0b4f7684e0c155229f68ba/artifacts/test preliminary - knl_patos" # TODO update URI - test_final: "mlflow-artifacts:/86/066af7c14b0b4f7684e0c155229f68ba/artifacts/test final - knl_patos" # TODO update URI \ No newline at end of file + test_preliminary: "mlflow-artifacts:/86/6782155362d54ecc9f1beccb4362d359/artifacts/test preliminary - knl_patos" # TODO update URI + test_final: "mlflow-artifacts:/86/6782155362d54ecc9f1beccb4362d359/artifacts/test final - knl_patos" # TODO update URI \ No newline at end of file diff --git a/configs/data/tiled/knl_patos/2_224.yaml b/configs/data/tiled/knl_patos/2_224.yaml index 8de3c91..3061d80 100644 --- a/configs/data/tiled/knl_patos/2_224.yaml +++ b/configs/data/tiled/knl_patos/2_224.yaml @@ -6,5 +6,5 @@ dataset: tile_extent: 224 level: 2 uris: - test_preliminary: "mlflow-artifacts:/86/4643080588b147b1a1b3cde12392209b/artifacts/test preliminary - knl_patos" # TODO update URI - test_final: "mlflow-artifacts:/86/4643080588b147b1a1b3cde12392209b/artifacts/test final - knl_patos" # TODO update URI \ No newline at end of file + test_preliminary: "mlflow-artifacts:/86/d7486bb6b667433989c3ce1c8ce31d60/artifacts/test preliminary - knl_patos" # TODO update URI + test_final: "mlflow-artifacts:/86/d7486bb6b667433989c3ce1c8ce31d60/artifacts/test final - knl_patos" # TODO update URI \ No newline at end of file From 177f79bee34bffd27d55cc21711de0bd5fa4bd14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Kuku=C4=8Dka?= Date: Sun, 1 Feb 2026 20:35:14 +0000 Subject: [PATCH 08/22] feat: add target level --- configs/preprocessing/tile_masks.yaml | 1 + preprocessing/tile_masks.py | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/configs/preprocessing/tile_masks.yaml b/configs/preprocessing/tile_masks.yaml index e753055..e2bb3f8 100644 --- a/configs/preprocessing/tile_masks.yaml +++ b/configs/preprocessing/tile_masks.yaml @@ -2,6 +2,7 @@ max_concurrent: 64 artifact_path: tile_masks +level: 3 metadata: run_name: "🎭 Tile Masks: ${dataset.institution}" diff --git a/preprocessing/tile_masks.py b/preprocessing/tile_masks.py index f78464d..b0c6884 100644 --- a/preprocessing/tile_masks.py +++ b/preprocessing/tile_masks.py @@ -13,6 +13,7 @@ from rationai.masks import process_items, tile_mask, write_big_tiff from rationai.mlkit import autolog, with_cli_args from rationai.mlkit.lightning.loggers import MLFlowLogger +from ratiopath.openslide import OpenSlide ray.init(runtime_env={"excludes": [".git", ".venv"]}) @@ -34,8 +35,12 @@ def download_slide_tiles(uris: Iterable[str]) -> tuple[pd.DataFrame, pd.DataFram def process_slide( slide: pd.Series, tiles: pd.DataFrame, # tiles are automatically serialized by Ray + level: int, output_folder: Path, ) -> None: + with OpenSlide(slide["path"]) as slide_wsi: + mask_extent_x, mask_extent_y = slide_wsi.level_dimensions[level] + slide_tiles = tiles[tiles["slide_id"] == slide.id] blur_slide_tiles = slide_tiles[slide_tiles["blur"] > 0.25] @@ -58,6 +63,10 @@ def process_slide( size=(slide["extent_x"], slide["extent_y"]), ) mask = cast("pyvips.Image", pyvips.Image.new_from_array(np.array(mask))) + mask = cast( + "pyvips.Image", + mask.resize(mask_extent_x / mask.width, vscale=mask_extent_y / mask.height), # type: ignore[reportCallIssue] + ) mask_path = output_folder / folder / f"{Path(slide['path']).stem}.tiff" mask_path.parent.mkdir(parents=True, exist_ok=True) From 3943872bd70d923a91aaa8d88dde2c5aab9ff453 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Kuku=C4=8Dka?= Date: Sun, 1 Feb 2026 20:42:31 +0000 Subject: [PATCH 09/22] fix: missing level --- preprocessing/tile_masks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/preprocessing/tile_masks.py b/preprocessing/tile_masks.py index b0c6884..109fe78 100644 --- a/preprocessing/tile_masks.py +++ b/preprocessing/tile_masks.py @@ -91,6 +91,7 @@ def main(config: DictConfig, logger: MLFlowLogger) -> None: process_item=process_slide, fn_kwargs={ "tiles": tiles_ref, + "level": config.level, "output_folder": Path(output_dir), }, max_concurrent=config.max_concurrent, From 9f67c7bec7cf6a382f79d1e07dabaeb03029eaa4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Kuku=C4=8Dka?= Date: Sun, 1 Feb 2026 21:58:59 +0000 Subject: [PATCH 10/22] fix: WIP --- preprocessing/tile_masks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/preprocessing/tile_masks.py b/preprocessing/tile_masks.py index 109fe78..17b4d9c 100644 --- a/preprocessing/tile_masks.py +++ b/preprocessing/tile_masks.py @@ -38,6 +38,7 @@ def process_slide( level: int, output_folder: Path, ) -> None: + print(slide["path"]) with OpenSlide(slide["path"]) as slide_wsi: mask_extent_x, mask_extent_y = slide_wsi.level_dimensions[level] From 37c977fe91fbf1e62f6c50c7fec23ea4a200d82c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Kuku=C4=8Dka?= Date: Sun, 1 Feb 2026 22:21:50 +0000 Subject: [PATCH 11/22] fix: dependencies --- preprocessing/tile_masks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/preprocessing/tile_masks.py b/preprocessing/tile_masks.py index 17b4d9c..06eb106 100644 --- a/preprocessing/tile_masks.py +++ b/preprocessing/tile_masks.py @@ -10,10 +10,10 @@ import pyvips import ray from omegaconf import DictConfig +from openslide import OpenSlide from rationai.masks import process_items, tile_mask, write_big_tiff from rationai.mlkit import autolog, with_cli_args from rationai.mlkit.lightning.loggers import MLFlowLogger -from ratiopath.openslide import OpenSlide ray.init(runtime_env={"excludes": [".git", ".venv"]}) From a21c8cd4274333190e6c231d8f1b984f7c2fabe2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Kuku=C4=8Dka?= Date: Mon, 2 Feb 2026 07:09:43 +0000 Subject: [PATCH 12/22] fix: WIP --- preprocessing/tile_masks.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/preprocessing/tile_masks.py b/preprocessing/tile_masks.py index 06eb106..2146045 100644 --- a/preprocessing/tile_masks.py +++ b/preprocessing/tile_masks.py @@ -38,9 +38,12 @@ def process_slide( level: int, output_folder: Path, ) -> None: - print(slide["path"]) - with OpenSlide(slide["path"]) as slide_wsi: - mask_extent_x, mask_extent_y = slide_wsi.level_dimensions[level] + try: + with OpenSlide(slide["path"]) as slide_wsi: + mask_extent_x, mask_extent_y = slide_wsi.level_dimensions[level] + except Exception as e: + print(f"Failed to open slide {slide['path']}: {e}") + return slide_tiles = tiles[tiles["slide_id"] == slide.id] From 6a8d5caf59fcf898a3e9732f1b2d4fe1097b53f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Kuku=C4=8Dka?= Date: Mon, 2 Feb 2026 07:33:13 +0000 Subject: [PATCH 13/22] fix: WIP --- pyproject.toml | 2 -- uv.lock | 1 - 2 files changed, 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 37038f3..5ae892f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,6 @@ dependencies = [ "rationai-sdk", "rationai-mlkit", "rationai-masks", - "rationai-tiling", "ray>=2.52.1", "torch>=2.9.0", "torchmetrics>=1.8.2", @@ -30,7 +29,6 @@ run = ["rationai-kube-jobs"] [tool.uv.sources] rationai-mlkit = { git = "https://gitlab.ics.muni.cz/rationai/digital-pathology/libraries/mlkit.git" } rationai-masks = { git = "https://gitlab.ics.muni.cz/rationai/digital-pathology/libraries/masks.git" } -rationai-tiling = { git = "https://gitlab.ics.muni.cz/rationai/digital-pathology/libraries/tiling.git" } rationai-kube-jobs = { git = "ssh://git@gitlab.ics.muni.cz/rationai/infrastructure/kube-jobs" } rationai-sdk = { git = "https://gitlab.ics.muni.cz/rationai/infrastructure/rationai-sdk-python.git" } ratiopath = { git = "https://github.com/RationAI/ratiopath.git", branch = "fix/read-overlay" } diff --git a/uv.lock b/uv.lock index b8f001d..8a0fca8 100644 --- a/uv.lock +++ b/uv.lock @@ -2627,7 +2627,6 @@ dependencies = [ { name = "rationai-masks" }, { name = "rationai-mlkit" }, { name = "rationai-sdk" }, - { name = "rationai-tiling" }, { name = "ratiopath" }, { name = "ray" }, { name = "torch" }, From b52a14db8843e55a7d2050ad92831aebafdc9a7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Kuku=C4=8Dka?= Date: Mon, 2 Feb 2026 08:25:52 +0000 Subject: [PATCH 14/22] fix: WIP --- configs/preprocessing/tile_masks.yaml | 4 +- preprocessing/tile_masks.py | 10 ++--- preprocessing/tile_masks2.py | 64 +++++++++++++++++++++++++++ 3 files changed, 70 insertions(+), 8 deletions(-) create mode 100644 preprocessing/tile_masks2.py diff --git a/configs/preprocessing/tile_masks.yaml b/configs/preprocessing/tile_masks.yaml index e2bb3f8..5a76f7b 100644 --- a/configs/preprocessing/tile_masks.yaml +++ b/configs/preprocessing/tile_masks.yaml @@ -6,4 +6,6 @@ level: 3 metadata: run_name: "🎭 Tile Masks: ${dataset.institution}" - description: Tile masks for ${dataset.institution} at level ${dataset.level} \ No newline at end of file + description: Tile masks for ${dataset.institution} at level ${dataset.level} + hyperparams: + level: ${level} \ No newline at end of file diff --git a/preprocessing/tile_masks.py b/preprocessing/tile_masks.py index 2146045..9b5f83d 100644 --- a/preprocessing/tile_masks.py +++ b/preprocessing/tile_masks.py @@ -11,7 +11,7 @@ import ray from omegaconf import DictConfig from openslide import OpenSlide -from rationai.masks import process_items, tile_mask, write_big_tiff +from rationai.masks import process_items, slide_resolution, tile_mask, write_big_tiff from rationai.mlkit import autolog, with_cli_args from rationai.mlkit.lightning.loggers import MLFlowLogger @@ -41,6 +41,7 @@ def process_slide( try: with OpenSlide(slide["path"]) as slide_wsi: mask_extent_x, mask_extent_y = slide_wsi.level_dimensions[level] + mpp_x, mpp_y = slide_resolution(slide_wsi, level) except Exception as e: print(f"Failed to open slide {slide['path']}: {e}") return @@ -74,12 +75,7 @@ def process_slide( mask_path = output_folder / folder / f"{Path(slide['path']).stem}.tiff" mask_path.parent.mkdir(parents=True, exist_ok=True) - write_big_tiff( - mask, - mask_path, - mpp_x=slide["mpp_x"], - mpp_y=slide["mpp_y"], - ) + write_big_tiff(mask, mask_path, mpp_x=mpp_x, mpp_y=mpp_y) @with_cli_args(["+preprocessing=tile_masks"]) diff --git a/preprocessing/tile_masks2.py b/preprocessing/tile_masks2.py new file mode 100644 index 0000000..fc16704 --- /dev/null +++ b/preprocessing/tile_masks2.py @@ -0,0 +1,64 @@ +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import cast + +import hydra +import mlflow.artifacts +import pandas as pd +import pyvips +import ray +from omegaconf import DictConfig +from openslide import OpenSlide +from rationai.masks import ( + process_items, + slide_resolution, + tissue_mask, + write_big_tiff, +) +from rationai.mlkit import autolog, with_cli_args +from rationai.mlkit.lightning.loggers import MLFlowLogger + + +ray.init(runtime_env={"excludes": [".git", ".venv"]}) + + +@ray.remote(memory=4 * 1024**3) +def process_slide(slide_path: str, level: int, output_path: Path) -> None: + with OpenSlide(slide_path) as slide: + mpp_x, mpp_y = slide_resolution(slide, level) + + slide = cast("pyvips.Image", pyvips.Image.new_from_file(slide_path, level=level)) + mask = tissue_mask(slide, mpp=(mpp_x + mpp_y) / 2) + mask_path = output_path / Path(slide_path).with_suffix(".tiff").name + + write_big_tiff(mask, path=mask_path, mpp_x=mpp_x, mpp_y=mpp_y) + + +def download_dataset(uri: str) -> pd.DataFrame: + path = mlflow.artifacts.download_artifacts(artifact_uri=uri) + df = pd.read_csv(path) + return df + + +@with_cli_args(["+preprocessing=tile_masks"]) +@hydra.main(config_path="../configs", config_name="preprocessing", version_base=None) +@autolog +def main(config: DictConfig, logger: MLFlowLogger) -> None: + dataset = download_dataset(config.dataset.uri) + + with TemporaryDirectory() as output_dir: + process_items( + dataset["path"].to_list(), + process_item=process_slide, + fn_kwargs={ + "level": config.level, + "output_path": Path(output_dir), + }, + max_concurrent=config.max_concurrent, + ) + + logger.log_artifacts(local_dir=output_dir, artifact_path=config.artifact_path) + + +if __name__ == "__main__": + main() From 717c0838b75c533afb84138286cb6b954dc0fbab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Kuku=C4=8Dka?= Date: Mon, 2 Feb 2026 08:29:33 +0000 Subject: [PATCH 15/22] fix: WIP --- preprocessing/tile_masks2.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/preprocessing/tile_masks2.py b/preprocessing/tile_masks2.py index fc16704..1c2a6f1 100644 --- a/preprocessing/tile_masks2.py +++ b/preprocessing/tile_masks2.py @@ -1,3 +1,4 @@ +from collections.abc import Iterable from pathlib import Path from tempfile import TemporaryDirectory from typing import cast @@ -22,6 +23,18 @@ ray.init(runtime_env={"excludes": [".git", ".venv"]}) +def download_slide_tiles(uris: Iterable[str]) -> tuple[pd.DataFrame, pd.DataFrame]: + slidess, tiless = [], [] + for uri in uris: + path = mlflow.artifacts.download_artifacts(artifact_uri=uri) + slidess.append(pd.read_parquet(Path(path) / "slides.parquet")) + tiless.append(pd.read_parquet(Path(path) / "tiles.parquet")) + + slides = pd.concat(slidess).reset_index(drop=True) + tiles = pd.concat(tiless).reset_index(drop=True) + return slides, tiles + + @ray.remote(memory=4 * 1024**3) def process_slide(slide_path: str, level: int, output_path: Path) -> None: with OpenSlide(slide_path) as slide: @@ -44,11 +57,12 @@ def download_dataset(uri: str) -> pd.DataFrame: @hydra.main(config_path="../configs", config_name="preprocessing", version_base=None) @autolog def main(config: DictConfig, logger: MLFlowLogger) -> None: - dataset = download_dataset(config.dataset.uri) + slides, tiles = download_slide_tiles(config.dataset.uris.values()) + # dataset = download_dataset(config.dataset.uri) with TemporaryDirectory() as output_dir: process_items( - dataset["path"].to_list(), + slides["path"].to_list(), process_item=process_slide, fn_kwargs={ "level": config.level, From 5737ea45a0392e014fed5b5564a3314c8ae82995 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Kuku=C4=8Dka?= Date: Mon, 2 Feb 2026 08:37:44 +0000 Subject: [PATCH 16/22] fix: WIP --- preprocessing/tile_masks2.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/preprocessing/tile_masks2.py b/preprocessing/tile_masks2.py index 1c2a6f1..5dcea24 100644 --- a/preprocessing/tile_masks2.py +++ b/preprocessing/tile_masks2.py @@ -23,16 +23,16 @@ ray.init(runtime_env={"excludes": [".git", ".venv"]}) -def download_slide_tiles(uris: Iterable[str]) -> tuple[pd.DataFrame, pd.DataFrame]: - slidess, tiless = [], [] - for uri in uris: - path = mlflow.artifacts.download_artifacts(artifact_uri=uri) - slidess.append(pd.read_parquet(Path(path) / "slides.parquet")) - tiless.append(pd.read_parquet(Path(path) / "tiles.parquet")) +# def download_slide_tiles(uris: Iterable[str]) -> tuple[pd.DataFrame, pd.DataFrame]: +# slidess, tiless = [], [] +# for uri in uris: +# path = mlflow.artifacts.download_artifacts(artifact_uri=uri) +# slidess.append(pd.read_parquet(Path(path) / "slides.parquet")) +# tiless.append(pd.read_parquet(Path(path) / "tiles.parquet")) - slides = pd.concat(slidess).reset_index(drop=True) - tiles = pd.concat(tiless).reset_index(drop=True) - return slides, tiles +# slides = pd.concat(slidess).reset_index(drop=True) +# tiles = pd.concat(tiless).reset_index(drop=True) +# return slides, tiles @ray.remote(memory=4 * 1024**3) @@ -57,12 +57,12 @@ def download_dataset(uri: str) -> pd.DataFrame: @hydra.main(config_path="../configs", config_name="preprocessing", version_base=None) @autolog def main(config: DictConfig, logger: MLFlowLogger) -> None: - slides, tiles = download_slide_tiles(config.dataset.uris.values()) - # dataset = download_dataset(config.dataset.uri) + # slides, tiles = download_slide_tiles(config.dataset.uris.values()) + dataset = download_dataset(config.dataset.uri) with TemporaryDirectory() as output_dir: process_items( - slides["path"].to_list(), + dataset["path"].to_list(), process_item=process_slide, fn_kwargs={ "level": config.level, From e23d84c93dbb5a8e7652966492423f62cfa84a86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Kuku=C4=8Dka?= Date: Mon, 2 Feb 2026 08:49:49 +0000 Subject: [PATCH 17/22] fix: WIP --- configs/preprocessing/tile_masks.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/preprocessing/tile_masks.yaml b/configs/preprocessing/tile_masks.yaml index 5a76f7b..f41407e 100644 --- a/configs/preprocessing/tile_masks.yaml +++ b/configs/preprocessing/tile_masks.yaml @@ -6,6 +6,6 @@ level: 3 metadata: run_name: "🎭 Tile Masks: ${dataset.institution}" - description: Tile masks for ${dataset.institution} at level ${dataset.level} + description: Tile masks for ${dataset.institution} at level # ${dataset.level} hyperparams: - level: ${level} \ No newline at end of file + mask_level: ${level} \ No newline at end of file From 1355b49af8522deaf35ad266a76e1035944990fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Kuku=C4=8Dka?= Date: Mon, 2 Feb 2026 08:55:13 +0000 Subject: [PATCH 18/22] fix: WIP --- preprocessing/tile_masks2.py | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/preprocessing/tile_masks2.py b/preprocessing/tile_masks2.py index 5dcea24..45285e3 100644 --- a/preprocessing/tile_masks2.py +++ b/preprocessing/tile_masks2.py @@ -1,4 +1,3 @@ -from collections.abc import Iterable from pathlib import Path from tempfile import TemporaryDirectory from typing import cast @@ -23,18 +22,6 @@ ray.init(runtime_env={"excludes": [".git", ".venv"]}) -# def download_slide_tiles(uris: Iterable[str]) -> tuple[pd.DataFrame, pd.DataFrame]: -# slidess, tiless = [], [] -# for uri in uris: -# path = mlflow.artifacts.download_artifacts(artifact_uri=uri) -# slidess.append(pd.read_parquet(Path(path) / "slides.parquet")) -# tiless.append(pd.read_parquet(Path(path) / "tiles.parquet")) - -# slides = pd.concat(slidess).reset_index(drop=True) -# tiles = pd.concat(tiless).reset_index(drop=True) -# return slides, tiles - - @ray.remote(memory=4 * 1024**3) def process_slide(slide_path: str, level: int, output_path: Path) -> None: with OpenSlide(slide_path) as slide: @@ -53,11 +40,10 @@ def download_dataset(uri: str) -> pd.DataFrame: return df -@with_cli_args(["+preprocessing=tile_masks"]) +@with_cli_args(["+preprocessing=tissue_masks"]) @hydra.main(config_path="../configs", config_name="preprocessing", version_base=None) @autolog def main(config: DictConfig, logger: MLFlowLogger) -> None: - # slides, tiles = download_slide_tiles(config.dataset.uris.values()) dataset = download_dataset(config.dataset.uri) with TemporaryDirectory() as output_dir: From d7386da934a221d23e3e7d64f8fe6df8b820ac5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Kuku=C4=8Dka?= Date: Mon, 2 Feb 2026 09:12:43 +0000 Subject: [PATCH 19/22] fix: revert to yesterday --- configs/preprocessing/tile_masks.yaml | 2 +- preprocessing/tile_masks.py | 10 ++--- preprocessing/tile_masks2.py | 64 --------------------------- pyproject.toml | 2 + uv.lock | 1 + 5 files changed, 7 insertions(+), 72 deletions(-) delete mode 100644 preprocessing/tile_masks2.py diff --git a/configs/preprocessing/tile_masks.yaml b/configs/preprocessing/tile_masks.yaml index f41407e..3d36502 100644 --- a/configs/preprocessing/tile_masks.yaml +++ b/configs/preprocessing/tile_masks.yaml @@ -6,6 +6,6 @@ level: 3 metadata: run_name: "🎭 Tile Masks: ${dataset.institution}" - description: Tile masks for ${dataset.institution} at level # ${dataset.level} + description: Tile masks for ${dataset.institution} at level ${dataset.level} hyperparams: mask_level: ${level} \ No newline at end of file diff --git a/preprocessing/tile_masks.py b/preprocessing/tile_masks.py index 9b5f83d..a6e3a3f 100644 --- a/preprocessing/tile_masks.py +++ b/preprocessing/tile_masks.py @@ -38,13 +38,9 @@ def process_slide( level: int, output_folder: Path, ) -> None: - try: - with OpenSlide(slide["path"]) as slide_wsi: - mask_extent_x, mask_extent_y = slide_wsi.level_dimensions[level] - mpp_x, mpp_y = slide_resolution(slide_wsi, level) - except Exception as e: - print(f"Failed to open slide {slide['path']}: {e}") - return + with OpenSlide(slide["path"]) as slide_wsi: + mask_extent_x, mask_extent_y = slide_wsi.level_dimensions[level] + mpp_x, mpp_y = slide_resolution(slide_wsi, level) slide_tiles = tiles[tiles["slide_id"] == slide.id] diff --git a/preprocessing/tile_masks2.py b/preprocessing/tile_masks2.py deleted file mode 100644 index 45285e3..0000000 --- a/preprocessing/tile_masks2.py +++ /dev/null @@ -1,64 +0,0 @@ -from pathlib import Path -from tempfile import TemporaryDirectory -from typing import cast - -import hydra -import mlflow.artifacts -import pandas as pd -import pyvips -import ray -from omegaconf import DictConfig -from openslide import OpenSlide -from rationai.masks import ( - process_items, - slide_resolution, - tissue_mask, - write_big_tiff, -) -from rationai.mlkit import autolog, with_cli_args -from rationai.mlkit.lightning.loggers import MLFlowLogger - - -ray.init(runtime_env={"excludes": [".git", ".venv"]}) - - -@ray.remote(memory=4 * 1024**3) -def process_slide(slide_path: str, level: int, output_path: Path) -> None: - with OpenSlide(slide_path) as slide: - mpp_x, mpp_y = slide_resolution(slide, level) - - slide = cast("pyvips.Image", pyvips.Image.new_from_file(slide_path, level=level)) - mask = tissue_mask(slide, mpp=(mpp_x + mpp_y) / 2) - mask_path = output_path / Path(slide_path).with_suffix(".tiff").name - - write_big_tiff(mask, path=mask_path, mpp_x=mpp_x, mpp_y=mpp_y) - - -def download_dataset(uri: str) -> pd.DataFrame: - path = mlflow.artifacts.download_artifacts(artifact_uri=uri) - df = pd.read_csv(path) - return df - - -@with_cli_args(["+preprocessing=tissue_masks"]) -@hydra.main(config_path="../configs", config_name="preprocessing", version_base=None) -@autolog -def main(config: DictConfig, logger: MLFlowLogger) -> None: - dataset = download_dataset(config.dataset.uri) - - with TemporaryDirectory() as output_dir: - process_items( - dataset["path"].to_list(), - process_item=process_slide, - fn_kwargs={ - "level": config.level, - "output_path": Path(output_dir), - }, - max_concurrent=config.max_concurrent, - ) - - logger.log_artifacts(local_dir=output_dir, artifact_path=config.artifact_path) - - -if __name__ == "__main__": - main() diff --git a/pyproject.toml b/pyproject.toml index 5ae892f..37038f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,7 @@ dependencies = [ "rationai-sdk", "rationai-mlkit", "rationai-masks", + "rationai-tiling", "ray>=2.52.1", "torch>=2.9.0", "torchmetrics>=1.8.2", @@ -29,6 +30,7 @@ run = ["rationai-kube-jobs"] [tool.uv.sources] rationai-mlkit = { git = "https://gitlab.ics.muni.cz/rationai/digital-pathology/libraries/mlkit.git" } rationai-masks = { git = "https://gitlab.ics.muni.cz/rationai/digital-pathology/libraries/masks.git" } +rationai-tiling = { git = "https://gitlab.ics.muni.cz/rationai/digital-pathology/libraries/tiling.git" } rationai-kube-jobs = { git = "ssh://git@gitlab.ics.muni.cz/rationai/infrastructure/kube-jobs" } rationai-sdk = { git = "https://gitlab.ics.muni.cz/rationai/infrastructure/rationai-sdk-python.git" } ratiopath = { git = "https://github.com/RationAI/ratiopath.git", branch = "fix/read-overlay" } diff --git a/uv.lock b/uv.lock index 8a0fca8..b8f001d 100644 --- a/uv.lock +++ b/uv.lock @@ -2627,6 +2627,7 @@ dependencies = [ { name = "rationai-masks" }, { name = "rationai-mlkit" }, { name = "rationai-sdk" }, + { name = "rationai-tiling" }, { name = "ratiopath" }, { name = "ray" }, { name = "torch" }, From 8bec2ddfa95d295220ef2687419ab45f199f5029 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Kuku=C4=8Dka?= Date: Mon, 2 Feb 2026 12:58:28 +0000 Subject: [PATCH 20/22] feat: script --- configs/preprocessing/tile_masks.yaml | 5 +++-- scripts/preprocessing/tile_masks.py | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/configs/preprocessing/tile_masks.yaml b/configs/preprocessing/tile_masks.yaml index 3d36502..7967088 100644 --- a/configs/preprocessing/tile_masks.yaml +++ b/configs/preprocessing/tile_masks.yaml @@ -6,6 +6,7 @@ level: 3 metadata: run_name: "🎭 Tile Masks: ${dataset.institution}" - description: Tile masks for ${dataset.institution} at level ${dataset.level} + description: Tile masks for ${dataset.institution} at tiling level ${dataset.level} hyperparams: - mask_level: ${level} \ No newline at end of file + mask_level: ${level} + tiling_level: ${dataset.level} \ No newline at end of file diff --git a/scripts/preprocessing/tile_masks.py b/scripts/preprocessing/tile_masks.py index 53c7ec4..e9dcaa4 100644 --- a/scripts/preprocessing/tile_masks.py +++ b/scripts/preprocessing/tile_masks.py @@ -1,4 +1,4 @@ -from kube_jobs import submit_job +from kube_jobs import storage, submit_job submit_job( @@ -13,4 +13,5 @@ "uv sync --frozen", "uv run --active -m preprocessing.tile_masks +data=tiled/.../...", ], + storage=[storage.secure.Data], ) From f5d292e5129377ce90f6bac4eeb4be74051e9dab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Kuku=C4=8Dka?= Date: Thu, 12 Feb 2026 20:29:27 +0000 Subject: [PATCH 21/22] feat: confs --- configs/data/tiled/ftn/0_320.yaml | 11 ----------- configs/data/tiled/ftn/0_430.yaml | 11 ----------- configs/data/tiled/ftn/1_224.yaml | 11 ----------- configs/data/tiled/ftn/2_224.yaml | 11 ----------- configs/data/tiled/ikem/0_320.yaml | 11 ----------- configs/data/tiled/ikem/0_430.yaml | 11 ----------- configs/data/tiled/ikem/1_224.yaml | 11 ----------- configs/data/tiled/ikem/2_224.yaml | 11 ----------- configs/data/tiled/knl_patos/0_320.yaml | 10 ---------- configs/data/tiled/knl_patos/0_430.yaml | 10 ---------- configs/data/tiled/knl_patos/1_224.yaml | 10 ---------- configs/data/tiled/knl_patos/2_224.yaml | 10 ---------- configs/dataset/tiled/ftn/0_320.yaml | 11 +++++++++++ configs/dataset/tiled/ftn/0_430.yaml | 11 +++++++++++ configs/dataset/tiled/ftn/1_224.yaml | 11 +++++++++++ configs/dataset/tiled/ftn/2_224.yaml | 11 +++++++++++ configs/dataset/tiled/ikem/0_320.yaml | 11 +++++++++++ configs/dataset/tiled/ikem/0_430.yaml | 11 +++++++++++ configs/dataset/tiled/ikem/1_224.yaml | 11 +++++++++++ configs/dataset/tiled/ikem/2_224.yaml | 11 +++++++++++ configs/dataset/tiled/knl_patos/0_320.yaml | 10 ++++++++++ configs/dataset/tiled/knl_patos/0_430.yaml | 10 ++++++++++ configs/dataset/tiled/knl_patos/1_224.yaml | 10 ++++++++++ configs/dataset/tiled/knl_patos/2_224.yaml | 10 ++++++++++ scripts/preprocessing/tile_masks.py | 4 ++-- 25 files changed, 130 insertions(+), 130 deletions(-) delete mode 100644 configs/data/tiled/ftn/0_320.yaml delete mode 100644 configs/data/tiled/ftn/0_430.yaml delete mode 100644 configs/data/tiled/ftn/1_224.yaml delete mode 100644 configs/data/tiled/ftn/2_224.yaml delete mode 100644 configs/data/tiled/ikem/0_320.yaml delete mode 100644 configs/data/tiled/ikem/0_430.yaml delete mode 100644 configs/data/tiled/ikem/1_224.yaml delete mode 100644 configs/data/tiled/ikem/2_224.yaml delete mode 100644 configs/data/tiled/knl_patos/0_320.yaml delete mode 100644 configs/data/tiled/knl_patos/0_430.yaml delete mode 100644 configs/data/tiled/knl_patos/1_224.yaml delete mode 100644 configs/data/tiled/knl_patos/2_224.yaml create mode 100644 configs/dataset/tiled/ftn/0_320.yaml create mode 100644 configs/dataset/tiled/ftn/0_430.yaml create mode 100644 configs/dataset/tiled/ftn/1_224.yaml create mode 100644 configs/dataset/tiled/ftn/2_224.yaml create mode 100644 configs/dataset/tiled/ikem/0_320.yaml create mode 100644 configs/dataset/tiled/ikem/0_430.yaml create mode 100644 configs/dataset/tiled/ikem/1_224.yaml create mode 100644 configs/dataset/tiled/ikem/2_224.yaml create mode 100644 configs/dataset/tiled/knl_patos/0_320.yaml create mode 100644 configs/dataset/tiled/knl_patos/0_430.yaml create mode 100644 configs/dataset/tiled/knl_patos/1_224.yaml create mode 100644 configs/dataset/tiled/knl_patos/2_224.yaml diff --git a/configs/data/tiled/ftn/0_320.yaml b/configs/data/tiled/ftn/0_320.yaml deleted file mode 100644 index 06afedd..0000000 --- a/configs/data/tiled/ftn/0_320.yaml +++ /dev/null @@ -1,11 +0,0 @@ -# @package _global_ - -dataset: - institution: ftn - mpp: 0.17 - tile_extent: 320 - level: 0 - uris: - train: "mlflow-artifacts:/86/bbbe4603bc30495d85ac99093fc9269a/artifacts/train - ftn" # TODO update URI - test_preliminary: "mlflow-artifacts:/86/bbbe4603bc30495d85ac99093fc9269a/artifacts/test preliminary - ftn" # TODO update URI - test_final: "mlflow-artifacts:/86/bbbe4603bc30495d85ac99093fc9269a/artifacts/test final - ftn" # TODO update URI \ No newline at end of file diff --git a/configs/data/tiled/ftn/0_430.yaml b/configs/data/tiled/ftn/0_430.yaml deleted file mode 100644 index 2d1f3ff..0000000 --- a/configs/data/tiled/ftn/0_430.yaml +++ /dev/null @@ -1,11 +0,0 @@ -# @package _global_ - -dataset: - institution: ftn - mpp: 0.17 - tile_extent: 430 - level: 0 - uris: - train: "mlflow-artifacts:/86/de450f835f0d4462a91b35f4a79a500f/artifacts/train - ftn" # TODO update URI - test_preliminary: "mlflow-artifacts:/86/de450f835f0d4462a91b35f4a79a500f/artifacts/test preliminary - ftn" # TODO update URI - test_final: "mlflow-artifacts:/86/de450f835f0d4462a91b35f4a79a500f/artifacts/test final - ftn" # TODO update URI \ No newline at end of file diff --git a/configs/data/tiled/ftn/1_224.yaml b/configs/data/tiled/ftn/1_224.yaml deleted file mode 100644 index b8904a8..0000000 --- a/configs/data/tiled/ftn/1_224.yaml +++ /dev/null @@ -1,11 +0,0 @@ -# @package _global_ - -dataset: - institution: ftn - mpp: 0.52 - tile_extent: 224 - level: 1 - uris: - train: "mlflow-artifacts:/86/f85b64a7f96c41e38f86d84956e2dbe9/artifacts/train - ftn" # TODO update URI - test_preliminary: "mlflow-artifacts:/86/f85b64a7f96c41e38f86d84956e2dbe9/artifacts/test preliminary - ftn" # TODO update URI - test_final: "mlflow-artifacts:/86/f85b64a7f96c41e38f86d84956e2dbe9/artifacts/test final - ftn" # TODO update URI \ No newline at end of file diff --git a/configs/data/tiled/ftn/2_224.yaml b/configs/data/tiled/ftn/2_224.yaml deleted file mode 100644 index 8e89157..0000000 --- a/configs/data/tiled/ftn/2_224.yaml +++ /dev/null @@ -1,11 +0,0 @@ -# @package _global_ - -dataset: - institution: ftn - mpp: 1.55 - tile_extent: 224 - level: 2 - uris: - train: "mlflow-artifacts:/86/5814484b6cd7467e9d712889655479af/artifacts/train - ftn" # TODO update URI - test_preliminary: "mlflow-artifacts:/86/5814484b6cd7467e9d712889655479af/artifacts/test preliminary - ftn" # TODO update URI - test_final: "mlflow-artifacts:/86/5814484b6cd7467e9d712889655479af/artifacts/test final - ftn" # TODO update URI \ No newline at end of file diff --git a/configs/data/tiled/ikem/0_320.yaml b/configs/data/tiled/ikem/0_320.yaml deleted file mode 100644 index 7951f25..0000000 --- a/configs/data/tiled/ikem/0_320.yaml +++ /dev/null @@ -1,11 +0,0 @@ -# @package _global_ - -dataset: - institution: ikem - mpp: 0.17 - tile_extent: 320 - level: 0 - uris: - train: "mlflow-artifacts:/86/4486e598446d412d926ac66dadb35e51/artifacts/train - ikem" # TODO update URI - test_preliminary: "mlflow-artifacts:/86/4486e598446d412d926ac66dadb35e51/artifacts/test preliminary - ikem" # TODO update URI - test_final: "mlflow-artifacts:/86/4486e598446d412d926ac66dadb35e51/artifacts/test final - ikem" # TODO update URI \ No newline at end of file diff --git a/configs/data/tiled/ikem/0_430.yaml b/configs/data/tiled/ikem/0_430.yaml deleted file mode 100644 index d63dfe0..0000000 --- a/configs/data/tiled/ikem/0_430.yaml +++ /dev/null @@ -1,11 +0,0 @@ -# @package _global_ - -dataset: - institution: ikem - mpp: 0.17 - tile_extent: 430 - level: 0 - uris: - train: "mlflow-artifacts:/86/fd112e63819c49d999502542b35bfce1/artifacts/train - ikem" # TODO update URI - test_preliminary: "mlflow-artifacts:/86/fd112e63819c49d999502542b35bfce1/artifacts/test preliminary - ikem" # TODO update URI - test_final: "mlflow-artifacts:/86/fd112e63819c49d999502542b35bfce1/artifacts/test final - ikem" # TODO update URI \ No newline at end of file diff --git a/configs/data/tiled/ikem/1_224.yaml b/configs/data/tiled/ikem/1_224.yaml deleted file mode 100644 index 467d3a4..0000000 --- a/configs/data/tiled/ikem/1_224.yaml +++ /dev/null @@ -1,11 +0,0 @@ -# @package _global_ - -dataset: - institution: ikem - mpp: 0.52 - tile_extent: 224 - level: 1 - uris: - train: "mlflow-artifacts:/86/ece822e7c0e3416f97212267c773c8ac/artifacts/train - ikem" # TODO update URI - test_preliminary: "mlflow-artifacts:/86/ece822e7c0e3416f97212267c773c8ac/artifacts/test preliminary - ikem" # TODO update URI - test_final: "mlflow-artifacts:/86/ece822e7c0e3416f97212267c773c8ac/artifacts/test final - ikem" # TODO update URI \ No newline at end of file diff --git a/configs/data/tiled/ikem/2_224.yaml b/configs/data/tiled/ikem/2_224.yaml deleted file mode 100644 index 0ec13aa..0000000 --- a/configs/data/tiled/ikem/2_224.yaml +++ /dev/null @@ -1,11 +0,0 @@ -# @package _global_ - -dataset: - institution: ikem - mpp: 1.55 - tile_extent: 224 - level: 2 - uris: - train: "mlflow-artifacts:/86/0c09e1c61d294fa3877b6b21703bab2f/artifacts/train - ikem" # TODO update URI - test_preliminary: "mlflow-artifacts:/86/0c09e1c61d294fa3877b6b21703bab2f/artifacts/test preliminary - ikem" # TODO update URI - test_final: "mlflow-artifacts:/86/0c09e1c61d294fa3877b6b21703bab2f/artifacts/test final - ikem" # TODO update URI \ No newline at end of file diff --git a/configs/data/tiled/knl_patos/0_320.yaml b/configs/data/tiled/knl_patos/0_320.yaml deleted file mode 100644 index e2c7d5b..0000000 --- a/configs/data/tiled/knl_patos/0_320.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# @package _global_ - -dataset: - institution: knl_patos - mpp: 0.17 - tile_extent: 320 - level: 0 - uris: - test_preliminary: "mlflow-artifacts:/86/7b9a446145b14965981bbac88e8e2c8b/artifacts/test preliminary - knl_patos" # TODO update URI - test_final: "mlflow-artifacts:/86/7b9a446145b14965981bbac88e8e2c8b/artifacts/test final - knl_patos" # TODO update URI diff --git a/configs/data/tiled/knl_patos/0_430.yaml b/configs/data/tiled/knl_patos/0_430.yaml deleted file mode 100644 index dafd755..0000000 --- a/configs/data/tiled/knl_patos/0_430.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# @package _global_ - -dataset: - institution: knl_patos - mpp: 0.17 - tile_extent: 430 - level: 0 - uris: - test_preliminary: "mlflow-artifacts:/86/eb29255c944d4dad926160a7cb102ad9/artifacts/test preliminary - knl_patos" # TODO update URI - test_final: "mlflow-artifacts:/86/eb29255c944d4dad926160a7cb102ad9/artifacts/test final - knl_patos" # TODO update URI \ No newline at end of file diff --git a/configs/data/tiled/knl_patos/1_224.yaml b/configs/data/tiled/knl_patos/1_224.yaml deleted file mode 100644 index 3c86e8e..0000000 --- a/configs/data/tiled/knl_patos/1_224.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# @package _global_ - -dataset: - institution: knl_patos - mpp: 0.52 - tile_extent: 224 - level: 1 - uris: - test_preliminary: "mlflow-artifacts:/86/6782155362d54ecc9f1beccb4362d359/artifacts/test preliminary - knl_patos" # TODO update URI - test_final: "mlflow-artifacts:/86/6782155362d54ecc9f1beccb4362d359/artifacts/test final - knl_patos" # TODO update URI \ No newline at end of file diff --git a/configs/data/tiled/knl_patos/2_224.yaml b/configs/data/tiled/knl_patos/2_224.yaml deleted file mode 100644 index 3061d80..0000000 --- a/configs/data/tiled/knl_patos/2_224.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# @package _global_ - -dataset: - institution: knl_patos - mpp: 1.55 - tile_extent: 224 - level: 2 - uris: - test_preliminary: "mlflow-artifacts:/86/d7486bb6b667433989c3ce1c8ce31d60/artifacts/test preliminary - knl_patos" # TODO update URI - test_final: "mlflow-artifacts:/86/d7486bb6b667433989c3ce1c8ce31d60/artifacts/test final - knl_patos" # TODO update URI \ No newline at end of file diff --git a/configs/dataset/tiled/ftn/0_320.yaml b/configs/dataset/tiled/ftn/0_320.yaml new file mode 100644 index 0000000..0f91d92 --- /dev/null +++ b/configs/dataset/tiled/ftn/0_320.yaml @@ -0,0 +1,11 @@ +defaults: + - /dataset/processed_w_masks/ftn@_here_ + - _self_ + +mpp: 0.17 +tile_extent: 320 +level: 0 +uris: + train: "mlflow-artifacts:/86/bbbe4603bc30495d85ac99093fc9269a/artifacts/train - ftn" # TODO update URI + test_preliminary: "mlflow-artifacts:/86/bbbe4603bc30495d85ac99093fc9269a/artifacts/test preliminary - ftn" # TODO update URI + test_final: "mlflow-artifacts:/86/bbbe4603bc30495d85ac99093fc9269a/artifacts/test final - ftn" # TODO update URI \ No newline at end of file diff --git a/configs/dataset/tiled/ftn/0_430.yaml b/configs/dataset/tiled/ftn/0_430.yaml new file mode 100644 index 0000000..4c549e1 --- /dev/null +++ b/configs/dataset/tiled/ftn/0_430.yaml @@ -0,0 +1,11 @@ +defaults: + - /dataset/processed_w_masks/ftn@_here_ + - _self_ + +mpp: 0.17 +tile_extent: 430 +level: 0 +uris: + train: "mlflow-artifacts:/86/de450f835f0d4462a91b35f4a79a500f/artifacts/train - ftn" # TODO update URI + test_preliminary: "mlflow-artifacts:/86/de450f835f0d4462a91b35f4a79a500f/artifacts/test preliminary - ftn" # TODO update URI + test_final: "mlflow-artifacts:/86/de450f835f0d4462a91b35f4a79a500f/artifacts/test final - ftn" # TODO update URI \ No newline at end of file diff --git a/configs/dataset/tiled/ftn/1_224.yaml b/configs/dataset/tiled/ftn/1_224.yaml new file mode 100644 index 0000000..cd2159a --- /dev/null +++ b/configs/dataset/tiled/ftn/1_224.yaml @@ -0,0 +1,11 @@ +defaults: + - /dataset/processed_w_masks/ftn@_here_ + - _self_ + +mpp: 0.52 +tile_extent: 224 +level: 1 +uris: + train: "mlflow-artifacts:/86/f85b64a7f96c41e38f86d84956e2dbe9/artifacts/train - ftn" # TODO update URI + test_preliminary: "mlflow-artifacts:/86/f85b64a7f96c41e38f86d84956e2dbe9/artifacts/test preliminary - ftn" # TODO update URI + test_final: "mlflow-artifacts:/86/f85b64a7f96c41e38f86d84956e2dbe9/artifacts/test final - ftn" # TODO update URI \ No newline at end of file diff --git a/configs/dataset/tiled/ftn/2_224.yaml b/configs/dataset/tiled/ftn/2_224.yaml new file mode 100644 index 0000000..11e9f49 --- /dev/null +++ b/configs/dataset/tiled/ftn/2_224.yaml @@ -0,0 +1,11 @@ +defaults: + - /dataset/processed_w_masks/ftn@_here_ + - _self_ + +mpp: 1.55 +tile_extent: 224 +level: 2 +uris: + train: "mlflow-artifacts:/86/5814484b6cd7467e9d712889655479af/artifacts/train - ftn" # TODO update URI + test_preliminary: "mlflow-artifacts:/86/5814484b6cd7467e9d712889655479af/artifacts/test preliminary - ftn" # TODO update URI + test_final: "mlflow-artifacts:/86/5814484b6cd7467e9d712889655479af/artifacts/test final - ftn" # TODO update URI \ No newline at end of file diff --git a/configs/dataset/tiled/ikem/0_320.yaml b/configs/dataset/tiled/ikem/0_320.yaml new file mode 100644 index 0000000..78c6e2d --- /dev/null +++ b/configs/dataset/tiled/ikem/0_320.yaml @@ -0,0 +1,11 @@ +defaults: + - /dataset/processed_w_masks/ikem@_here_ + - _self_ + +mpp: 0.17 +tile_extent: 320 +level: 0 +uris: + train: "mlflow-artifacts:/86/4486e598446d412d926ac66dadb35e51/artifacts/train - ikem" # TODO update URI + test_preliminary: "mlflow-artifacts:/86/4486e598446d412d926ac66dadb35e51/artifacts/test preliminary - ikem" # TODO update URI + test_final: "mlflow-artifacts:/86/4486e598446d412d926ac66dadb35e51/artifacts/test final - ikem" # TODO update URI \ No newline at end of file diff --git a/configs/dataset/tiled/ikem/0_430.yaml b/configs/dataset/tiled/ikem/0_430.yaml new file mode 100644 index 0000000..6289e39 --- /dev/null +++ b/configs/dataset/tiled/ikem/0_430.yaml @@ -0,0 +1,11 @@ +defaults: + - /dataset/processed_w_masks/ikem@_here_ + - _self_ + +mpp: 0.17 +tile_extent: 430 +level: 0 +uris: + train: "mlflow-artifacts:/86/fd112e63819c49d999502542b35bfce1/artifacts/train - ikem" # TODO update URI + test_preliminary: "mlflow-artifacts:/86/fd112e63819c49d999502542b35bfce1/artifacts/test preliminary - ikem" # TODO update URI + test_final: "mlflow-artifacts:/86/fd112e63819c49d999502542b35bfce1/artifacts/test final - ikem" # TODO update URI \ No newline at end of file diff --git a/configs/dataset/tiled/ikem/1_224.yaml b/configs/dataset/tiled/ikem/1_224.yaml new file mode 100644 index 0000000..0a716fe --- /dev/null +++ b/configs/dataset/tiled/ikem/1_224.yaml @@ -0,0 +1,11 @@ +defaults: + - /dataset/processed_w_masks/ikem@_here_ + - _self_ + +mpp: 0.52 +tile_extent: 224 +level: 1 +uris: + train: "mlflow-artifacts:/86/ece822e7c0e3416f97212267c773c8ac/artifacts/train - ikem" # TODO update URI + test_preliminary: "mlflow-artifacts:/86/ece822e7c0e3416f97212267c773c8ac/artifacts/test preliminary - ikem" # TODO update URI + test_final: "mlflow-artifacts:/86/ece822e7c0e3416f97212267c773c8ac/artifacts/test final - ikem" # TODO update URI \ No newline at end of file diff --git a/configs/dataset/tiled/ikem/2_224.yaml b/configs/dataset/tiled/ikem/2_224.yaml new file mode 100644 index 0000000..4ee48d5 --- /dev/null +++ b/configs/dataset/tiled/ikem/2_224.yaml @@ -0,0 +1,11 @@ +defaults: + - /dataset/processed_w_masks/ikem@_here_ + - _self_ + +mpp: 1.55 +tile_extent: 224 +level: 2 +uris: + train: "mlflow-artifacts:/86/0c09e1c61d294fa3877b6b21703bab2f/artifacts/train - ikem" # TODO update URI + test_preliminary: "mlflow-artifacts:/86/0c09e1c61d294fa3877b6b21703bab2f/artifacts/test preliminary - ikem" # TODO update URI + test_final: "mlflow-artifacts:/86/0c09e1c61d294fa3877b6b21703bab2f/artifacts/test final - ikem" # TODO update URI \ No newline at end of file diff --git a/configs/dataset/tiled/knl_patos/0_320.yaml b/configs/dataset/tiled/knl_patos/0_320.yaml new file mode 100644 index 0000000..d81b100 --- /dev/null +++ b/configs/dataset/tiled/knl_patos/0_320.yaml @@ -0,0 +1,10 @@ +defaults: + - /dataset/processed_w_masks/knl_patos@_here_ + - _self_ + +mpp: 0.17 +tile_extent: 320 +level: 0 +uris: + test_preliminary: "mlflow-artifacts:/86/7b9a446145b14965981bbac88e8e2c8b/artifacts/test preliminary - knl_patos" # TODO update URI + test_final: "mlflow-artifacts:/86/7b9a446145b14965981bbac88e8e2c8b/artifacts/test final - knl_patos" # TODO update URI diff --git a/configs/dataset/tiled/knl_patos/0_430.yaml b/configs/dataset/tiled/knl_patos/0_430.yaml new file mode 100644 index 0000000..68b8e43 --- /dev/null +++ b/configs/dataset/tiled/knl_patos/0_430.yaml @@ -0,0 +1,10 @@ +defaults: + - /dataset/processed_w_masks/knl_patos@_here_ + - _self_ + +mpp: 0.17 +tile_extent: 430 +level: 0 +uris: + test_preliminary: "mlflow-artifacts:/86/eb29255c944d4dad926160a7cb102ad9/artifacts/test preliminary - knl_patos" # TODO update URI + test_final: "mlflow-artifacts:/86/eb29255c944d4dad926160a7cb102ad9/artifacts/test final - knl_patos" # TODO update URI \ No newline at end of file diff --git a/configs/dataset/tiled/knl_patos/1_224.yaml b/configs/dataset/tiled/knl_patos/1_224.yaml new file mode 100644 index 0000000..612c671 --- /dev/null +++ b/configs/dataset/tiled/knl_patos/1_224.yaml @@ -0,0 +1,10 @@ +defaults: + - /dataset/processed_w_masks/knl_patos@_here_ + - _self_ + +mpp: 0.52 +tile_extent: 224 +level: 1 +uris: + test_preliminary: "mlflow-artifacts:/86/6782155362d54ecc9f1beccb4362d359/artifacts/test preliminary - knl_patos" # TODO update URI + test_final: "mlflow-artifacts:/86/6782155362d54ecc9f1beccb4362d359/artifacts/test final - knl_patos" # TODO update URI \ No newline at end of file diff --git a/configs/dataset/tiled/knl_patos/2_224.yaml b/configs/dataset/tiled/knl_patos/2_224.yaml new file mode 100644 index 0000000..31785a9 --- /dev/null +++ b/configs/dataset/tiled/knl_patos/2_224.yaml @@ -0,0 +1,10 @@ +defaults: + - /dataset/processed_w_masks/knl_patos@_here_ + - _self_ + +mpp: 1.55 +tile_extent: 224 +level: 2 +uris: + test_preliminary: "mlflow-artifacts:/86/d7486bb6b667433989c3ce1c8ce31d60/artifacts/test preliminary - knl_patos" # TODO update URI + test_final: "mlflow-artifacts:/86/d7486bb6b667433989c3ce1c8ce31d60/artifacts/test final - knl_patos" # TODO update URI \ No newline at end of file diff --git a/scripts/preprocessing/tile_masks.py b/scripts/preprocessing/tile_masks.py index e9dcaa4..8bdcc53 100644 --- a/scripts/preprocessing/tile_masks.py +++ b/scripts/preprocessing/tile_masks.py @@ -8,10 +8,10 @@ cpu=64, memory="32Gi", script=[ - "git clone https://gitlab.ics.muni.cz/rationai/digital-pathology/pathology/ulcerative-colitis.git workdir", + "git clone https://github.com/RationAI/ulcerative-colitis.git workdir", "cd workdir", "uv sync --frozen", - "uv run --active -m preprocessing.tile_masks +data=tiled/.../...", + "uv run --active -m preprocessing.tile_masks +dataset=tiled/.../...", ], storage=[storage.secure.Data], ) From e4d7dadc2e3e7774a3d8fc21d82ddf69444948a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Kuku=C4=8Dka?= Date: Thu, 12 Feb 2026 20:37:19 +0000 Subject: [PATCH 22/22] feat: tiling uris --- configs/dataset/tiled/ftn/0_320.yaml | 2 +- configs/dataset/tiled/ftn/0_430.yaml | 2 +- configs/dataset/tiled/ftn/1_224.yaml | 2 +- configs/dataset/tiled/ftn/2_224.yaml | 2 +- configs/dataset/tiled/ikem/0_320.yaml | 2 +- configs/dataset/tiled/ikem/0_430.yaml | 2 +- configs/dataset/tiled/ikem/1_224.yaml | 2 +- configs/dataset/tiled/ikem/2_224.yaml | 2 +- configs/dataset/tiled/knl_patos/0_320.yaml | 2 +- configs/dataset/tiled/knl_patos/0_430.yaml | 2 +- configs/dataset/tiled/knl_patos/1_224.yaml | 2 +- configs/dataset/tiled/knl_patos/2_224.yaml | 2 +- preprocessing/tile_masks.py | 6 ++---- 13 files changed, 14 insertions(+), 16 deletions(-) diff --git a/configs/dataset/tiled/ftn/0_320.yaml b/configs/dataset/tiled/ftn/0_320.yaml index 0f91d92..ff91a77 100644 --- a/configs/dataset/tiled/ftn/0_320.yaml +++ b/configs/dataset/tiled/ftn/0_320.yaml @@ -5,7 +5,7 @@ defaults: mpp: 0.17 tile_extent: 320 level: 0 -uris: +tiling_uris: train: "mlflow-artifacts:/86/bbbe4603bc30495d85ac99093fc9269a/artifacts/train - ftn" # TODO update URI test_preliminary: "mlflow-artifacts:/86/bbbe4603bc30495d85ac99093fc9269a/artifacts/test preliminary - ftn" # TODO update URI test_final: "mlflow-artifacts:/86/bbbe4603bc30495d85ac99093fc9269a/artifacts/test final - ftn" # TODO update URI \ No newline at end of file diff --git a/configs/dataset/tiled/ftn/0_430.yaml b/configs/dataset/tiled/ftn/0_430.yaml index 4c549e1..79b9c8e 100644 --- a/configs/dataset/tiled/ftn/0_430.yaml +++ b/configs/dataset/tiled/ftn/0_430.yaml @@ -5,7 +5,7 @@ defaults: mpp: 0.17 tile_extent: 430 level: 0 -uris: +tiling_uris: train: "mlflow-artifacts:/86/de450f835f0d4462a91b35f4a79a500f/artifacts/train - ftn" # TODO update URI test_preliminary: "mlflow-artifacts:/86/de450f835f0d4462a91b35f4a79a500f/artifacts/test preliminary - ftn" # TODO update URI test_final: "mlflow-artifacts:/86/de450f835f0d4462a91b35f4a79a500f/artifacts/test final - ftn" # TODO update URI \ No newline at end of file diff --git a/configs/dataset/tiled/ftn/1_224.yaml b/configs/dataset/tiled/ftn/1_224.yaml index cd2159a..91a27da 100644 --- a/configs/dataset/tiled/ftn/1_224.yaml +++ b/configs/dataset/tiled/ftn/1_224.yaml @@ -5,7 +5,7 @@ defaults: mpp: 0.52 tile_extent: 224 level: 1 -uris: +tiling_uris: train: "mlflow-artifacts:/86/f85b64a7f96c41e38f86d84956e2dbe9/artifacts/train - ftn" # TODO update URI test_preliminary: "mlflow-artifacts:/86/f85b64a7f96c41e38f86d84956e2dbe9/artifacts/test preliminary - ftn" # TODO update URI test_final: "mlflow-artifacts:/86/f85b64a7f96c41e38f86d84956e2dbe9/artifacts/test final - ftn" # TODO update URI \ No newline at end of file diff --git a/configs/dataset/tiled/ftn/2_224.yaml b/configs/dataset/tiled/ftn/2_224.yaml index 11e9f49..55da441 100644 --- a/configs/dataset/tiled/ftn/2_224.yaml +++ b/configs/dataset/tiled/ftn/2_224.yaml @@ -5,7 +5,7 @@ defaults: mpp: 1.55 tile_extent: 224 level: 2 -uris: +tiling_uris: train: "mlflow-artifacts:/86/5814484b6cd7467e9d712889655479af/artifacts/train - ftn" # TODO update URI test_preliminary: "mlflow-artifacts:/86/5814484b6cd7467e9d712889655479af/artifacts/test preliminary - ftn" # TODO update URI test_final: "mlflow-artifacts:/86/5814484b6cd7467e9d712889655479af/artifacts/test final - ftn" # TODO update URI \ No newline at end of file diff --git a/configs/dataset/tiled/ikem/0_320.yaml b/configs/dataset/tiled/ikem/0_320.yaml index 78c6e2d..065bca1 100644 --- a/configs/dataset/tiled/ikem/0_320.yaml +++ b/configs/dataset/tiled/ikem/0_320.yaml @@ -5,7 +5,7 @@ defaults: mpp: 0.17 tile_extent: 320 level: 0 -uris: +tiling_uris: train: "mlflow-artifacts:/86/4486e598446d412d926ac66dadb35e51/artifacts/train - ikem" # TODO update URI test_preliminary: "mlflow-artifacts:/86/4486e598446d412d926ac66dadb35e51/artifacts/test preliminary - ikem" # TODO update URI test_final: "mlflow-artifacts:/86/4486e598446d412d926ac66dadb35e51/artifacts/test final - ikem" # TODO update URI \ No newline at end of file diff --git a/configs/dataset/tiled/ikem/0_430.yaml b/configs/dataset/tiled/ikem/0_430.yaml index 6289e39..98c8ff1 100644 --- a/configs/dataset/tiled/ikem/0_430.yaml +++ b/configs/dataset/tiled/ikem/0_430.yaml @@ -5,7 +5,7 @@ defaults: mpp: 0.17 tile_extent: 430 level: 0 -uris: +tiling_uris: train: "mlflow-artifacts:/86/fd112e63819c49d999502542b35bfce1/artifacts/train - ikem" # TODO update URI test_preliminary: "mlflow-artifacts:/86/fd112e63819c49d999502542b35bfce1/artifacts/test preliminary - ikem" # TODO update URI test_final: "mlflow-artifacts:/86/fd112e63819c49d999502542b35bfce1/artifacts/test final - ikem" # TODO update URI \ No newline at end of file diff --git a/configs/dataset/tiled/ikem/1_224.yaml b/configs/dataset/tiled/ikem/1_224.yaml index 0a716fe..0b40c0e 100644 --- a/configs/dataset/tiled/ikem/1_224.yaml +++ b/configs/dataset/tiled/ikem/1_224.yaml @@ -5,7 +5,7 @@ defaults: mpp: 0.52 tile_extent: 224 level: 1 -uris: +tiling_uris: train: "mlflow-artifacts:/86/ece822e7c0e3416f97212267c773c8ac/artifacts/train - ikem" # TODO update URI test_preliminary: "mlflow-artifacts:/86/ece822e7c0e3416f97212267c773c8ac/artifacts/test preliminary - ikem" # TODO update URI test_final: "mlflow-artifacts:/86/ece822e7c0e3416f97212267c773c8ac/artifacts/test final - ikem" # TODO update URI \ No newline at end of file diff --git a/configs/dataset/tiled/ikem/2_224.yaml b/configs/dataset/tiled/ikem/2_224.yaml index 4ee48d5..c68e542 100644 --- a/configs/dataset/tiled/ikem/2_224.yaml +++ b/configs/dataset/tiled/ikem/2_224.yaml @@ -5,7 +5,7 @@ defaults: mpp: 1.55 tile_extent: 224 level: 2 -uris: +tiling_uris: train: "mlflow-artifacts:/86/0c09e1c61d294fa3877b6b21703bab2f/artifacts/train - ikem" # TODO update URI test_preliminary: "mlflow-artifacts:/86/0c09e1c61d294fa3877b6b21703bab2f/artifacts/test preliminary - ikem" # TODO update URI test_final: "mlflow-artifacts:/86/0c09e1c61d294fa3877b6b21703bab2f/artifacts/test final - ikem" # TODO update URI \ No newline at end of file diff --git a/configs/dataset/tiled/knl_patos/0_320.yaml b/configs/dataset/tiled/knl_patos/0_320.yaml index d81b100..a84731d 100644 --- a/configs/dataset/tiled/knl_patos/0_320.yaml +++ b/configs/dataset/tiled/knl_patos/0_320.yaml @@ -5,6 +5,6 @@ defaults: mpp: 0.17 tile_extent: 320 level: 0 -uris: +tiling_uris: test_preliminary: "mlflow-artifacts:/86/7b9a446145b14965981bbac88e8e2c8b/artifacts/test preliminary - knl_patos" # TODO update URI test_final: "mlflow-artifacts:/86/7b9a446145b14965981bbac88e8e2c8b/artifacts/test final - knl_patos" # TODO update URI diff --git a/configs/dataset/tiled/knl_patos/0_430.yaml b/configs/dataset/tiled/knl_patos/0_430.yaml index 68b8e43..0e6bb2c 100644 --- a/configs/dataset/tiled/knl_patos/0_430.yaml +++ b/configs/dataset/tiled/knl_patos/0_430.yaml @@ -5,6 +5,6 @@ defaults: mpp: 0.17 tile_extent: 430 level: 0 -uris: +tiling_uris: test_preliminary: "mlflow-artifacts:/86/eb29255c944d4dad926160a7cb102ad9/artifacts/test preliminary - knl_patos" # TODO update URI test_final: "mlflow-artifacts:/86/eb29255c944d4dad926160a7cb102ad9/artifacts/test final - knl_patos" # TODO update URI \ No newline at end of file diff --git a/configs/dataset/tiled/knl_patos/1_224.yaml b/configs/dataset/tiled/knl_patos/1_224.yaml index 612c671..71e47d9 100644 --- a/configs/dataset/tiled/knl_patos/1_224.yaml +++ b/configs/dataset/tiled/knl_patos/1_224.yaml @@ -5,6 +5,6 @@ defaults: mpp: 0.52 tile_extent: 224 level: 1 -uris: +tiling_uris: test_preliminary: "mlflow-artifacts:/86/6782155362d54ecc9f1beccb4362d359/artifacts/test preliminary - knl_patos" # TODO update URI test_final: "mlflow-artifacts:/86/6782155362d54ecc9f1beccb4362d359/artifacts/test final - knl_patos" # TODO update URI \ No newline at end of file diff --git a/configs/dataset/tiled/knl_patos/2_224.yaml b/configs/dataset/tiled/knl_patos/2_224.yaml index 31785a9..b814d37 100644 --- a/configs/dataset/tiled/knl_patos/2_224.yaml +++ b/configs/dataset/tiled/knl_patos/2_224.yaml @@ -5,6 +5,6 @@ defaults: mpp: 1.55 tile_extent: 224 level: 2 -uris: +tiling_uris: test_preliminary: "mlflow-artifacts:/86/d7486bb6b667433989c3ce1c8ce31d60/artifacts/test preliminary - knl_patos" # TODO update URI test_final: "mlflow-artifacts:/86/d7486bb6b667433989c3ce1c8ce31d60/artifacts/test final - knl_patos" # TODO update URI \ No newline at end of file diff --git a/preprocessing/tile_masks.py b/preprocessing/tile_masks.py index a6e3a3f..e1cf8d0 100644 --- a/preprocessing/tile_masks.py +++ b/preprocessing/tile_masks.py @@ -16,9 +16,6 @@ from rationai.mlkit.lightning.loggers import MLFlowLogger -ray.init(runtime_env={"excludes": [".git", ".venv"]}) - - def download_slide_tiles(uris: Iterable[str]) -> tuple[pd.DataFrame, pd.DataFrame]: slidess, tiless = [], [] for uri in uris: @@ -78,7 +75,7 @@ def process_slide( @hydra.main(config_path="../configs", config_name="preprocessing", version_base=None) @autolog def main(config: DictConfig, logger: MLFlowLogger) -> None: - slides, tiles = download_slide_tiles(config.dataset.uris.values()) + slides, tiles = download_slide_tiles(config.dataset.tiling_uris.values()) tiles_ref = ray.put(tiles) with TemporaryDirectory() as output_dir: @@ -96,4 +93,5 @@ def main(config: DictConfig, logger: MLFlowLogger) -> None: if __name__ == "__main__": + ray.init(runtime_env={"excludes": [".git", ".venv"]}) main()