diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e0e1d99..211b1f6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,7 +4,7 @@ default_language_version: python: python3.10 repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v5.0.0 + rev: v6.0.0 hooks: - id: check-added-large-files - id: check-toml @@ -14,7 +14,7 @@ repos: - id: end-of-file-fixer - id: trailing-whitespace - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.8.2 + rev: v0.16.4 hooks: - id: ruff args: @@ -23,17 +23,17 @@ repos: files: ^source/ - id: ruff-format - repo: https://github.com/gitleaks/gitleaks - rev: v8.21.2 + rev: v8.30.0 hooks: - id: gitleaks - repo: https://github.com/codespell-project/codespell - rev: v2.3.0 + rev: v2.4.3 hooks: - id: codespell additional_dependencies: - tomli - repo: https://github.com/compilerla/conventional-pre-commit - rev: v3.6.0 + rev: v4.4.0 hooks: - id: conventional-pre-commit stages: [commit-msg] diff --git a/source/analysis/artifacts.py b/source/analysis/artifacts.py index 4f28936..163a2ec 100644 --- a/source/analysis/artifacts.py +++ b/source/analysis/artifacts.py @@ -1,5 +1,4 @@ from pathlib import Path -from typing import Optional import matplotlib.pyplot as plt import numpy as np @@ -51,7 +50,7 @@ def _set_save_path(self, dir_name: str) -> Path: save_path.mkdir(parents=True, exist_ok=True) return save_path - def _get_save_path(self, dir_name: str, file_name: str) -> Optional[Path]: + def _get_save_path(self, dir_name: str, file_name: str) -> Path | None: save_path = self._artifacts_path / dir_name / file_name if save_path.exists(): return save_path @@ -61,17 +60,17 @@ def set_dir_params(self, params: DirParams): self._dir_params = params self._artifacts_path = OUT_DIR / params_to_path(params) - def save_study(self, study: Study, band_reduction: Optional[int] = None): + def save_study(self, study: Study, band_reduction: int | None = None): self.save_metric(study.best_value, band_reduction) self.save_params(study.best_params, band_reduction) - def save_metric(self, metric: float, band_reduction: Optional[int] = None): + def save_metric(self, metric: float, band_reduction: int | None = None): save_path = self._set_save_path(STUDY) if not band_reduction: write_txt(str(metric), save_path / STUDY_BEST_METRIC) logger.info(f"Metric saved: {metric}") - def save_params(self, params: dict, band_reduction: Optional[int] = None): + def save_params(self, params: dict, band_reduction: int | None = None): save_path = self._set_save_path(STUDY) if band_reduction: write_json(params, save_path / STUDY_BEST_PARAMS_REDUCED) @@ -79,7 +78,7 @@ def save_params(self, params: dict, band_reduction: Optional[int] = None): write_json(params, save_path / STUDY_BEST_PARAMS) logger.info(f"Params saved: {params}") - def load_params(self, band_reduction: Optional[int] = None) -> Optional[dict]: + def load_params(self, band_reduction: int | None = None) -> dict | None: if band_reduction: save_path = self._get_save_path(STUDY, STUDY_BEST_PARAMS_REDUCED) else: @@ -100,16 +99,14 @@ def load_encoder(self) -> LabelEncoder: "Encoder could not be found. Make sure you train the model first (cmd: train_model)" ) - def load_unfit_model(self, band_reduction: Optional[int] = None) -> BaseEstimator: + def load_unfit_model(self, band_reduction: int | None = None) -> BaseEstimator: params = self.load_params(band_reduction) model = import_model(self._dir_params.estimator_name) if params: model.set_params(**params) return model - def save_metrics( - self, metrics: list[Metrics], band_reduction: Optional[int] = None - ): + def save_metrics(self, metrics: list[Metrics], band_reduction: int | None = None): table, metrics_all = present.generate_metrics_table(metrics) save_path = self._set_save_path(RESULTS) if band_reduction: @@ -119,7 +116,7 @@ def save_metrics( for m in metrics_all: write_txt(f"{m.mean:.2f}", save_path / f"{m.name}") - def load_metrics(self) -> Optional[dict[str, list[Metrics]]]: + def load_metrics(self) -> dict[str, list[Metrics]] | None: if not OUT_DIR: logger.warning("No output directory.") return None @@ -163,7 +160,7 @@ def save_shap_values(self, values: np.ndarray): save_path = self._set_save_path(RESULTS) np.save(save_path / RESULT_SHAP_VALUES, values) - def load_shap_values(self) -> Optional[np.ndarray]: + def load_shap_values(self) -> np.ndarray | None: save_path = self._get_save_path(RESULTS, RESULT_SHAP_VALUES) if save_path: return np.load(save_path) @@ -173,7 +170,7 @@ def load_shap_values(self) -> Optional[np.ndarray]: ) return None - def load_spectral_bands(self) -> Optional[np.ndarray]: + def load_spectral_bands(self) -> np.ndarray | None: SPECTRAL_BANDS_DIR.mkdir(parents=True, exist_ok=True) save_path = SPECTRAL_BANDS_DIR / "bands.npy" if save_path.exists(): diff --git a/source/analysis/extensions.py b/source/analysis/extensions.py index 5fc309f..2f3e8d3 100644 --- a/source/analysis/extensions.py +++ b/source/analysis/extensions.py @@ -1,5 +1,3 @@ -from typing import Optional - from source.analysis.base import BaseDataLoader, BaseOptParameters, BaseSklearnModel from source.core import settings from source.utils.utils import import_class @@ -13,13 +11,13 @@ DEFAULT_PARAMETERS_NAME = "ParamsSVC" -def import_data_loader(data_loader_name: Optional[str]) -> BaseDataLoader: +def import_data_loader(data_loader_name: str | None) -> BaseDataLoader: return import_class(data_loader_name, _DATA_LOADERS_DIR, DEFAULT_DATA_LOADER_NAME) -def import_model(model_name: Optional[str]) -> BaseSklearnModel: +def import_model(model_name: str | None) -> BaseSklearnModel: return import_class(model_name, _MODELS_DIR, DEFAULT_ESTIMATOR_NAME) -def import_parameters(parameters_name: Optional[str]) -> BaseOptParameters: +def import_parameters(parameters_name: str | None) -> BaseOptParameters: return import_class(parameters_name, _PARAMETERS_DIR, DEFAULT_PARAMETERS_NAME) diff --git a/source/analysis/metrics.py b/source/analysis/metrics.py index 19d9a62..f017f3a 100644 --- a/source/analysis/metrics.py +++ b/source/analysis/metrics.py @@ -33,7 +33,7 @@ @dataclass class MetricsContainer: def __init__(self): - self._metrics = {key: [] for key in METRIC_FUNC.keys()} + self._metrics = {key: [] for key in METRIC_FUNC} def calculate(self, y_test: ArrayLike, y_pred: ArrayLike): for metric_name, metric_func in METRIC_FUNC.items(): diff --git a/source/analysis/params.py b/source/analysis/params.py index c375f2c..9c95927 100644 --- a/source/analysis/params.py +++ b/source/analysis/params.py @@ -1,5 +1,4 @@ from pathlib import Path -from typing import Optional from pydantic import BaseModel, model_validator from source.analysis.extensions import DEFAULT_DATA_LOADER_NAME, DEFAULT_ESTIMATOR_NAME @@ -7,9 +6,9 @@ class DirParams(BaseModel): - estimator_name: Optional[str] = None + estimator_name: str | None = None estimator_is_optimized: bool = False - data_loader_name: Optional[str] = None + data_loader_name: str | None = None @model_validator(mode="after") def set_default_names(self) -> Self: diff --git a/source/analysis/plots.py b/source/analysis/plots.py index ec1e006..4239ae0 100644 --- a/source/analysis/plots.py +++ b/source/analysis/plots.py @@ -1,9 +1,7 @@ -from typing import Optional - -import matplotlib.cm as cm import matplotlib.pyplot as plt import numpy as np import umap +from matplotlib import cm from matplotlib.figure import Figure from matplotlib.lines import Line2D from sklearn.base import BaseEstimator, clone @@ -16,7 +14,7 @@ def relevant_features( - relevances: np.ndarray, bands: Optional[np.ndarray] = None + relevances: np.ndarray, bands: np.ndarray | None = None ) -> Figure: y = smooth_relevances(relevances) indices_by_relevance = np.argsort(y)[::-1] @@ -55,7 +53,7 @@ def relevant_features( def relevant_amplitudes( - relevances: np.ndarray, bands: Optional[np.ndarray] = None + relevances: np.ndarray, bands: np.ndarray | None = None ) -> Figure: if bands is None: bands = np.arange(len(relevances)) @@ -121,7 +119,7 @@ def signatures_display( encoder: LabelEncoder, X: np.ndarray, y: np.ndarray, - bands: Optional[np.ndarray] = None, + bands: np.ndarray | None = None, *, x_label: str = "Spectral bands", y_label: str = "", @@ -186,7 +184,7 @@ def umap_display( encoder: LabelEncoder, X: np.ndarray, y: np.ndarray, - meta: Optional[np.ndarray] = None, + meta: np.ndarray | None = None, ) -> Figure: y_encoded = np.array(encoder.fit_transform(y)) classes = encoder.classes_ diff --git a/source/analysis/present.py b/source/analysis/present.py index a3091ba..270af18 100644 --- a/source/analysis/present.py +++ b/source/analysis/present.py @@ -1,5 +1,4 @@ from collections import defaultdict -from typing import Optional from rich import print as RichPrint from rich.text import Text as RichText @@ -22,9 +21,9 @@ def generate_metrics_table(metrics: list[Metrics]): def display_metrics( metrics: dict[str, list[Metrics]], - model: Optional[str] = None, - do_optimize: Optional[bool] = None, - data_loader: Optional[str] = None, + model: str | None = None, + do_optimize: bool | None = None, + data_loader: str | None = None, ): def check_filter(params: DirParams) -> bool: if model is not None and params.estimator_name != model: diff --git a/source/cli/analysis.py b/source/cli/analysis.py index 6c5f1db..b5dc121 100644 --- a/source/cli/analysis.py +++ b/source/cli/analysis.py @@ -1,5 +1,4 @@ from functools import reduce -from typing import Optional import typer from rich import print @@ -19,7 +18,7 @@ @app.command() def test_load_data( - data_loader: Optional[str] = None, + data_loader: str | None = None, ): loader = import_data_loader(data_loader) loader.load_data() @@ -32,10 +31,10 @@ def test_load_data( @app.command() def train_model( - model: Optional[str] = None, - data_loader: Optional[str] = None, + model: str | None = None, + data_loader: str | None = None, do_optimize: bool = False, - parameters: Optional[list[str]] = None, + parameters: list[str] | None = None, ): artifacts.set_dir_params( DirParams( @@ -69,8 +68,8 @@ def train_model( @app.command() def generate_metrics( - model: Optional[str] = None, - data_loader: Optional[str] = None, + model: str | None = None, + data_loader: str | None = None, do_optimize: bool = False, ): artifacts.set_dir_params( @@ -93,8 +92,8 @@ def generate_metrics( @app.command() def generate_plots( - model: Optional[str] = None, - data_loader: Optional[str] = None, + model: str | None = None, + data_loader: str | None = None, do_optimize: bool = False, ): artifacts.set_dir_params( @@ -127,8 +126,8 @@ def generate_plots( @app.command() def calculate_relevances( - model: Optional[str] = None, - data_loader: Optional[str] = None, + model: str | None = None, + data_loader: str | None = None, do_optimize: bool = False, ): artifacts.set_dir_params( @@ -149,8 +148,8 @@ def calculate_relevances( @app.command() def display_metrics( - model: Optional[str] = None, - data_loader: Optional[str] = None, + model: str | None = None, + data_loader: str | None = None, do_optimize: bool = False, ): metrics_ = artifacts.load_metrics() diff --git a/source/cli/misc.py b/source/cli/misc.py index 789dd65..507f515 100644 --- a/source/cli/misc.py +++ b/source/cli/misc.py @@ -3,7 +3,6 @@ from source.misc.display_image import ( display_spectral_image, ) -from typing import Optional app = typer.Typer() @@ -14,5 +13,5 @@ def check_images(): @app.command() -def display_image(label: Optional[str] = None): +def display_image(label: str | None = None): display_spectral_image(label) diff --git a/source/cli/segment.py b/source/cli/segment.py index 75d58f7..3a72538 100644 --- a/source/cli/segment.py +++ b/source/cli/segment.py @@ -1,14 +1,4 @@ -from typing import Optional - import typer -from source.segmentation.artifacts import ( - load_all_selected_areas, - load_model, - load_transformation_matrix, - save_model, - save_selected_areas, - save_transformation_matrix, -) from source.misc.display_image import ( display_spectral_images_with_areas, ) @@ -21,6 +11,14 @@ select_areas_on_images, train_xgboost_model, ) +from source.segmentation.artifacts import ( + load_all_selected_areas, + load_model, + load_transformation_matrix, + save_model, + save_selected_areas, + save_transformation_matrix, +) app = typer.Typer() @@ -52,7 +50,7 @@ def train_model(): @app.command() -def segment_images(label: Optional[str] = None): +def segment_images(label: str | None = None): encoder_cam1, model_cam1, encoder_cam2, model_cam2 = load_model() matx = load_transformation_matrix() perform_segmentation( diff --git a/source/core/logger.py b/source/core/logger.py index bdf4676..8d98260 100644 --- a/source/core/logger.py +++ b/source/core/logger.py @@ -12,7 +12,7 @@ def emit(self, record: logging.LogRecord) -> None: logger_opt.log(record.levelname, record.getMessage()) -@lru_cache() +@lru_cache def setup_logger(debug: bool = False) -> loguru._Logger: # type: ignore LOGGING_LEVEL = logging.DEBUG if debug else logging.INFO logging.basicConfig( diff --git a/source/misc/check_images.py b/source/misc/check_images.py index 866d242..e417568 100644 --- a/source/misc/check_images.py +++ b/source/misc/check_images.py @@ -32,6 +32,6 @@ def check_spectral_images(): f" Number of labels: {len(labels_cam1)} (cam1), {len(labels_cam2)} (cam2)\n" ) msg += f" Number of unique labels: {len(labels_unique)} \n" - msg += f" Labels: \n{str(labels_unique)} \n" - msg += f" Duplicated labels: \n{str(labels_duplicated)} \n" + msg += f" Labels: \n{labels_unique!s} \n" + msg += f" Duplicated labels: \n{labels_duplicated!s} \n" logger.info(f"Report: \n{msg}") diff --git a/source/misc/display_image.py b/source/misc/display_image.py index c53d838..2a558a0 100644 --- a/source/misc/display_image.py +++ b/source/misc/display_image.py @@ -1,5 +1,3 @@ -from typing import Optional - import numpy as np from matplotlib import pyplot as plt from siapy.entities import Pixels @@ -14,7 +12,7 @@ ) -def display_spectral_image(label: Optional[str] = None): +def display_spectral_image(label: str | None = None): image_set_cam1, image_set_cam2 = read_spectral_images() if not image_set_cam1 or not image_set_cam2: diff --git a/source/segmentation/__init__.py b/source/segmentation/__init__.py index 20fed7a..664b6bc 100644 --- a/source/segmentation/__init__.py +++ b/source/segmentation/__init__.py @@ -6,11 +6,11 @@ from .transformator import find_transformation_between_images __all__ = [ + "convert_images_to_reflectance", + "convert_selected_areas_to_train_data", + "create_spectral_signatures", "find_transformation_between_images", + "perform_segmentation", "select_areas_on_images", "train_xgboost_model", - "convert_selected_areas_to_train_data", - "perform_segmentation", - "create_spectral_signatures", - "convert_images_to_reflectance", ] diff --git a/source/segmentation/model.py b/source/segmentation/model.py index 06eb472..4e2f3e0 100644 --- a/source/segmentation/model.py +++ b/source/segmentation/model.py @@ -2,14 +2,13 @@ from siapy.entities import Pixels from siapy.transformations import corregistrator from sklearn.preprocessing import LabelEncoder -from xgboost import XGBClassifier - from source.core import logger from source.misc.helpers import ( extract_labels_from_spectral_images, get_images_by_label, read_spectral_images, ) +from xgboost import XGBClassifier def convert_selected_areas_to_train_data( diff --git a/source/segmentation/segmentator.py b/source/segmentation/segmentator.py index bef5fd5..50e7937 100644 --- a/source/segmentation/segmentator.py +++ b/source/segmentation/segmentator.py @@ -11,14 +11,14 @@ ) from sklearn.preprocessing import LabelEncoder from source.core import logger, settings -from source.segmentation.artifacts import ( - read_spectral_images, - save_radiance_image, -) from source.misc.helpers import ( extract_labels_from_spectral_images, get_images_by_label, ) +from source.segmentation.artifacts import ( + read_spectral_images, + save_radiance_image, +) from xgboost import XGBClassifier @@ -150,9 +150,7 @@ def perform_segmentation( logger.info(f"Processed index: '__ {index} __' ") logger.info( - "Processed files:\n -> Camera 1 '{}'\n -> Camera 2 '{}'".format( - image_cam1.filepath.stem, image_cam2.filepath.stem - ) + f"Processed files:\n -> Camera 1 '{image_cam1.filepath.stem}'\n -> Camera 2 '{image_cam2.filepath.stem}'" ) selected_areas_cam1 = pixels_select_lasso(image_cam1) diff --git a/source/segmentation/selector.py b/source/segmentation/selector.py index 8f21c86..dfb30e3 100644 --- a/source/segmentation/selector.py +++ b/source/segmentation/selector.py @@ -2,7 +2,6 @@ from siapy.utils.plots import ( pixels_select_lasso, ) - from source.core import logger from source.misc.helpers import ( extract_labels_from_spectral_images, diff --git a/source/utils/utils.py b/source/utils/utils.py index 0a99d9d..eb105db 100644 --- a/source/utils/utils.py +++ b/source/utils/utils.py @@ -3,43 +3,44 @@ import pickle import sys from collections import OrderedDict +from collections.abc import Generator from pathlib import Path -from typing import Any, Generator, Optional, Union +from typing import Any from source.core import logger -def read_json(fname: Union[str, Path]) -> OrderedDict: +def read_json(fname: str | Path) -> OrderedDict: fname = Path(fname) with fname.open("rt") as handle: return json.load(handle, object_hook=OrderedDict) -def write_json(content: Any, fname: Union[str, Path]) -> None: +def write_json(content: Any, fname: str | Path) -> None: fname = Path(fname) with fname.open("wt") as handle: json.dump(content, handle, indent=4, sort_keys=False) -def write_pickle(content: Any, fname: Union[str, Path]) -> None: +def write_pickle(content: Any, fname: str | Path) -> None: fname = Path(fname) with open(fname, "wb") as f: pickle.dump(content, f) -def read_pickle(fname: Union[str, Path]) -> Any: +def read_pickle(fname: str | Path) -> Any: fname = Path(fname) with open(fname, "rb") as f: return pickle.load(f) -def write_txt(content: str, fname: Union[str, Path]) -> None: +def write_txt(content: str, fname: str | Path) -> None: fname = Path(fname) with fname.open("w") as handle: handle.write(content) -def read_txt(fname: Union[str, Path]) -> str: +def read_txt(fname: str | Path) -> str: fname = Path(fname) with fname.open("r") as handle: return handle.read() @@ -58,7 +59,7 @@ def dict_zip(*dicts: dict[str, Any]) -> Generator[tuple[str, Any, Any], None, No def import_class( - class_name: Optional[str], + class_name: str | None, directory: Path, default_class_name: str, ) -> Any: