diff --git a/.github/scripts/run_smoke.py b/.github/scripts/run_smoke.py new file mode 100644 index 0000000..73d9576 --- /dev/null +++ b/.github/scripts/run_smoke.py @@ -0,0 +1,113 @@ +""" +Run the workspace smoke test suite. + +Reads `smoke_tests.txt` from the workspace root and `config/build/env_vars.yaml` +for per-script env var overrides, then runs each listed script with the +appropriate environment. Continues through failures and exits non-zero +if any script failed. + +The env resolution itself is NOT implemented here: it is PyAutoBuild's +`autobuild/env_config.py`, imported below. This file used to carry a copy, and +the copy had already drifted (its `load_env_config` hardcoded +`config/build/env_vars.yaml`, so the PR gate was structurally unable to read +the release profile — the seed incident's failure mode 4/7). One resolver +means the PR gate and the release runner cannot disagree about what a script's +environment is. See PyAutoBuild docs/env_profile_redesign.md §5 (#161 step 2). + +Mirrors the logic of the `/smoke-test` skill so CI and local runs stay +in sync. +""" + +from __future__ import annotations + +import subprocess +import sys +import time +from pathlib import Path + + +WORKSPACE = Path(__file__).resolve().parents[2] +SMOKE_FILE = WORKSPACE / "smoke_tests.txt" +ENV_VARS_FILE = WORKSPACE / "config" / "build" / "env_vars.yaml" +SCRIPTS_DIR = WORKSPACE / "scripts" + +# CI puts PyAutoBuild/autobuild on PYTHONPATH (PyAutoHeart's reusable +# smoke-tests.yml clones it alongside the dependency chain); for local runs, +# fall back to the sibling checkout. +try: + from env_config import build_env_for_script, load_env_config +except ImportError: # pragma: no cover - local-run fallback + sys.path.insert(0, str(WORKSPACE.parent / "PyAutoBuild" / "autobuild")) + from env_config import build_env_for_script, load_env_config + + +def load_smoke_scripts() -> list[str]: + scripts: list[str] = [] + for line in SMOKE_FILE.read_text().splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + scripts.append(line) + return scripts + + +def load_cfg() -> dict | None: + """Parsed env profile, or None when the workspace has none. + + None flows through build_env_for_script -> None -> subprocess inherits the + parent environment, which is what the old local copy's empty-config path + did by hand. + """ + if not ENV_VARS_FILE.exists(): + return None + return load_env_config(ENV_VARS_FILE) + + +def run_one(script_rel: str, cfg: dict | None) -> tuple[str, int, float, str]: + env = build_env_for_script(Path(script_rel), cfg) + script_path = SCRIPTS_DIR / script_rel + t0 = time.time() + result = subprocess.run( + [sys.executable, str(script_path)], + cwd=str(WORKSPACE), + env=env, + capture_output=True, + text=True, + ) + elapsed = time.time() - t0 + output = result.stdout + result.stderr + return script_rel, result.returncode, elapsed, output + + +def main() -> int: + if not SMOKE_FILE.exists(): + print(f"ERROR: no smoke_tests.txt at {SMOKE_FILE}", file=sys.stderr) + return 1 + scripts = load_smoke_scripts() + if not scripts: + print("No smoke test scripts listed.") + return 0 + cfg = load_cfg() + + print(f"Running {len(scripts)} smoke test script(s) from {SMOKE_FILE.name}\n") + failures: list[tuple[str, int, str]] = [] + for script_rel in scripts: + print(f"::group::{script_rel}") + name, rc, elapsed, output = run_one(script_rel, cfg) + print(output, end="") + status = "PASS" if rc == 0 else f"FAIL (exit {rc})" + print(f"\n[{status}] {name} — {elapsed:.1f}s") + print("::endgroup::") + if rc != 0: + failures.append((name, rc, output)) + + total = len(scripts) + passed = total - len(failures) + print(f"\n=== Smoke test summary: {passed}/{total} passed ===") + for name, rc, _ in failures: + print(f" FAIL {name} (exit {rc})") + return 0 if not failures else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/smoke_install.sh b/.github/scripts/smoke_install.sh new file mode 100755 index 0000000..3b7c8ef --- /dev/null +++ b/.github/scripts/smoke_install.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# Workspace-owned install epilogue for the reusable Smoke Tests workflow +# (PyAutoHeart/.github/workflows/smoke-tests.yml). Runs with cwd at the +# checkout root (the dependency chain is cloned beside `workspace/`). +set -e + +# arcticpy (the C++ arctic clocking code) is a hard import of autocti but not +# a pip dependency: its sdist is source-only (needs libgsl-dev + cython) and +# its own requirements downgrade numpy below 2.0. +sudo apt-get update && sudo apt-get install -y libgsl-dev +pip install numpy cython +pip install arcticpy==2.6 --no-build-isolation --no-deps + +pip install ./PyAutoConf ./PyAutoFit ./PyAutoArray ./PyAutoCTI +pip install "./PyAutoArray[optional]" +# The re-resolution above can upgrade autoconf to the stale PyPI release; +# pin the local source last so recent autoconf APIs are importable. +pip install --force-reinstall --no-deps ./PyAutoConf diff --git a/.github/workflows/smoke_tests.yml b/.github/workflows/smoke_tests.yml new file mode 100644 index 0000000..345dc56 --- /dev/null +++ b/.github/workflows/smoke_tests.yml @@ -0,0 +1,15 @@ +name: Smoke Tests + +# Thin caller for PyAutoHeart's reusable smoke-test workflow. The ceremony +# (chain checkout, branch matching, runner, Slack) lives centrally; this +# workspace declares its dependency chain and owns its install epilogue +# (.github/scripts/smoke_install.sh) and runner (.github/scripts/run_smoke.py). + +on: [push, pull_request] + +jobs: + smoke: + uses: PyAutoLabs/PyAutoHeart/.github/workflows/smoke-tests.yml@main + with: + chain: "PyAutoConf PyAutoFit PyAutoArray PyAutoCTI" + secrets: inherit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..45cbc90 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +output/ +*.sqlite +__pycache__/ +*.pyc +/*.png diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..8cc83da --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,51 @@ +# PyAutoCTI Workspace Test — Agent Instructions + +This is the integration-test suite for **PyAutoCTI**, run in CI to verify the core library works +end-to-end. It is **not** a user-facing workspace — see `../autocti_workspace` for examples and +tutorials. These are the canonical, agent-agnostic instructions for this repo. + +Dependencies: `autocti`, `autofit`, `autoarray`, and **arcticpy** (source-only C++ sdist — install +with `pip install arcticpy==2.6 --no-build-isolation --no-deps` after numpy+cython, with +`libgsl-dev` present; see `PyAutoCTI/AGENTS.md`). + +## Repository Structure + +``` +scripts/ Integration-test scripts run in CI + dataset_1d/model_fit.py 1D calibration: simulate -> factor-graph fit -> aggregator round-trip + imaging_ci/model_fit.py 2D charge injection: simulate -> factor-graph fit -> result inspection + plot/subplots.py Drives the autocti.plot function surface to files +legacy/ Euclid VIS heritage (2022-2023, pre-resurrection API — not runnable; + see legacy/README.md) +config/ Current PyAutoCTI config (mirrors autocti_workspace) +config/build/env_vars.yaml Per-script env for smoke runs (PYAUTO_TEST_MODE=2 defaults) +smoke_tests.txt The curated smoke list (small on purpose) +``` + +## Running + +```bash +python .github/scripts/run_smoke.py # the smoke list, with env_vars.yaml applied +python scripts/dataset_1d/model_fit.py # one script, real search (no env applied) +``` + +CI runs the smoke list through PyAutoHeart's reusable smoke workflow (thin caller in +`.github/workflows/smoke_tests.yml`, chain `PyAutoConf PyAutoFit PyAutoArray PyAutoCTI`; the +arcticpy build lives in `.github/scripts/smoke_install.sh`). + +## Conventions + +- Keep `smoke_tests.txt` a **small curated subset** — do not mass-promote scripts. +- Integration scripts are self-contained (simulate their own data, small shapes) and single-trap + (identical-prior ordered-trap models tie at prior medians under the `PYAUTO_TEST_MODE=2` + bypass and raise their own assertion — a filed autofit issue; avoid the pattern here). +- The test-mode knob is `PYAUTO_TEST_MODE` (`2` bypasses sampling); `PYAUTOFIT_TEST_MODE` + does not exist. +- Never edit `legacy/` — it is preserved Euclid VIS history. + +## Never rewrite history + +Never rewrite pushed history on any repo with a remote — no `git init` over a +tracked repo, no force-push to `main`, no fresh-start "Initial commit", no +`filter-repo` / `filter-branch` / `rebase -i` on pushed branches. To get a +clean tree: `git fetch origin && git reset --hard origin/main && git clean -fd`. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..b90aefe --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,7 @@ +# autocti_workspace_test — agent instructions + +The canonical, agent-agnostic instructions live in `AGENTS.md`. Claude Code loads them via the +import below; if your tool does not process `@`-imports, open `AGENTS.md` in this directory and +read it directly. + +@AGENTS.md diff --git a/config/build/env_vars.yaml b/config/build/env_vars.yaml new file mode 100644 index 0000000..2831f9e --- /dev/null +++ b/config/build/env_vars.yaml @@ -0,0 +1,10 @@ +# Per-script environment variable configuration for automated runs +# (smoke tests, pre-release checks, CI). Same schema as the other +# *_workspace_test repos. +defaults: + PYAUTO_TEST_MODE: "2" + PYAUTO_SKIP_WORKSPACE_VERSION_CHECK: "1" + MPLBACKEND: "Agg" + NUMBA_CACHE_DIR: "/tmp/numba_cache" + MPLCONFIGDIR: "/tmp/matplotlib" +overrides: {} diff --git a/config/general.yaml b/config/general.yaml index 25b7e68..26d57b0 100644 --- a/config/general.yaml +++ b/config/general.yaml @@ -6,10 +6,14 @@ fits: hpc: hpc_mode: false # If True, use HPC mode, which disables GUI visualization, logging to screen and other settings which are not suited to running on a super computer. iterations_per_update: 5000 # The number of iterations between every update (visualization, results output, etc) in HPC mode. +inversion: + check_reconstruction: true # If True, the inversion's reconstruction is checked to ensure the solution of a meshs's mapper is not an invalid solution where the values are all the same. + reconstruction_vmax_factor: 0.5 # Plots of an Inversion's reconstruction use the reconstructed data's bright value multiplied by this factor. model: ignore_prior_limits: false # If ``True`` the limits applied to priors will be ignored, where limits set upper / lower limits. This stops PriorLimitException's from being raised. output: - force_pickle_overwrite: false # If True pickle files output by a search (e.g. samples.pickle) are recreated when a new model-fit is performed. + force_pickle_overwrite: false # force_pickle_overwrite: false # If True, pickle files output by a search (e.g. samples.pickle) are recreated when a new model-fit is performed. + force_visualize_overwrite: true # If True, visualization images output by a search (e.g. subplots of the fit) are recreated when a new model-fit is performed. info_whitespace_length: 80 # Length of whitespace between the parameter names and values in the model.info / result.info log_level: INFO # The level of information output by logging. log_to_file: false # If True, outputs the non-linear search log to a file (and not printed to screen). @@ -17,16 +21,20 @@ output: model_results_decimal_places: 3 # Number of decimal places estimated parameter values / errors are output in model.results. remove_files: false # If True, all output files of a non-linear search (e.g. samples, visualization, etc.) are deleted once the model-fit has completed, such that only the .zip file remains. samples_to_csv: true # If True, non-linear search samples are written to a .csv file. + unconverged_sample_size : 100 # If outputting results of an unconverged search, the number of samples used to estimate the median PDF values and errors. parallel: warn_environment_variables: true # If True, a warning is displayed when the search's number of CPU > 1 and enviromment variables related to threading are also > 1. + profiling: parallel_profile: false # If True, the parallelization of the fit is profiled outputting a cPython graph. should_profile: false # If True, the ``profile_log_likelihood_function()`` function of an analysis class is called throughout a model-fit, profiling run times. repeats: 1 # The number of repeat function calls used to measure run-times when profiling. structures: - use_dataset_grids: true # If True, dataset objects (e.g. Imaging) have a grid of (y,x) coordinates computed for them. + native_binned_only: false # If True, data structures are only stored in their native and binned format. This is used to reduce memory usage in autocti. test: - check_figure_of_merit_sanity: false - bypass_figure_of_merit_sanity: false + check_likelihood_function: true # if True, when a search is resumed the likelihood of a previous sample is recalculated to ensure it is consistent with the previous run. check_preloads: false + exception_override: false + lh_timeout_seconds: # If a float is input, the log_likelihood_function call is timed out after this many seconds, to diagnose infinite loops. Default is None, meaning no timeout. + preloads_check_threshold: 1.0 # If the figure of merit of a fit with and without preloads is greater than this threshold, the check preload test fails and an exception raised for a model-fit. disable_positions_lh_inversion_check: false diff --git a/config/logging.yaml b/config/logging.yaml index af59968..9897e4d 100644 --- a/config/logging.yaml +++ b/config/logging.yaml @@ -1,21 +1,17 @@ version: 1 disable_existing_loggers: false + handlers: console: class: logging.StreamHandler level: INFO stream: ext://sys.stdout formatter: formatter - file: - class: logging.FileHandler - level: INFO - filename: root.log - formatter: formatter + root: - level: DEBUG - handlers: - - console - - file + level: INFO + handlers: [ console ] + formatters: formatter: format: '%(asctime)s - %(name)s - %(levelname)s - %(message)s' diff --git a/config/non_linear/README.rst b/config/non_linear/README.rst index 11774c4..1fd2d7c 100644 --- a/config/non_linear/README.rst +++ b/config/non_linear/README.rst @@ -1,9 +1,9 @@ The ``non_linear`` folder contains configuration files which customize the default behaviour of non-linear searches in -**PyAutoLens**. +**PyAutoCTI**. Files ----- - ``mcmc.yaml``: Settings default behaviour of MCMC non-linear searches (e.g. Emcee). - ``nest.yaml``: Settings default behaviour of nested sampler non-linear searches (e.g. Dynesty). -- ``optimizer.yaml``: Settings default behaviour of optimizer non-linear searches (e.g. PySwarms). \ No newline at end of file +- ``mle.yaml``: Settings default behaviour of maximum likelihood estimator (mle) searches (e.g. PySwarms). \ No newline at end of file diff --git a/config/non_linear/mcmc.yaml b/config/non_linear/mcmc.yaml index 6da407e..0293100 100644 --- a/config/non_linear/mcmc.yaml +++ b/config/non_linear/mcmc.yaml @@ -26,10 +26,6 @@ Emcee: number_of_cores: 1 # The number of cores the search is parallelized over by default, using Python multiprocessing. printing: silence: false # If True, the default print output of the non-linear search is silcened and not printed by the Python interpreter. - prior_passer: - sigma: 3.0 # For non-linear search chaining and model prior passing, the sigma value of the inferred model parameter used as the sigma of the passed Gaussian prior. - use_errors: true # If True, the errors of the previous model's results are used when passing priors. - use_widths: true # If True the width of the model parameters defined in the priors config file are used. updates: iterations_per_update: 500 # The number of iterations of the non-linear search performed between every 'update', where an update performs tasks like outputting model.results. remove_state_files_at_end: true # Whether to remove the savestate of the seach (e.g. the Emcee hdf5 file) at the end to save hard-disk space (results are still stored as PyAutoFit pickles and loadable). @@ -61,10 +57,7 @@ Zeus: number_of_cores: 1 # The number of cores the search is parallelized over by default, using Python multiprocessing. printing: silence: false # If True, the default print output of the non-linear search is silenced and not printed by the Python interpreter. - prior_passer: - sigma: 3.0 # For non-linear search chaining and model prior passing, the sigma value of the inferred model parameter used as the sigma of the passed Gaussian prior. - use_errors: true # If True, the errors of the previous model's results are used when passing priors. - use_widths: true # If True the width of the model parameters defined in the priors config file are used. + updates: iterations_per_update: 500 # The number of iterations of the non-linear search performed between every 'update', where an update performs tasks like outputting model.results. remove_state_files_at_end: true # Whether to remove the savestate of the seach (e.g. the Emcee hdf5 file) at the end to save hard-disk space (results are still stored as PyAutoFit pickles and loadable). \ No newline at end of file diff --git a/config/non_linear/mle.yaml b/config/non_linear/mle.yaml new file mode 100644 index 0000000..d8423c9 --- /dev/null +++ b/config/non_linear/mle.yaml @@ -0,0 +1,89 @@ +# Configuration files that customize the default behaviour of non-linear searches. + +# **PyAutoFit** supports the following maximum likelihood estimator (MLE) algorithms: + +# - PySwarms: https://github.com/ljvmiranda921/pyswarms / https://pyswarms.readthedocs.io/en/latest/index.html + +# Settings in the [search], [run] and [options] entries are specific to each nested algorithm and should be +# determined by consulting that method's own readthedocs. + +PySwarmsGlobal: + run: + iters: 2000 + search: + cognitive: 0.5 + ftol: -.inf + inertia: 0.9 + n_particles: 50 + social: 0.3 + initialize: # The method used to generate where walkers are initialized in parameter space {prior | ball}. + method: ball # priors: samples are initialized by randomly drawing from each parameter's prior. ball: samples are initialized by randomly drawing unit values from a narrow uniform distribution. + ball_lower_limit: 0.49 # The lower limit of the uniform distribution unit values are drawn from when initializing walkers using the ball method. + ball_upper_limit: 0.51 # The upper limit of the uniform distribution unit values are drawn from when initializing walkers using the ball method. + parallel: + number_of_cores: 1 # The number of cores the search is parallelized over by default, using Python multiprocessing. + printing: + silence: false # If True, the default print output of the non-linear search is silcened and not printed by the Python interpreter. + updates: + iterations_per_update: 500 # The number of iterations of the non-linear search performed between every 'update', where an update performs tasks like outputting model.results. + remove_state_files_at_end: true # Whether to remove the savestate of the seach (e.g. the Emcee hdf5 file) at the end to save hard-disk space (results are still stored as PyAutoFit pickles and loadable). +PySwarmsLocal: + run: + iters: 2000 + search: + cognitive: 0.5 + ftol: -.inf + inertia: 0.9 + minkowski_p_norm: 2 + n_particles: 50 + number_of_k_neighbors: 3 + social: 0.3 + initialize: # The method used to generate where walkers are initialized in parameter space {prior | ball}. + method: ball # priors: samples are initialized by randomly drawing from each parameter's prior. ball: samples are initialized by randomly drawing unit values from a narrow uniform distribution. + ball_lower_limit: 0.49 # The lower limit of the uniform distribution unit values are drawn from when initializing walkers using the ball method. + ball_upper_limit: 0.51 # The upper limit of the uniform distribution unit values are drawn from when initializing walkers using the ball method. + parallel: + number_of_cores: 1 # The number of cores the search is parallelized over by default, using Python multiprocessing. + printing: + silence: false # If True, the default print output of the non-linear search is silcened and not printed by the Python interpreter. + updates: + iterations_per_update: 500 # The number of iterations of the non-linear search performed between every 'update', where an update performs tasks like outputting model.results. + remove_state_files_at_end: true # Whether to remove the savestate of the seach (e.g. the Emcee hdf5 file) at the end to save hard-disk space (results are still stored as PyAutoFit pickles and loadable). +LBFGS: + search: + tol: null + options: + disp: false + eps: 1.0e-08 + ftol: 2.220446049250313e-09 + gtol: 1.0e-05 + iprint: -1.0 + maxcor: 10 + maxfun: 15000 + maxiter: 15000 + maxls: 20 + initialize: # The method used to generate where walkers are initialized in parameter space {prior | ball}. + method: ball # priors: samples are initialized by randomly drawing from each parameter's prior. ball: samples are initialized by randomly drawing unit values from a narrow uniform distribution. + ball_lower_limit: 0.49 # The lower limit of the uniform distribution unit values are drawn from when initializing walkers using the ball method. + ball_upper_limit: 0.51 # The upper limit of the uniform distribution unit values are drawn from when initializing walkers using the ball method. + parallel: + number_of_cores: 1 # The number of cores the search is parallelized over by default, using Python multiprocessing. + printing: + silence: false # If True, the default print output of the non-linear search is silcened and not printed by the Python interpreter. + updates: + iterations_per_update: 500 # The number of iterations of the non-linear search performed between every 'update', where an update performs tasks like outputting model.results. + remove_state_files_at_end: true # Whether to remove the savestate of the seach (e.g. the Emcee hdf5 file) at the end to save hard-disk space (results are still stored as PyAutoFit pickles and loadable). +Drawer: + search: + total_draws: 50 + initialize: # The method used to generate where walkers are initialized in parameter space {prior | ball}. + method: ball # priors: samples are initialized by randomly drawing from each parameter's prior. ball: samples are initialized by randomly drawing unit values from a narrow uniform distribution. + ball_lower_limit: 0.49 # The lower limit of the uniform distribution unit values are drawn from when initializing walkers using the ball method. + ball_upper_limit: 0.51 # The upper limit of the uniform distribution unit values are drawn from when initializing walkers using the ball method. + parallel: + number_of_cores: 1 # The number of cores the search is parallelized over by default, using Python multiprocessing. + printing: + silence: false # If True, the default print output of the non-linear search is silcened and not printed by the Python interpreter. + updates: + iterations_per_update: 500 # The number of iterations of the non-linear search performed between every 'update', where an update performs tasks like outputting model.results. + remove_state_files_at_end: true # Whether to remove the savestate of the seach (e.g. the Emcee hdf5 file) at the end to save hard-disk space (results are still stored as PyAutoFit pickles and loadable). \ No newline at end of file diff --git a/config/non_linear/nest.yaml b/config/non_linear/nest.yaml index ac74ac7..32fac19 100644 --- a/config/non_linear/nest.yaml +++ b/config/non_linear/nest.yaml @@ -1,13 +1,40 @@ # Configuration files that customize the default behaviour of non-linear searches. -# **PyAutoFit** supports the following nested sampling algorithms: - # - Dynesty: https://github.com/joshspeagle/dynesty / https://dynesty.readthedocs.io/en/latest/index.html +# - Nautilus https://https://github.com/johannesulf/nautilus / https://nautilus-sampler.readthedocs.io/en/stable/index.html # - UltraNest: https://github.com/JohannesBuchner/UltraNest / https://johannesbuchner.github.io/UltraNest/readme.html # Settings in the [search] and [run] entries are specific to each nested algorithm and should be determined by # consulting that MCMC method's own readthedocs. - +Nautilus: + search: + n_live: 200 # Number of so-called live points. New bounds are constructed so that they encompass the live points. + n_update: # The maximum number of additions to the live set before a new bound is created + enlarge_per_dim: 1.1 # Along each dimension, outer ellipsoidal bounds are enlarged by this factor. + n_points_min: # The minimum number of points each ellipsoid should have. Effectively, ellipsoids with less than twice that number will not be split further. + split_threshold: 100 # Threshold used for splitting the multi-ellipsoidal bound used for sampling. + n_networks: 4 # Number of networks used in the estimator. + n_batch: 100 # Number of likelihood evaluations that are performed at each step. If likelihood evaluations are parallelized, should be multiple of the number of parallel processes. + n_like_new_bound: # The maximum number of likelihood calls before a new bounds is created. If None, use 10 times n_live. + vectorized: false # If True, the likelihood function can receive multiple input sets at once. + seed: # Seed for random number generation used for reproducible results accross different runs. + run: + f_live: 0.01 # Maximum fraction of the evidence contained in the live set before building the initial shells terminates. + n_shell: 1 # Minimum number of points in each shell. The algorithm will sample from the shells until this is reached. Default is 1. + n_eff: 500 # Minimum effective sample size. The algorithm will sample from the shells until this is reached. Default is 10000. + n_like_max: .inf # Maximum number of likelihood evaluations. Regardless of progress, the sampler will stop if this value is reached. Default is infinity. + discard_exploration: false # Whether to discard points drawn in the exploration phase. This is required for a fully unbiased posterior and evidence estimate. + verbose: true # Whether to print information about the run. + initialize: # The method used to generate where walkers are initialized in parameter space {prior}. + method: prior # priors: samples are initialized by randomly drawing from each parameter's prior. + parallel: + number_of_cores: 1 # The number of cores the search is parallelized over by default, using Python multiprocessing. + force_x1_cpu: false # Force Dynesty to not use Python multiprocessing Pool, which can fix issues on certain operating systems. + printing: + silence: false # If True, the default print output of the non-linear search is silenced and not printed by the Python interpreter. + updates: + iterations_per_update: 500 # The number of iterations of the non-linear search performed between every 'update', where an update performs tasks like outputting model.results. + remove_state_files_at_end: true # Whether to remove the savestate of the seach (e.g. the Emcee hdf5 file) at the end to save hard-disk space (results are still stored as PyAutoFit pickles and loadable). DynestyStatic: search: nlive: 50 @@ -35,12 +62,9 @@ DynestyStatic: force_x1_cpu: false # Force Dynesty to not use Python multiprocessing Pool, which can fix issues on certain operating systems. printing: silence: false # If True, the default print output of the non-linear search is silenced and not printed by the Python interpreter. - prior_passer: - sigma: 3.0 # For non-linear search chaining and model prior passing, the sigma value of the inferred model parameter used as the sigma of the passed Gaussian prior. - use_errors: true # If True, the errors of the previous model's results are used when passing priors. - use_widths: true # If True the width of the model parameters defined in the priors config file are used. + updates: - iterations_per_update: 2000 # The number of iterations of the non-linear search performed between every 'update', where an update performs tasks like outputting model.results. + iterations_per_update: 2500 # The number of iterations of the non-linear search performed between every 'update', where an update performs tasks like outputting model.results. remove_state_files_at_end: true # Whether to remove the savestate of the seach (e.g. the Emcee hdf5 file) at the end to save hard-disk space (results are still stored as PyAutoFit pickles and loadable). DynestyDynamic: search: @@ -72,12 +96,9 @@ DynestyDynamic: force_x1_cpu: false # Force Dynesty to not use Python multiprocessing Pool, which can fix issues on certain operating systems. printing: silence: false # If True, the default print output of the non-linear search is silenced and not printed by the Python interpreter. - prior_passer: - sigma: 3.0 # For non-linear search chaining and model prior passing, the sigma value of the inferred model parameter used as the sigma of the passed Gaussian prior. - use_errors: true # If True, the errors of the previous model's results are used when passing priors. - use_widths: true # If True the width of the model parameters defined in the priors config file are used. + updates: - iterations_per_update: 500 # The number of iterations of the non-linear search performed between every 'update', where an update performs tasks like outputting model.results. + iterations_per_update: 2500 # The number of iterations of the non-linear search performed between every 'update', where an update performs tasks like outputting model.results. remove_state_files_at_end: true # Whether to remove the savestate of the seach (e.g. the Emcee hdf5 file) at the end to save hard-disk space (results are still stored as PyAutoFit pickles and loadable). UltraNest: search: @@ -123,10 +144,7 @@ UltraNest: number_of_cores: 1 # The number of cores the search is parallelized over by default, using Python multiprocessing. printing: silence: false # If True, the default print output of the non-linear search is silenced and not printed by the Python interpreter. - prior_passer: - sigma: 3.0 # For non-linear search chaining and model prior passing, the sigma value of the inferred model parameter used as the sigma of the passed Gaussian prior. - use_errors: true # If True, the errors of the previous model's results are used when passing priors. - use_widths: true # If True the width of the model parameters defined in the priors config file are used. + updates: - iterations_per_update: 500 # The number of iterations of the non-linear search performed between every 'update', where an update performs tasks like outputting model.results. + iterations_per_update: 2500 # The number of iterations of the non-linear search performed between every 'update', where an update performs tasks like outputting model.results. remove_state_files_at_end: true # Whether to remove the savestate of the seach (e.g. the Emcee hdf5 file) at the end to save hard-disk space (results are still stored as PyAutoFit pickles and loadable). diff --git a/config/notation.yaml b/config/notation.yaml index c6bb31c..86543ba 100644 --- a/config/notation.yaml +++ b/config/notation.yaml @@ -15,6 +15,10 @@ label: label: density: \rho full_well_depth: h + gamma: \gamma + ka: ka + kv: kv + omega: \omega release_timescale: \tau release_timescale_sigma: \sigma scale_factor: \omega @@ -26,6 +30,7 @@ label: CCDComplex: ccd CCDPhase: ccd HyperCINoiseScalar: H + PixelBounce: pb TrapInstantCapture: s TrapInstantCaptureContinuum: sc @@ -37,6 +42,10 @@ label_format: format: density: '{:.2f}' full_well_depth: '{:.2f}' + gamma: '{:.2f}' + ka: '{:.2f}' + kv: '{:.2f}' + omega: '{:.2f}' release_timescale: '{:.2f}' release_timescale_sigma: '{:.2f}' scale_factor: '{:.2f}' diff --git a/config/priors/README.rst b/config/priors/README.rst index 1e924d8..c24379b 100644 --- a/config/priors/README.rst +++ b/config/priors/README.rst @@ -14,7 +14,7 @@ They appear as follows: "type": "Absolute", "value": 0.2 }, - "gaussian_limits": { + "limits": { "lower": 0.0, "upper": "inf" } @@ -30,8 +30,8 @@ The sections of this example config set the following: width_modifier When the results of a search are passed to a subsequent search to set up the priors of its non-linear search, this entry describes how the Prior is passed. For a full description of prior passing, checkout the examples - in 'autolens_workspace/examples/complex/linking'. - gaussian_limits + in 'autocti_workspace/examples/complex/linking'. + limits When the results of a search are passed to a subsequent search, they are passed using a GaussianPrior. The - gaussian_limits set the physical lower and upper limits of this GaussianPrior, such that parameter samples + limits set the physical lower and upper limits of this GaussianPrior, such that parameter samples can not go beyond these limits. \ No newline at end of file diff --git a/config/priors/ccd.yaml b/config/priors/ccd.yaml index 1aa3e08..7ce1d56 100644 --- a/config/priors/ccd.yaml +++ b/config/priors/ccd.yaml @@ -6,7 +6,7 @@ CCDPhase: width_modifier: type: Absolute value: 0.2 - gaussian_limits: + limits: lower: 0.0 upper: 1.0 well_notch_depth: @@ -16,7 +16,7 @@ CCDPhase: width_modifier: type: Absolute value: 0.2 - gaussian_limits: + limits: lower: 0.0 upper: 1.0 full_well_depth: @@ -26,6 +26,9 @@ CCDPhase: width_modifier: type: Absolute value: 0.2 - gaussian_limits: + limits: lower: 0.0 upper: 1.0 + first_electron_fill: + type: Constant + value: 0.0 \ No newline at end of file diff --git a/config/priors/hyper.yaml b/config/priors/hyper.yaml index 474c81d..11972c9 100644 --- a/config/priors/hyper.yaml +++ b/config/priors/hyper.yaml @@ -6,6 +6,6 @@ HyperCINoiseScalar: width_modifier: type: Relative value: 0.5 - gaussian_limits: + limits: lower: 0.0 upper: inf diff --git a/config/priors/pixel_bounce.yaml b/config/priors/pixel_bounce.yaml new file mode 100644 index 0000000..7ce1d56 --- /dev/null +++ b/config/priors/pixel_bounce.yaml @@ -0,0 +1,34 @@ +CCDPhase: + well_fill_power: + type: Uniform + lower_limit: 0.0 + upper_limit: 1.0 + width_modifier: + type: Absolute + value: 0.2 + limits: + lower: 0.0 + upper: 1.0 + well_notch_depth: + type: Uniform + lower_limit: 0.0 + upper_limit: 1.0 + width_modifier: + type: Absolute + value: 0.2 + limits: + lower: 0.0 + upper: 1.0 + full_well_depth: + type: Uniform + lower_limit: 0.0 + upper_limit: 200000.0 + width_modifier: + type: Absolute + value: 0.2 + limits: + lower: 0.0 + upper: 1.0 + first_electron_fill: + type: Constant + value: 0.0 \ No newline at end of file diff --git a/config/priors/traps.yaml b/config/priors/traps.yaml index cf88fe8..898ff30 100644 --- a/config/priors/traps.yaml +++ b/config/priors/traps.yaml @@ -6,7 +6,7 @@ TrapInstantCapture: width_modifier: type: Relative value: 0.5 - gaussian_limits: + limits: lower: 0.0 upper: inf release_timescale: @@ -16,7 +16,7 @@ TrapInstantCapture: width_modifier: type: Relative value: 0.5 - gaussian_limits: + limits: lower: 0.0 upper: inf fractional_volume_none_exposed: @@ -33,7 +33,7 @@ TrapInstantCaptureContinuum: width_modifier: type: Relative value: 0.5 - gaussian_limits: + limits: lower: 0.0 upper: inf release_timescale: @@ -43,7 +43,7 @@ TrapInstantCaptureContinuum: width_modifier: type: Relative value: 0.5 - gaussian_limits: + limits: lower: 0.0 upper: inf release_timescale_sigma: @@ -53,6 +53,6 @@ TrapInstantCaptureContinuum: width_modifier: type: Relative value: 0.5 - gaussian_limits: + limits: lower: 0.0 upper: inf diff --git a/config/visualize/README.rst b/config/visualize/README.rst index 2f80541..10548f4 100644 --- a/config/visualize/README.rst +++ b/config/visualize/README.rst @@ -1,4 +1,4 @@ -The ``config`` folder contains configuration files which customize default **PyAutoLens**. +The ``config`` folder contains configuration files which customize default **PyAutoCTI**. Files ----- diff --git a/config/visualize/general.yaml b/config/visualize/general.yaml index 9d67e10..3907a37 100644 --- a/config/visualize/general.yaml +++ b/config/visualize/general.yaml @@ -2,6 +2,5 @@ general: backend: default # The matploblib backend used for visualization. `default` uses the system default, can specifiy specific backend (e.g. TKAgg, Qt5Agg, WXAgg). imshow_origin: upper # The `origin` input of `imshow`, determining if pixel values are ascending or descending on the y-axis. zoom_around_mask: true # If True, plots of data structures with a mask automatically zoom in the masked region. -units: - in_kpc: false # If True, plots that are normally in arc-seconds are instead plotted using kiloparsecs. - use_scaled: false + symmetric_cmap_value: 100.0 # The vmin and vmax of all pre-cti data residual-maps. + subplot_ascending_fpr: true # If True, subplots showing FPR / EPER trails of many datasets are in ascending order of FPR value. diff --git a/config/visualize/plots.yaml b/config/visualize/plots.yaml index 64d6631..6a51fd9 100644 --- a/config/visualize/plots.yaml +++ b/config/visualize/plots.yaml @@ -1,38 +1,17 @@ -# The `plots` section customizes every image that is output to hard-disk during a model-fit. - -# For example, if `plots: fit: subplot_fit: true``, the ``fit_imaging.png`` subplot file will be plotted every time visualization is performed. - +subplot_format: [png] # Output format of all plots, can be png, pdf or both (e.g. [png, pdf]). +combined_only: false # If True, only the combined subplots of multi-dataset analyses are output (no per-dataset visualization). dataset: - subplot_dataset: true - data: false - noise_map: false - inverse_noise_map: false - signal_to_noise_map: false - pre_cti_data: false - cosmic_ray_map: false - noise_scaling_map_dict: false - absolute_signal_to_noise_map: false - potential_chi_squared_map: false - data_with_noise_map: true - data_with_noise_map_logy: true -imaging: - psf: false + subplot_dataset: true # Plot the subplot of all dataset quantities (2D for charge injection imaging, 1D for Dataset1D)? + subplot_dataset_regions: true # Plot per-region binned 1D subplots (e.g. the parallel/serial FPR and EPER)? + data: true # Plot single 1D figures of the data extracted and binned over each region? + data_logy: true # Plot single 1D figures of the data over each region with a log10 y-axis? + data_binned: true # Plot the data binned over rows / columns with and without the FPR (charge injection only)? + fpr_non_uniformity: false # Include the fpr_non_uniformity region in the per-region plots (charge injection only)? fit: - all_at_end_png: false - all_at_end_fits: false - subplot_fit: true - subplot_residual_maps: true - subplot_normalized_residual_maps: true - subplot_chi_squared_maps: true - data: true - noise_map: false - signal_to_noise_map: false - pre_cti_data: false - post_cti_data: true - residual_map: false - normalized_residual_map: true - chi_squared_map: false - data_with_noise_map: true - data_with_noise_map_logy: true - noise_scaling_map_dict: false -fit_imaging: {} + subplot_fit: true # Plot the subplot of all fit quantities (e.g. model data, residual-map, chi-squared map)? + subplot_fit_regions: true # Plot per-region binned 1D fit subplots (e.g. the parallel/serial FPR and EPER)? + data: true # Plot single 1D figures of the fit data (with model overlay) over each region? + data_logy: true # Plot single 1D figures of the fit data over each region with a log10 y-axis? + residual_map: true # Plot single 1D figures of the residual map over each region? + residual_map_logy: true # Plot single 1D figures of the residual map over each region with a log10 y-axis? + fits_fit: true # Output a fit.fits file containing the model data, residual map, normalized residual map and chi-squared map? diff --git a/config/visualize/plots_search.yaml b/config/visualize/plots_search.yaml index e2fd08c..228d4aa 100644 --- a/config/visualize/plots_search.yaml +++ b/config/visualize/plots_search.yaml @@ -1,24 +1,6 @@ -dynesty: - cornerplot: true # Output Dynesty cornerplot figure during a non-linear search fit? - cornerpoints: false # Output Dynesty cornerpoints figure during a non-linear search fit? - runplot: true # Output Dynesty runplot figure during a non-linear search fit? - traceplot: true # Output Dynesty traceplot figure during a non-linear search fit? -emcee: - corner: true # Output Emcee corner figure during a non-linear search fit? - likelihood_series: true # Output Emcee likelihood series figure during a non-linear search fit? - time_series: true # Output Emcee time series figure during a non-linear search fit? - trajectories: true # Output Emcee trajectories figure during a non-linear search fit? -pyswarms: - contour: true # Output PySwarms contour figure during a non-linear search fit? - cost_history: true # Output PySwarms cost_history figure during a non-linear search fit? - time_series: true # Output PySwarms time_series figure during a non-linear search fit? - trajectories: true # Output PySwarms trajectories figure during a non-linear search fit? -ultranest: - cornerplot: true # Output Ultranest cornerplot figure during a non-linear search fit? - runplot: true # Output Ultranest runplot figure during a non-linear search fit? - traceplot: true # Output Ultranest traceplot figure during a non-linear search fit? -zeus: - corner: true # Output Zeus corner figure during a non-linear search fit? - likelihood_series: true # Output Zeus likelihood series figure during a non-linear search fit? - time_series: true # Output Zeus time series figure during a non-linear search fit? - trajectories: true # Output Zeus trajectories figure during a non-linear search fit? \ No newline at end of file +nest: + corner_anesthetic: true # Output corner figure (using anestetic) during a non-linear search fit? +mcmc: + corner_cornerpy: true # Output corner figure (using corner.py) during a non-linear search fit? +mle: + corner_cornerpy: true # Output corner figure (using corner.py) during a non-linear search fit? \ No newline at end of file diff --git a/legacy/README.md b/legacy/README.md new file mode 100644 index 0000000..fa7f6b2 --- /dev/null +++ b/legacy/README.md @@ -0,0 +1,11 @@ +# Legacy — Euclid VIS heritage (2022-2023) + +This directory preserves, verbatim, the pre-resurrection contents of +`autocti_workspace_test`: Euclid VIS thermal-vacuum (`tvac/`), temporal trap-evolution +(`temporal/`), Euclid pipeline glue (`euclid/`), validation studies (`validation/`), +overview output checks (`overview_output/`) and the era's config (`config_2023/`). + +These scripts target the pre-2025 PyAutoCTI API (the removed Plotter object stack, +analysis summing, etc.) and are **not runnable against the current stack**. They are kept +as Euclid VIS calibration history — see the CTI resurrection epic (PyAutoCTI#82) for the +modernization; current integration tests live in `scripts/`. diff --git a/config/.gitignore b/legacy/config_2023/.gitignore similarity index 100% rename from config/.gitignore rename to legacy/config_2023/.gitignore diff --git a/legacy/config_2023/README.rst b/legacy/config_2023/README.rst new file mode 100644 index 0000000..be4a2cb --- /dev/null +++ b/legacy/config_2023/README.rst @@ -0,0 +1,15 @@ +The ``config`` folder contains configuration files which customize default **PyAutoGalaxy**. + +Folders +------- + +- ``non-linear``: Configs for default non-linear search (e.g. MCMC, nested sampling) settings. +- ``notation``: Configs defining labels and formatting of model parameters when used for visualization. +- ``priors``: Configs defining default priors assumed on every model component and set of parameters. +- ``visualize``: Configs defining what images are output by a model fit. + +Files +----- + +- ``general.yaml``: Customizes general **PyAutoGalaxy** settings. +- ``logging.yaml``: Customizes the logging behaviour of **PyAutoGalaxy**. \ No newline at end of file diff --git a/legacy/config_2023/general.yaml b/legacy/config_2023/general.yaml new file mode 100644 index 0000000..25b7e68 --- /dev/null +++ b/legacy/config_2023/general.yaml @@ -0,0 +1,32 @@ +analysis: + n_cores: 1 # The number of cores a parallelized sum of Analysis classes uses by default. + preload_attempts: 250 +fits: + flip_for_ds9: false +hpc: + hpc_mode: false # If True, use HPC mode, which disables GUI visualization, logging to screen and other settings which are not suited to running on a super computer. + iterations_per_update: 5000 # The number of iterations between every update (visualization, results output, etc) in HPC mode. +model: + ignore_prior_limits: false # If ``True`` the limits applied to priors will be ignored, where limits set upper / lower limits. This stops PriorLimitException's from being raised. +output: + force_pickle_overwrite: false # If True pickle files output by a search (e.g. samples.pickle) are recreated when a new model-fit is performed. + info_whitespace_length: 80 # Length of whitespace between the parameter names and values in the model.info / result.info + log_level: INFO # The level of information output by logging. + log_to_file: false # If True, outputs the non-linear search log to a file (and not printed to screen). + log_file: output.log # The name of the file the logged output is written to (in the non-linear search output folder) + model_results_decimal_places: 3 # Number of decimal places estimated parameter values / errors are output in model.results. + remove_files: false # If True, all output files of a non-linear search (e.g. samples, visualization, etc.) are deleted once the model-fit has completed, such that only the .zip file remains. + samples_to_csv: true # If True, non-linear search samples are written to a .csv file. +parallel: + warn_environment_variables: true # If True, a warning is displayed when the search's number of CPU > 1 and enviromment variables related to threading are also > 1. +profiling: + parallel_profile: false # If True, the parallelization of the fit is profiled outputting a cPython graph. + should_profile: false # If True, the ``profile_log_likelihood_function()`` function of an analysis class is called throughout a model-fit, profiling run times. + repeats: 1 # The number of repeat function calls used to measure run-times when profiling. +structures: + use_dataset_grids: true # If True, dataset objects (e.g. Imaging) have a grid of (y,x) coordinates computed for them. +test: + check_figure_of_merit_sanity: false + bypass_figure_of_merit_sanity: false + check_preloads: false + disable_positions_lh_inversion_check: false diff --git a/legacy/config_2023/logging.yaml b/legacy/config_2023/logging.yaml new file mode 100644 index 0000000..af59968 --- /dev/null +++ b/legacy/config_2023/logging.yaml @@ -0,0 +1,21 @@ +version: 1 +disable_existing_loggers: false +handlers: + console: + class: logging.StreamHandler + level: INFO + stream: ext://sys.stdout + formatter: formatter + file: + class: logging.FileHandler + level: INFO + filename: root.log + formatter: formatter +root: + level: DEBUG + handlers: + - console + - file +formatters: + formatter: + format: '%(asctime)s - %(name)s - %(levelname)s - %(message)s' diff --git a/legacy/config_2023/non_linear/GridSearch.yaml b/legacy/config_2023/non_linear/GridSearch.yaml new file mode 100644 index 0000000..af6daba --- /dev/null +++ b/legacy/config_2023/non_linear/GridSearch.yaml @@ -0,0 +1,5 @@ +# The settings of a parallelized grid search of non-linear searches. + +parallel: + number_of_cores: 3 # The number of cores the search is parallelized over by default, using Python multiprocessing. + step_size: 0.1 # The default step size of each grid search parameter, in terms of unit values of the priors. \ No newline at end of file diff --git a/legacy/config_2023/non_linear/README.rst b/legacy/config_2023/non_linear/README.rst new file mode 100644 index 0000000..11774c4 --- /dev/null +++ b/legacy/config_2023/non_linear/README.rst @@ -0,0 +1,9 @@ +The ``non_linear`` folder contains configuration files which customize the default behaviour of non-linear searches in +**PyAutoLens**. + +Files +----- + +- ``mcmc.yaml``: Settings default behaviour of MCMC non-linear searches (e.g. Emcee). +- ``nest.yaml``: Settings default behaviour of nested sampler non-linear searches (e.g. Dynesty). +- ``optimizer.yaml``: Settings default behaviour of optimizer non-linear searches (e.g. PySwarms). \ No newline at end of file diff --git a/legacy/config_2023/non_linear/mcmc.yaml b/legacy/config_2023/non_linear/mcmc.yaml new file mode 100644 index 0000000..6da407e --- /dev/null +++ b/legacy/config_2023/non_linear/mcmc.yaml @@ -0,0 +1,70 @@ +# Configuration files that customize the default behaviour of non-linear searches. + +# **PyAutoFit** supports the following MCMC algorithms: + +# - Emcee: https://github.com/dfm/emcee / https://emcee.readthedocs.io/en/stable/ +# - Zeus: https://github.com/minaskar/zeus / https://zeus-mcmc.readthedocs.io/en/latest/ + +# Settings in the [search] and [run] entries are specific to each nested algorithm and should be determined by +# consulting that MCMC method's own readthedocs. + +Emcee: + run: + nsteps: 2000 + search: + nwalkers: 50 + auto_correlations: + change_threshold: 0.01 # The threshold value by which if the change in auto_correlations is below sampling will be terminated early. + check_for_convergence: true # Whether the auto-correlation lengths of the Emcee samples are checked to determine the stopping criteria. If `True`, Emcee may stop before nsteps are performed. + check_size: 100 # The length of the samples used to check the auto-correlation lengths (from the latest sample backwards). + required_length: 50 # The length an auto_correlation chain must be for it to be used to evaluate whether its change threshold is sufficiently small to terminate sampling early. + initialize: # The method used to generate where walkers are initialized in parameter space {prior | ball}. + method: ball # priors: samples are initialized by randomly drawing from each parameter's prior. ball: samples are initialized by randomly drawing unit values from a narrow uniform distribution. + ball_lower_limit: 0.49 # The lower limit of the uniform distribution unit values are drawn from when initializing walkers using the ball method. + ball_upper_limit: 0.51 # The upper limit of the uniform distribution unit values are drawn from when initializing walkers using the ball method. + parallel: + number_of_cores: 1 # The number of cores the search is parallelized over by default, using Python multiprocessing. + printing: + silence: false # If True, the default print output of the non-linear search is silcened and not printed by the Python interpreter. + prior_passer: + sigma: 3.0 # For non-linear search chaining and model prior passing, the sigma value of the inferred model parameter used as the sigma of the passed Gaussian prior. + use_errors: true # If True, the errors of the previous model's results are used when passing priors. + use_widths: true # If True the width of the model parameters defined in the priors config file are used. + updates: + iterations_per_update: 500 # The number of iterations of the non-linear search performed between every 'update', where an update performs tasks like outputting model.results. + remove_state_files_at_end: true # Whether to remove the savestate of the seach (e.g. the Emcee hdf5 file) at the end to save hard-disk space (results are still stored as PyAutoFit pickles and loadable). +Zeus: + run: + check_walkers: true + light_mode: false + maxiter: 10000 + maxsteps: 10000 + mu: 1.0 + nsteps: 2000 + patience: 5 + shuffle_ensemble: true + tolerance: 0.05 + tune: true + vectorize: false + search: + nwalkers: 50 + auto_correlations: + change_threshold: 0.01 # The threshold value by which if the change in auto_correlations is below sampling will be terminated early. + check_for_convergence: true # Whether the auto-correlation lengths of the Emcee samples are checked to determine the stopping criteria. If `True`, Emcee may stop before nsteps are performed. + check_size: 100 # The length of the samples used to check the auto-correlation lengths (from the latest sample backwards). + required_length: 50 # The length an auto_correlation chain must be for it to be used to evaluate whether its change threshold is sufficiently small to terminate sampling early. + initialize: # The method used to generate where walkers are initialized in parameter space {prior | ball}. + method: ball # priors: samples are initialized by randomly drawing from each parameter's prior. ball: samples are initialized by randomly drawing unit values from a narrow uniform distribution. + ball_lower_limit: 0.49 # The lower limit of the uniform distribution unit values are drawn from when initializing walkers using the ball method. + ball_upper_limit: 0.51 # The upper limit of the uniform distribution unit values are drawn from when initializing walkers using the ball method. + parallel: + number_of_cores: 1 # The number of cores the search is parallelized over by default, using Python multiprocessing. + printing: + silence: false # If True, the default print output of the non-linear search is silenced and not printed by the Python interpreter. + prior_passer: + sigma: 3.0 # For non-linear search chaining and model prior passing, the sigma value of the inferred model parameter used as the sigma of the passed Gaussian prior. + use_errors: true # If True, the errors of the previous model's results are used when passing priors. + use_widths: true # If True the width of the model parameters defined in the priors config file are used. + updates: + iterations_per_update: 500 # The number of iterations of the non-linear search performed between every 'update', where an update performs tasks like outputting model.results. + remove_state_files_at_end: true # Whether to remove the savestate of the seach (e.g. the Emcee hdf5 file) at the end to save hard-disk space (results are still stored as PyAutoFit pickles and loadable). \ No newline at end of file diff --git a/legacy/config_2023/non_linear/nest.yaml b/legacy/config_2023/non_linear/nest.yaml new file mode 100644 index 0000000..ac74ac7 --- /dev/null +++ b/legacy/config_2023/non_linear/nest.yaml @@ -0,0 +1,132 @@ +# Configuration files that customize the default behaviour of non-linear searches. + +# **PyAutoFit** supports the following nested sampling algorithms: + +# - Dynesty: https://github.com/joshspeagle/dynesty / https://dynesty.readthedocs.io/en/latest/index.html +# - UltraNest: https://github.com/JohannesBuchner/UltraNest / https://johannesbuchner.github.io/UltraNest/readme.html + +# Settings in the [search] and [run] entries are specific to each nested algorithm and should be determined by +# consulting that MCMC method's own readthedocs. + +DynestyStatic: + search: + nlive: 50 + sample: rwalk + walks: 5 + bootstrap: null + bound: multi + enlarge: null + facc: 0.2 + first_update: null + fmove: 0.9 + max_move: 100 + slices: 5 + update_interval: null + run: + dlogz: null + logl_max: .inf + maxcall: null + maxiter: null + n_effective: null + initialize: # The method used to generate where walkers are initialized in parameter space {prior}. + method: prior # priors: samples are initialized by randomly drawing from each parameter's prior. + parallel: + number_of_cores: 1 # The number of cores the search is parallelized over by default, using Python multiprocessing. + force_x1_cpu: false # Force Dynesty to not use Python multiprocessing Pool, which can fix issues on certain operating systems. + printing: + silence: false # If True, the default print output of the non-linear search is silenced and not printed by the Python interpreter. + prior_passer: + sigma: 3.0 # For non-linear search chaining and model prior passing, the sigma value of the inferred model parameter used as the sigma of the passed Gaussian prior. + use_errors: true # If True, the errors of the previous model's results are used when passing priors. + use_widths: true # If True the width of the model parameters defined in the priors config file are used. + updates: + iterations_per_update: 2000 # The number of iterations of the non-linear search performed between every 'update', where an update performs tasks like outputting model.results. + remove_state_files_at_end: true # Whether to remove the savestate of the seach (e.g. the Emcee hdf5 file) at the end to save hard-disk space (results are still stored as PyAutoFit pickles and loadable). +DynestyDynamic: + search: + sample: rwalk + walks: 5 + bootstrap: null + bound: multi + enlarge: null + facc: 0.2 + first_update: null + fmove: 0.9 + max_move: 100 + slices: 5 + update_interval: null + run: + dlogz_init: 0.01 + logl_max_init: .inf + maxcall: null + maxcall_init: null + maxiter: null + maxiter_init: null + n_effective: .inf + n_effective_init: .inf + nlive_init: 500 + initialize: # The method used to generate where walkers are initialized in parameter space {prior}. + method: prior # priors: samples are initialized by randomly drawing from each parameter's prior. + parallel: + number_of_cores: 1 # The number of cores the search is parallelized over by default, using Python multiprocessing. + force_x1_cpu: false # Force Dynesty to not use Python multiprocessing Pool, which can fix issues on certain operating systems. + printing: + silence: false # If True, the default print output of the non-linear search is silenced and not printed by the Python interpreter. + prior_passer: + sigma: 3.0 # For non-linear search chaining and model prior passing, the sigma value of the inferred model parameter used as the sigma of the passed Gaussian prior. + use_errors: true # If True, the errors of the previous model's results are used when passing priors. + use_widths: true # If True the width of the model parameters defined in the priors config file are used. + updates: + iterations_per_update: 500 # The number of iterations of the non-linear search performed between every 'update', where an update performs tasks like outputting model.results. + remove_state_files_at_end: true # Whether to remove the savestate of the seach (e.g. the Emcee hdf5 file) at the end to save hard-disk space (results are still stored as PyAutoFit pickles and loadable). +UltraNest: + search: + draw_multiple: true + ndraw_max: 65536 + ndraw_min: 128 + num_bootstraps: 30 + num_test_samples: 2 + resume: true + run_num: null + storage_backend: hdf5 + vectorized: false + warmstart_max_tau: -1.0 + run: + cluster_num_live_points: 40 + dkl: 0.5 + dlogz: 0.5 + frac_remain: 0.01 + insertion_test_window: 10 + insertion_test_zscore_threshold: 2 + lepsilon: 0.001 + log_interval: null + max_iters: null + max_ncalls: null + max_num_improvement_loops: -1.0 + min_ess: 400 + min_num_live_points: 400 + show_status: true + update_interval_ncall: null + update_interval_volume_fraction: 0.8 + viz_callback: auto + stepsampler: + adaptive_nsteps: false + log: false + max_nsteps: 1000 + nsteps: 25 + region_filter: false + scale: 1.0 + stepsampler_cls: null + initialize: # The method used to generate where walkers are initialized in parameter space {prior}. + method: prior # priors: samples are initialized by randomly drawing from each parameter's prior. + parallel: + number_of_cores: 1 # The number of cores the search is parallelized over by default, using Python multiprocessing. + printing: + silence: false # If True, the default print output of the non-linear search is silenced and not printed by the Python interpreter. + prior_passer: + sigma: 3.0 # For non-linear search chaining and model prior passing, the sigma value of the inferred model parameter used as the sigma of the passed Gaussian prior. + use_errors: true # If True, the errors of the previous model's results are used when passing priors. + use_widths: true # If True the width of the model parameters defined in the priors config file are used. + updates: + iterations_per_update: 500 # The number of iterations of the non-linear search performed between every 'update', where an update performs tasks like outputting model.results. + remove_state_files_at_end: true # Whether to remove the savestate of the seach (e.g. the Emcee hdf5 file) at the end to save hard-disk space (results are still stored as PyAutoFit pickles and loadable). diff --git a/config/non_linear/optimize.yaml b/legacy/config_2023/non_linear/optimize.yaml similarity index 100% rename from config/non_linear/optimize.yaml rename to legacy/config_2023/non_linear/optimize.yaml diff --git a/legacy/config_2023/notation.yaml b/legacy/config_2023/notation.yaml new file mode 100644 index 0000000..c6bb31c --- /dev/null +++ b/legacy/config_2023/notation.yaml @@ -0,0 +1,46 @@ +# The notation configs define the labels of every model parameter and its derived quantities, which are used when +# visualizing results (for example labeling the axis of the PDF triangle plots output by a non-linear search). + + +# label: The label given to the each parameter, for plots like PDF corner plots. + +# For example, if `centre=x`, the plot axis will be labeled 'x'. + + +# superscript: the superscript used on certain plots that show the results of different model-components. + +# For example, if `TrapInstantCapture=s`, plots where the parameters of the TrapInstantCapture model-component have superscript `s`. + +label: + label: + density: \rho + full_well_depth: h + release_timescale: \tau + release_timescale_sigma: \sigma + scale_factor: \omega + well_fill_alpha: \alpha + well_fill_gamma: \gamma + well_fill_power: \beta + well_notch_depth: d + superscript: + CCDComplex: ccd + CCDPhase: ccd + HyperCINoiseScalar: H + TrapInstantCapture: s + TrapInstantCaptureContinuum: sc + +# label_format: The format certain parameters are output as in output files like the `model.results` file. + +# For example, if `density={:.2f}`, the format of the centre parameter in results files will use this Python format. + +label_format: + format: + density: '{:.2f}' + full_well_depth: '{:.2f}' + release_timescale: '{:.2f}' + release_timescale_sigma: '{:.2f}' + scale_factor: '{:.2f}' + well_fill_alpha: '{:.2f}' + well_fill_gamma: '{:.2f}' + well_fill_power: '{:.2f}' + well_notch_depth: '{:.2f}' diff --git a/legacy/config_2023/priors/README.rst b/legacy/config_2023/priors/README.rst new file mode 100644 index 0000000..1e924d8 --- /dev/null +++ b/legacy/config_2023/priors/README.rst @@ -0,0 +1,37 @@ +The prior config files contain the default priors and related variables for every light profile and mass profile +when it is used as a model. + +They appear as follows: + +.. code-block:: bash + + "IsothermalSph": { + "einstein_radius": { + "type": "Uniform", + "lower_limit": 0.0, + "upper_limit": 4.0, + "width_modifier": { + "type": "Absolute", + "value": 0.2 + }, + "gaussian_limits": { + "lower": 0.0, + "upper": "inf" + } + } + +The sections of this example config set the following: + + type {Uniform, Gaussian, LogUniform} + The default prior given to this parameter when used by the non-linear search. In the example above, a + UniformPrior is used with lower_limit of 0.0 and upper_limit of 4.0. A GaussianPrior could be used by + putting "Gaussian" in the "type" box, with "mean" and "sigma" used to set the default values. Any prior can be + set in an analogous fashion (see the example configs). + width_modifier + When the results of a search are passed to a subsequent search to set up the priors of its non-linear search, + this entry describes how the Prior is passed. For a full description of prior passing, checkout the examples + in 'autolens_workspace/examples/complex/linking'. + gaussian_limits + When the results of a search are passed to a subsequent search, they are passed using a GaussianPrior. The + gaussian_limits set the physical lower and upper limits of this GaussianPrior, such that parameter samples + can not go beyond these limits. \ No newline at end of file diff --git a/legacy/config_2023/priors/ccd.yaml b/legacy/config_2023/priors/ccd.yaml new file mode 100644 index 0000000..1aa3e08 --- /dev/null +++ b/legacy/config_2023/priors/ccd.yaml @@ -0,0 +1,31 @@ +CCDPhase: + well_fill_power: + type: Uniform + lower_limit: 0.0 + upper_limit: 1.0 + width_modifier: + type: Absolute + value: 0.2 + gaussian_limits: + lower: 0.0 + upper: 1.0 + well_notch_depth: + type: Uniform + lower_limit: 0.0 + upper_limit: 1.0 + width_modifier: + type: Absolute + value: 0.2 + gaussian_limits: + lower: 0.0 + upper: 1.0 + full_well_depth: + type: Uniform + lower_limit: 0.0 + upper_limit: 200000.0 + width_modifier: + type: Absolute + value: 0.2 + gaussian_limits: + lower: 0.0 + upper: 1.0 diff --git a/legacy/config_2023/priors/hyper.yaml b/legacy/config_2023/priors/hyper.yaml new file mode 100644 index 0000000..474c81d --- /dev/null +++ b/legacy/config_2023/priors/hyper.yaml @@ -0,0 +1,11 @@ +HyperCINoiseScalar: + scale_factor: + type: Uniform + lower_limit: 0.0 + upper_limit: 10.0 + width_modifier: + type: Relative + value: 0.5 + gaussian_limits: + lower: 0.0 + upper: inf diff --git a/legacy/config_2023/priors/traps.yaml b/legacy/config_2023/priors/traps.yaml new file mode 100644 index 0000000..cf88fe8 --- /dev/null +++ b/legacy/config_2023/priors/traps.yaml @@ -0,0 +1,58 @@ +TrapInstantCapture: + density: + type: Uniform + lower_limit: 0.0 + upper_limit: 10.0 + width_modifier: + type: Relative + value: 0.5 + gaussian_limits: + lower: 0.0 + upper: inf + release_timescale: + type: Uniform + lower_limit: 0.0 + upper_limit: 50.0 + width_modifier: + type: Relative + value: 0.5 + gaussian_limits: + lower: 0.0 + upper: inf + fractional_volume_none_exposed: + type: Constant + value: 0.0 + fractional_volume_full_exposed: + type: Constant + value: 0.0 +TrapInstantCaptureContinuum: + density: + type: Uniform + lower_limit: 0.0 + upper_limit: 10.0 + width_modifier: + type: Relative + value: 0.5 + gaussian_limits: + lower: 0.0 + upper: inf + release_timescale: + type: Uniform + lower_limit: 0.0 + upper_limit: 50.0 + width_modifier: + type: Relative + value: 0.5 + gaussian_limits: + lower: 0.0 + upper: inf + release_timescale_sigma: + type: Uniform + lower_limit: 0.0 + upper_limit: 1.0 + width_modifier: + type: Relative + value: 0.5 + gaussian_limits: + lower: 0.0 + upper: inf diff --git a/legacy/config_2023/visualize/README.rst b/legacy/config_2023/visualize/README.rst new file mode 100644 index 0000000..2f80541 --- /dev/null +++ b/legacy/config_2023/visualize/README.rst @@ -0,0 +1,12 @@ +The ``config`` folder contains configuration files which customize default **PyAutoLens**. + +Files +----- + +- ``general.yaml``: Customizes general visualization settings (e.g. the matplotlib backend). +- ``include.yaml``: Customize features that appears on plotted images by default (e.g. a mask, a grid). +- ``plots.yaml``: Customize which figures are output during a model-fit. +- ``plots_search.yaml``: Customize which non-linear search figures are output during a model-fit. +- ``mat_wrap.yaml``: Specify the default matplotlib settings when figures and subplots are plotted. +- ``mat_wrap_1d.yaml``: Specify the default matplotlib settings when 1D figures and subplots are plotted. +- ``mat_wrap_2d.yaml``: Specify the default matplotlib settings when 2D figures and subplots are plotted. \ No newline at end of file diff --git a/legacy/config_2023/visualize/general.yaml b/legacy/config_2023/visualize/general.yaml new file mode 100644 index 0000000..9d67e10 --- /dev/null +++ b/legacy/config_2023/visualize/general.yaml @@ -0,0 +1,7 @@ +general: + backend: default # The matploblib backend used for visualization. `default` uses the system default, can specifiy specific backend (e.g. TKAgg, Qt5Agg, WXAgg). + imshow_origin: upper # The `origin` input of `imshow`, determining if pixel values are ascending or descending on the y-axis. + zoom_around_mask: true # If True, plots of data structures with a mask automatically zoom in the masked region. +units: + in_kpc: false # If True, plots that are normally in arc-seconds are instead plotted using kiloparsecs. + use_scaled: false diff --git a/config/visualize/include.yaml b/legacy/config_2023/visualize/include.yaml similarity index 100% rename from config/visualize/include.yaml rename to legacy/config_2023/visualize/include.yaml diff --git a/config/visualize/mat_wrap.yaml b/legacy/config_2023/visualize/mat_wrap.yaml similarity index 100% rename from config/visualize/mat_wrap.yaml rename to legacy/config_2023/visualize/mat_wrap.yaml diff --git a/config/visualize/mat_wrap_1d.yaml b/legacy/config_2023/visualize/mat_wrap_1d.yaml similarity index 100% rename from config/visualize/mat_wrap_1d.yaml rename to legacy/config_2023/visualize/mat_wrap_1d.yaml diff --git a/config/visualize/mat_wrap_2d.yaml b/legacy/config_2023/visualize/mat_wrap_2d.yaml similarity index 100% rename from config/visualize/mat_wrap_2d.yaml rename to legacy/config_2023/visualize/mat_wrap_2d.yaml diff --git a/legacy/config_2023/visualize/plots.yaml b/legacy/config_2023/visualize/plots.yaml new file mode 100644 index 0000000..64d6631 --- /dev/null +++ b/legacy/config_2023/visualize/plots.yaml @@ -0,0 +1,38 @@ +# The `plots` section customizes every image that is output to hard-disk during a model-fit. + +# For example, if `plots: fit: subplot_fit: true``, the ``fit_imaging.png`` subplot file will be plotted every time visualization is performed. + +dataset: + subplot_dataset: true + data: false + noise_map: false + inverse_noise_map: false + signal_to_noise_map: false + pre_cti_data: false + cosmic_ray_map: false + noise_scaling_map_dict: false + absolute_signal_to_noise_map: false + potential_chi_squared_map: false + data_with_noise_map: true + data_with_noise_map_logy: true +imaging: + psf: false +fit: + all_at_end_png: false + all_at_end_fits: false + subplot_fit: true + subplot_residual_maps: true + subplot_normalized_residual_maps: true + subplot_chi_squared_maps: true + data: true + noise_map: false + signal_to_noise_map: false + pre_cti_data: false + post_cti_data: true + residual_map: false + normalized_residual_map: true + chi_squared_map: false + data_with_noise_map: true + data_with_noise_map_logy: true + noise_scaling_map_dict: false +fit_imaging: {} diff --git a/legacy/config_2023/visualize/plots_search.yaml b/legacy/config_2023/visualize/plots_search.yaml new file mode 100644 index 0000000..e2fd08c --- /dev/null +++ b/legacy/config_2023/visualize/plots_search.yaml @@ -0,0 +1,24 @@ +dynesty: + cornerplot: true # Output Dynesty cornerplot figure during a non-linear search fit? + cornerpoints: false # Output Dynesty cornerpoints figure during a non-linear search fit? + runplot: true # Output Dynesty runplot figure during a non-linear search fit? + traceplot: true # Output Dynesty traceplot figure during a non-linear search fit? +emcee: + corner: true # Output Emcee corner figure during a non-linear search fit? + likelihood_series: true # Output Emcee likelihood series figure during a non-linear search fit? + time_series: true # Output Emcee time series figure during a non-linear search fit? + trajectories: true # Output Emcee trajectories figure during a non-linear search fit? +pyswarms: + contour: true # Output PySwarms contour figure during a non-linear search fit? + cost_history: true # Output PySwarms cost_history figure during a non-linear search fit? + time_series: true # Output PySwarms time_series figure during a non-linear search fit? + trajectories: true # Output PySwarms trajectories figure during a non-linear search fit? +ultranest: + cornerplot: true # Output Ultranest cornerplot figure during a non-linear search fit? + runplot: true # Output Ultranest runplot figure during a non-linear search fit? + traceplot: true # Output Ultranest traceplot figure during a non-linear search fit? +zeus: + corner: true # Output Zeus corner figure during a non-linear search fit? + likelihood_series: true # Output Zeus likelihood series figure during a non-linear search fit? + time_series: true # Output Zeus time series figure during a non-linear search fit? + trajectories: true # Output Zeus trajectories figure during a non-linear search fit? \ No newline at end of file diff --git a/euclid/__init__.py b/legacy/euclid/__init__.py similarity index 100% rename from euclid/__init__.py rename to legacy/euclid/__init__.py diff --git a/euclid/clocker_json.py b/legacy/euclid/clocker_json.py similarity index 100% rename from euclid/clocker_json.py rename to legacy/euclid/clocker_json.py diff --git a/euclid/model_json.py b/legacy/euclid/model_json.py similarity index 100% rename from euclid/model_json.py rename to legacy/euclid/model_json.py diff --git a/overview_output/__init__.py b/legacy/overview_output/__init__.py similarity index 100% rename from overview_output/__init__.py rename to legacy/overview_output/__init__.py diff --git a/overview_output/overview_1_what_is_cti.py b/legacy/overview_output/overview_1_what_is_cti.py similarity index 100% rename from overview_output/overview_1_what_is_cti.py rename to legacy/overview_output/overview_1_what_is_cti.py diff --git a/overview_output/overview_2_parallel_and_serial.py b/legacy/overview_output/overview_2_parallel_and_serial.py similarity index 100% rename from overview_output/overview_2_parallel_and_serial.py rename to legacy/overview_output/overview_2_parallel_and_serial.py diff --git a/overview_output/overview_3_cti_features.py b/legacy/overview_output/overview_3_cti_features.py similarity index 100% rename from overview_output/overview_3_cti_features.py rename to legacy/overview_output/overview_3_cti_features.py diff --git a/overview_output/overview_4_charge_injection_data.py b/legacy/overview_output/overview_4_charge_injection_data.py similarity index 100% rename from overview_output/overview_4_charge_injection_data.py rename to legacy/overview_output/overview_4_charge_injection_data.py diff --git a/overview_output/overview_5_fitting.py b/legacy/overview_output/overview_5_fitting.py similarity index 100% rename from overview_output/overview_5_fitting.py rename to legacy/overview_output/overview_5_fitting.py diff --git a/overview_output/overview_5_fitting_out.py b/legacy/overview_output/overview_5_fitting_out.py similarity index 100% rename from overview_output/overview_5_fitting_out.py rename to legacy/overview_output/overview_5_fitting_out.py diff --git a/overview_output/overview_6_cti_calibration.py b/legacy/overview_output/overview_6_cti_calibration.py similarity index 100% rename from overview_output/overview_6_cti_calibration.py rename to legacy/overview_output/overview_6_cti_calibration.py diff --git a/temporal/__init__.py b/legacy/temporal/__init__.py similarity index 100% rename from temporal/__init__.py rename to legacy/temporal/__init__.py diff --git a/temporal/database.py b/legacy/temporal/database.py similarity index 100% rename from temporal/database.py rename to legacy/temporal/database.py diff --git a/temporal/dataset/parallel_x1/time_0/clocker.json b/legacy/temporal/dataset/parallel_x1/time_0/clocker.json similarity index 100% rename from temporal/dataset/parallel_x1/time_0/clocker.json rename to legacy/temporal/dataset/parallel_x1/time_0/clocker.json diff --git a/temporal/dataset/parallel_x1/time_0/cti.json b/legacy/temporal/dataset/parallel_x1/time_0/cti.json similarity index 100% rename from temporal/dataset/parallel_x1/time_0/cti.json rename to legacy/temporal/dataset/parallel_x1/time_0/cti.json diff --git a/temporal/dataset/parallel_x1/time_0/data_100.fits b/legacy/temporal/dataset/parallel_x1/time_0/data_100.fits similarity index 100% rename from temporal/dataset/parallel_x1/time_0/data_100.fits rename to legacy/temporal/dataset/parallel_x1/time_0/data_100.fits diff --git a/temporal/dataset/parallel_x1/time_0/data_200000.fits b/legacy/temporal/dataset/parallel_x1/time_0/data_200000.fits similarity index 100% rename from temporal/dataset/parallel_x1/time_0/data_200000.fits rename to legacy/temporal/dataset/parallel_x1/time_0/data_200000.fits diff --git a/temporal/dataset/parallel_x1/time_0/data_25000.fits b/legacy/temporal/dataset/parallel_x1/time_0/data_25000.fits similarity index 100% rename from temporal/dataset/parallel_x1/time_0/data_25000.fits rename to legacy/temporal/dataset/parallel_x1/time_0/data_25000.fits diff --git a/temporal/dataset/parallel_x1/time_0/data_5000.fits b/legacy/temporal/dataset/parallel_x1/time_0/data_5000.fits similarity index 100% rename from temporal/dataset/parallel_x1/time_0/data_5000.fits rename to legacy/temporal/dataset/parallel_x1/time_0/data_5000.fits diff --git a/temporal/dataset/parallel_x1/time_0/norm_100/binned_1d/image_parallel_eper.png b/legacy/temporal/dataset/parallel_x1/time_0/norm_100/binned_1d/image_parallel_eper.png similarity index 100% rename from temporal/dataset/parallel_x1/time_0/norm_100/binned_1d/image_parallel_eper.png rename to legacy/temporal/dataset/parallel_x1/time_0/norm_100/binned_1d/image_parallel_eper.png diff --git a/temporal/dataset/parallel_x1/time_0/norm_100/binned_1d/image_parallel_fpr.png b/legacy/temporal/dataset/parallel_x1/time_0/norm_100/binned_1d/image_parallel_fpr.png similarity index 100% rename from temporal/dataset/parallel_x1/time_0/norm_100/binned_1d/image_parallel_fpr.png rename to legacy/temporal/dataset/parallel_x1/time_0/norm_100/binned_1d/image_parallel_fpr.png diff --git a/temporal/dataset/parallel_x1/time_0/norm_100/imaging_ci.png b/legacy/temporal/dataset/parallel_x1/time_0/norm_100/imaging_ci.png similarity index 100% rename from temporal/dataset/parallel_x1/time_0/norm_100/imaging_ci.png rename to legacy/temporal/dataset/parallel_x1/time_0/norm_100/imaging_ci.png diff --git a/temporal/dataset/parallel_x1/time_0/norm_100/noise_map.fits b/legacy/temporal/dataset/parallel_x1/time_0/norm_100/noise_map.fits similarity index 100% rename from temporal/dataset/parallel_x1/time_0/norm_100/noise_map.fits rename to legacy/temporal/dataset/parallel_x1/time_0/norm_100/noise_map.fits diff --git a/temporal/dataset/parallel_x1/time_0/norm_100/pre_cti_data.fits b/legacy/temporal/dataset/parallel_x1/time_0/norm_100/pre_cti_data.fits similarity index 100% rename from temporal/dataset/parallel_x1/time_0/norm_100/pre_cti_data.fits rename to legacy/temporal/dataset/parallel_x1/time_0/norm_100/pre_cti_data.fits diff --git a/temporal/dataset/parallel_x1/time_0/norm_200000/binned_1d/image_parallel_eper.png b/legacy/temporal/dataset/parallel_x1/time_0/norm_200000/binned_1d/image_parallel_eper.png similarity index 100% rename from temporal/dataset/parallel_x1/time_0/norm_200000/binned_1d/image_parallel_eper.png rename to legacy/temporal/dataset/parallel_x1/time_0/norm_200000/binned_1d/image_parallel_eper.png diff --git a/temporal/dataset/parallel_x1/time_0/norm_200000/binned_1d/image_parallel_fpr.png b/legacy/temporal/dataset/parallel_x1/time_0/norm_200000/binned_1d/image_parallel_fpr.png similarity index 100% rename from temporal/dataset/parallel_x1/time_0/norm_200000/binned_1d/image_parallel_fpr.png rename to legacy/temporal/dataset/parallel_x1/time_0/norm_200000/binned_1d/image_parallel_fpr.png diff --git a/temporal/dataset/parallel_x1/time_0/norm_200000/imaging_ci.png b/legacy/temporal/dataset/parallel_x1/time_0/norm_200000/imaging_ci.png similarity index 100% rename from temporal/dataset/parallel_x1/time_0/norm_200000/imaging_ci.png rename to legacy/temporal/dataset/parallel_x1/time_0/norm_200000/imaging_ci.png diff --git a/temporal/dataset/parallel_x1/time_0/norm_200000/noise_map.fits b/legacy/temporal/dataset/parallel_x1/time_0/norm_200000/noise_map.fits similarity index 100% rename from temporal/dataset/parallel_x1/time_0/norm_200000/noise_map.fits rename to legacy/temporal/dataset/parallel_x1/time_0/norm_200000/noise_map.fits diff --git a/temporal/dataset/parallel_x1/time_0/norm_200000/pre_cti_data.fits b/legacy/temporal/dataset/parallel_x1/time_0/norm_200000/pre_cti_data.fits similarity index 100% rename from temporal/dataset/parallel_x1/time_0/norm_200000/pre_cti_data.fits rename to legacy/temporal/dataset/parallel_x1/time_0/norm_200000/pre_cti_data.fits diff --git a/temporal/dataset/parallel_x1/time_0/norm_25000/binned_1d/image_parallel_eper.png b/legacy/temporal/dataset/parallel_x1/time_0/norm_25000/binned_1d/image_parallel_eper.png similarity index 100% rename from temporal/dataset/parallel_x1/time_0/norm_25000/binned_1d/image_parallel_eper.png rename to legacy/temporal/dataset/parallel_x1/time_0/norm_25000/binned_1d/image_parallel_eper.png diff --git a/temporal/dataset/parallel_x1/time_0/norm_25000/binned_1d/image_parallel_fpr.png b/legacy/temporal/dataset/parallel_x1/time_0/norm_25000/binned_1d/image_parallel_fpr.png similarity index 100% rename from temporal/dataset/parallel_x1/time_0/norm_25000/binned_1d/image_parallel_fpr.png rename to legacy/temporal/dataset/parallel_x1/time_0/norm_25000/binned_1d/image_parallel_fpr.png diff --git a/temporal/dataset/parallel_x1/time_0/norm_25000/imaging_ci.png b/legacy/temporal/dataset/parallel_x1/time_0/norm_25000/imaging_ci.png similarity index 100% rename from temporal/dataset/parallel_x1/time_0/norm_25000/imaging_ci.png rename to legacy/temporal/dataset/parallel_x1/time_0/norm_25000/imaging_ci.png diff --git a/temporal/dataset/parallel_x1/time_0/norm_25000/noise_map.fits b/legacy/temporal/dataset/parallel_x1/time_0/norm_25000/noise_map.fits similarity index 100% rename from temporal/dataset/parallel_x1/time_0/norm_25000/noise_map.fits rename to legacy/temporal/dataset/parallel_x1/time_0/norm_25000/noise_map.fits diff --git a/temporal/dataset/parallel_x1/time_0/norm_25000/pre_cti_data.fits b/legacy/temporal/dataset/parallel_x1/time_0/norm_25000/pre_cti_data.fits similarity index 100% rename from temporal/dataset/parallel_x1/time_0/norm_25000/pre_cti_data.fits rename to legacy/temporal/dataset/parallel_x1/time_0/norm_25000/pre_cti_data.fits diff --git a/temporal/dataset/parallel_x1/time_0/norm_5000/binned_1d/image_parallel_eper.png b/legacy/temporal/dataset/parallel_x1/time_0/norm_5000/binned_1d/image_parallel_eper.png similarity index 100% rename from temporal/dataset/parallel_x1/time_0/norm_5000/binned_1d/image_parallel_eper.png rename to legacy/temporal/dataset/parallel_x1/time_0/norm_5000/binned_1d/image_parallel_eper.png diff --git a/temporal/dataset/parallel_x1/time_0/norm_5000/binned_1d/image_parallel_fpr.png b/legacy/temporal/dataset/parallel_x1/time_0/norm_5000/binned_1d/image_parallel_fpr.png similarity index 100% rename from temporal/dataset/parallel_x1/time_0/norm_5000/binned_1d/image_parallel_fpr.png rename to legacy/temporal/dataset/parallel_x1/time_0/norm_5000/binned_1d/image_parallel_fpr.png diff --git a/temporal/dataset/parallel_x1/time_0/norm_5000/imaging_ci.png b/legacy/temporal/dataset/parallel_x1/time_0/norm_5000/imaging_ci.png similarity index 100% rename from temporal/dataset/parallel_x1/time_0/norm_5000/imaging_ci.png rename to legacy/temporal/dataset/parallel_x1/time_0/norm_5000/imaging_ci.png diff --git a/temporal/dataset/parallel_x1/time_0/norm_5000/noise_map.fits b/legacy/temporal/dataset/parallel_x1/time_0/norm_5000/noise_map.fits similarity index 100% rename from temporal/dataset/parallel_x1/time_0/norm_5000/noise_map.fits rename to legacy/temporal/dataset/parallel_x1/time_0/norm_5000/noise_map.fits diff --git a/temporal/dataset/parallel_x1/time_0/norm_5000/pre_cti_data.fits b/legacy/temporal/dataset/parallel_x1/time_0/norm_5000/pre_cti_data.fits similarity index 100% rename from temporal/dataset/parallel_x1/time_0/norm_5000/pre_cti_data.fits rename to legacy/temporal/dataset/parallel_x1/time_0/norm_5000/pre_cti_data.fits diff --git a/temporal/dataset/parallel_x1/time_1/clocker.json b/legacy/temporal/dataset/parallel_x1/time_1/clocker.json similarity index 100% rename from temporal/dataset/parallel_x1/time_1/clocker.json rename to legacy/temporal/dataset/parallel_x1/time_1/clocker.json diff --git a/temporal/dataset/parallel_x1/time_1/cti.json b/legacy/temporal/dataset/parallel_x1/time_1/cti.json similarity index 100% rename from temporal/dataset/parallel_x1/time_1/cti.json rename to legacy/temporal/dataset/parallel_x1/time_1/cti.json diff --git a/temporal/dataset/parallel_x1/time_1/data_100.fits b/legacy/temporal/dataset/parallel_x1/time_1/data_100.fits similarity index 100% rename from temporal/dataset/parallel_x1/time_1/data_100.fits rename to legacy/temporal/dataset/parallel_x1/time_1/data_100.fits diff --git a/temporal/dataset/parallel_x1/time_1/data_200000.fits b/legacy/temporal/dataset/parallel_x1/time_1/data_200000.fits similarity index 100% rename from temporal/dataset/parallel_x1/time_1/data_200000.fits rename to legacy/temporal/dataset/parallel_x1/time_1/data_200000.fits diff --git a/temporal/dataset/parallel_x1/time_1/data_25000.fits b/legacy/temporal/dataset/parallel_x1/time_1/data_25000.fits similarity index 100% rename from temporal/dataset/parallel_x1/time_1/data_25000.fits rename to legacy/temporal/dataset/parallel_x1/time_1/data_25000.fits diff --git a/temporal/dataset/parallel_x1/time_1/data_5000.fits b/legacy/temporal/dataset/parallel_x1/time_1/data_5000.fits similarity index 100% rename from temporal/dataset/parallel_x1/time_1/data_5000.fits rename to legacy/temporal/dataset/parallel_x1/time_1/data_5000.fits diff --git a/temporal/dataset/parallel_x1/time_1/norm_100/binned_1d/image_parallel_eper.png b/legacy/temporal/dataset/parallel_x1/time_1/norm_100/binned_1d/image_parallel_eper.png similarity index 100% rename from temporal/dataset/parallel_x1/time_1/norm_100/binned_1d/image_parallel_eper.png rename to legacy/temporal/dataset/parallel_x1/time_1/norm_100/binned_1d/image_parallel_eper.png diff --git a/temporal/dataset/parallel_x1/time_1/norm_100/binned_1d/image_parallel_fpr.png b/legacy/temporal/dataset/parallel_x1/time_1/norm_100/binned_1d/image_parallel_fpr.png similarity index 100% rename from temporal/dataset/parallel_x1/time_1/norm_100/binned_1d/image_parallel_fpr.png rename to legacy/temporal/dataset/parallel_x1/time_1/norm_100/binned_1d/image_parallel_fpr.png diff --git a/temporal/dataset/parallel_x1/time_1/norm_100/imaging_ci.png b/legacy/temporal/dataset/parallel_x1/time_1/norm_100/imaging_ci.png similarity index 100% rename from temporal/dataset/parallel_x1/time_1/norm_100/imaging_ci.png rename to legacy/temporal/dataset/parallel_x1/time_1/norm_100/imaging_ci.png diff --git a/temporal/dataset/parallel_x1/time_1/norm_100/noise_map.fits b/legacy/temporal/dataset/parallel_x1/time_1/norm_100/noise_map.fits similarity index 100% rename from temporal/dataset/parallel_x1/time_1/norm_100/noise_map.fits rename to legacy/temporal/dataset/parallel_x1/time_1/norm_100/noise_map.fits diff --git a/temporal/dataset/parallel_x1/time_1/norm_100/pre_cti_data.fits b/legacy/temporal/dataset/parallel_x1/time_1/norm_100/pre_cti_data.fits similarity index 100% rename from temporal/dataset/parallel_x1/time_1/norm_100/pre_cti_data.fits rename to legacy/temporal/dataset/parallel_x1/time_1/norm_100/pre_cti_data.fits diff --git a/temporal/dataset/parallel_x1/time_1/norm_200000/binned_1d/image_parallel_eper.png b/legacy/temporal/dataset/parallel_x1/time_1/norm_200000/binned_1d/image_parallel_eper.png similarity index 100% rename from temporal/dataset/parallel_x1/time_1/norm_200000/binned_1d/image_parallel_eper.png rename to legacy/temporal/dataset/parallel_x1/time_1/norm_200000/binned_1d/image_parallel_eper.png diff --git a/temporal/dataset/parallel_x1/time_1/norm_200000/binned_1d/image_parallel_fpr.png b/legacy/temporal/dataset/parallel_x1/time_1/norm_200000/binned_1d/image_parallel_fpr.png similarity index 100% rename from temporal/dataset/parallel_x1/time_1/norm_200000/binned_1d/image_parallel_fpr.png rename to legacy/temporal/dataset/parallel_x1/time_1/norm_200000/binned_1d/image_parallel_fpr.png diff --git a/temporal/dataset/parallel_x1/time_1/norm_200000/imaging_ci.png b/legacy/temporal/dataset/parallel_x1/time_1/norm_200000/imaging_ci.png similarity index 100% rename from temporal/dataset/parallel_x1/time_1/norm_200000/imaging_ci.png rename to legacy/temporal/dataset/parallel_x1/time_1/norm_200000/imaging_ci.png diff --git a/temporal/dataset/parallel_x1/time_1/norm_200000/noise_map.fits b/legacy/temporal/dataset/parallel_x1/time_1/norm_200000/noise_map.fits similarity index 100% rename from temporal/dataset/parallel_x1/time_1/norm_200000/noise_map.fits rename to legacy/temporal/dataset/parallel_x1/time_1/norm_200000/noise_map.fits diff --git a/temporal/dataset/parallel_x1/time_1/norm_200000/pre_cti_data.fits b/legacy/temporal/dataset/parallel_x1/time_1/norm_200000/pre_cti_data.fits similarity index 100% rename from temporal/dataset/parallel_x1/time_1/norm_200000/pre_cti_data.fits rename to legacy/temporal/dataset/parallel_x1/time_1/norm_200000/pre_cti_data.fits diff --git a/temporal/dataset/parallel_x1/time_1/norm_25000/binned_1d/image_parallel_eper.png b/legacy/temporal/dataset/parallel_x1/time_1/norm_25000/binned_1d/image_parallel_eper.png similarity index 100% rename from temporal/dataset/parallel_x1/time_1/norm_25000/binned_1d/image_parallel_eper.png rename to legacy/temporal/dataset/parallel_x1/time_1/norm_25000/binned_1d/image_parallel_eper.png diff --git a/temporal/dataset/parallel_x1/time_1/norm_25000/binned_1d/image_parallel_fpr.png b/legacy/temporal/dataset/parallel_x1/time_1/norm_25000/binned_1d/image_parallel_fpr.png similarity index 100% rename from temporal/dataset/parallel_x1/time_1/norm_25000/binned_1d/image_parallel_fpr.png rename to legacy/temporal/dataset/parallel_x1/time_1/norm_25000/binned_1d/image_parallel_fpr.png diff --git a/temporal/dataset/parallel_x1/time_1/norm_25000/imaging_ci.png b/legacy/temporal/dataset/parallel_x1/time_1/norm_25000/imaging_ci.png similarity index 100% rename from temporal/dataset/parallel_x1/time_1/norm_25000/imaging_ci.png rename to legacy/temporal/dataset/parallel_x1/time_1/norm_25000/imaging_ci.png diff --git a/temporal/dataset/parallel_x1/time_1/norm_25000/noise_map.fits b/legacy/temporal/dataset/parallel_x1/time_1/norm_25000/noise_map.fits similarity index 100% rename from temporal/dataset/parallel_x1/time_1/norm_25000/noise_map.fits rename to legacy/temporal/dataset/parallel_x1/time_1/norm_25000/noise_map.fits diff --git a/temporal/dataset/parallel_x1/time_1/norm_25000/pre_cti_data.fits b/legacy/temporal/dataset/parallel_x1/time_1/norm_25000/pre_cti_data.fits similarity index 100% rename from temporal/dataset/parallel_x1/time_1/norm_25000/pre_cti_data.fits rename to legacy/temporal/dataset/parallel_x1/time_1/norm_25000/pre_cti_data.fits diff --git a/temporal/dataset/parallel_x1/time_1/norm_5000/binned_1d/image_parallel_eper.png b/legacy/temporal/dataset/parallel_x1/time_1/norm_5000/binned_1d/image_parallel_eper.png similarity index 100% rename from temporal/dataset/parallel_x1/time_1/norm_5000/binned_1d/image_parallel_eper.png rename to legacy/temporal/dataset/parallel_x1/time_1/norm_5000/binned_1d/image_parallel_eper.png diff --git a/temporal/dataset/parallel_x1/time_1/norm_5000/binned_1d/image_parallel_fpr.png b/legacy/temporal/dataset/parallel_x1/time_1/norm_5000/binned_1d/image_parallel_fpr.png similarity index 100% rename from temporal/dataset/parallel_x1/time_1/norm_5000/binned_1d/image_parallel_fpr.png rename to legacy/temporal/dataset/parallel_x1/time_1/norm_5000/binned_1d/image_parallel_fpr.png diff --git a/temporal/dataset/parallel_x1/time_1/norm_5000/imaging_ci.png b/legacy/temporal/dataset/parallel_x1/time_1/norm_5000/imaging_ci.png similarity index 100% rename from temporal/dataset/parallel_x1/time_1/norm_5000/imaging_ci.png rename to legacy/temporal/dataset/parallel_x1/time_1/norm_5000/imaging_ci.png diff --git a/temporal/dataset/parallel_x1/time_1/norm_5000/noise_map.fits b/legacy/temporal/dataset/parallel_x1/time_1/norm_5000/noise_map.fits similarity index 100% rename from temporal/dataset/parallel_x1/time_1/norm_5000/noise_map.fits rename to legacy/temporal/dataset/parallel_x1/time_1/norm_5000/noise_map.fits diff --git a/temporal/dataset/parallel_x1/time_1/norm_5000/pre_cti_data.fits b/legacy/temporal/dataset/parallel_x1/time_1/norm_5000/pre_cti_data.fits similarity index 100% rename from temporal/dataset/parallel_x1/time_1/norm_5000/pre_cti_data.fits rename to legacy/temporal/dataset/parallel_x1/time_1/norm_5000/pre_cti_data.fits diff --git a/temporal/dataset/parallel_x1/time_2/clocker.json b/legacy/temporal/dataset/parallel_x1/time_2/clocker.json similarity index 100% rename from temporal/dataset/parallel_x1/time_2/clocker.json rename to legacy/temporal/dataset/parallel_x1/time_2/clocker.json diff --git a/temporal/dataset/parallel_x1/time_2/cti.json b/legacy/temporal/dataset/parallel_x1/time_2/cti.json similarity index 100% rename from temporal/dataset/parallel_x1/time_2/cti.json rename to legacy/temporal/dataset/parallel_x1/time_2/cti.json diff --git a/temporal/dataset/parallel_x1/time_2/data_100.fits b/legacy/temporal/dataset/parallel_x1/time_2/data_100.fits similarity index 100% rename from temporal/dataset/parallel_x1/time_2/data_100.fits rename to legacy/temporal/dataset/parallel_x1/time_2/data_100.fits diff --git a/temporal/dataset/parallel_x1/time_2/data_200000.fits b/legacy/temporal/dataset/parallel_x1/time_2/data_200000.fits similarity index 100% rename from temporal/dataset/parallel_x1/time_2/data_200000.fits rename to legacy/temporal/dataset/parallel_x1/time_2/data_200000.fits diff --git a/temporal/dataset/parallel_x1/time_2/data_25000.fits b/legacy/temporal/dataset/parallel_x1/time_2/data_25000.fits similarity index 100% rename from temporal/dataset/parallel_x1/time_2/data_25000.fits rename to legacy/temporal/dataset/parallel_x1/time_2/data_25000.fits diff --git a/temporal/dataset/parallel_x1/time_2/data_5000.fits b/legacy/temporal/dataset/parallel_x1/time_2/data_5000.fits similarity index 100% rename from temporal/dataset/parallel_x1/time_2/data_5000.fits rename to legacy/temporal/dataset/parallel_x1/time_2/data_5000.fits diff --git a/temporal/dataset/parallel_x1/time_2/norm_100/binned_1d/image_parallel_eper.png b/legacy/temporal/dataset/parallel_x1/time_2/norm_100/binned_1d/image_parallel_eper.png similarity index 100% rename from temporal/dataset/parallel_x1/time_2/norm_100/binned_1d/image_parallel_eper.png rename to legacy/temporal/dataset/parallel_x1/time_2/norm_100/binned_1d/image_parallel_eper.png diff --git a/temporal/dataset/parallel_x1/time_2/norm_100/binned_1d/image_parallel_fpr.png b/legacy/temporal/dataset/parallel_x1/time_2/norm_100/binned_1d/image_parallel_fpr.png similarity index 100% rename from temporal/dataset/parallel_x1/time_2/norm_100/binned_1d/image_parallel_fpr.png rename to legacy/temporal/dataset/parallel_x1/time_2/norm_100/binned_1d/image_parallel_fpr.png diff --git a/temporal/dataset/parallel_x1/time_2/norm_100/imaging_ci.png b/legacy/temporal/dataset/parallel_x1/time_2/norm_100/imaging_ci.png similarity index 100% rename from temporal/dataset/parallel_x1/time_2/norm_100/imaging_ci.png rename to legacy/temporal/dataset/parallel_x1/time_2/norm_100/imaging_ci.png diff --git a/temporal/dataset/parallel_x1/time_2/norm_100/noise_map.fits b/legacy/temporal/dataset/parallel_x1/time_2/norm_100/noise_map.fits similarity index 100% rename from temporal/dataset/parallel_x1/time_2/norm_100/noise_map.fits rename to legacy/temporal/dataset/parallel_x1/time_2/norm_100/noise_map.fits diff --git a/temporal/dataset/parallel_x1/time_2/norm_100/pre_cti_data.fits b/legacy/temporal/dataset/parallel_x1/time_2/norm_100/pre_cti_data.fits similarity index 100% rename from temporal/dataset/parallel_x1/time_2/norm_100/pre_cti_data.fits rename to legacy/temporal/dataset/parallel_x1/time_2/norm_100/pre_cti_data.fits diff --git a/temporal/dataset/parallel_x1/time_2/norm_200000/binned_1d/image_parallel_eper.png b/legacy/temporal/dataset/parallel_x1/time_2/norm_200000/binned_1d/image_parallel_eper.png similarity index 100% rename from temporal/dataset/parallel_x1/time_2/norm_200000/binned_1d/image_parallel_eper.png rename to legacy/temporal/dataset/parallel_x1/time_2/norm_200000/binned_1d/image_parallel_eper.png diff --git a/temporal/dataset/parallel_x1/time_2/norm_200000/binned_1d/image_parallel_fpr.png b/legacy/temporal/dataset/parallel_x1/time_2/norm_200000/binned_1d/image_parallel_fpr.png similarity index 100% rename from temporal/dataset/parallel_x1/time_2/norm_200000/binned_1d/image_parallel_fpr.png rename to legacy/temporal/dataset/parallel_x1/time_2/norm_200000/binned_1d/image_parallel_fpr.png diff --git a/temporal/dataset/parallel_x1/time_2/norm_200000/imaging_ci.png b/legacy/temporal/dataset/parallel_x1/time_2/norm_200000/imaging_ci.png similarity index 100% rename from temporal/dataset/parallel_x1/time_2/norm_200000/imaging_ci.png rename to legacy/temporal/dataset/parallel_x1/time_2/norm_200000/imaging_ci.png diff --git a/temporal/dataset/parallel_x1/time_2/norm_200000/noise_map.fits b/legacy/temporal/dataset/parallel_x1/time_2/norm_200000/noise_map.fits similarity index 100% rename from temporal/dataset/parallel_x1/time_2/norm_200000/noise_map.fits rename to legacy/temporal/dataset/parallel_x1/time_2/norm_200000/noise_map.fits diff --git a/temporal/dataset/parallel_x1/time_2/norm_200000/pre_cti_data.fits b/legacy/temporal/dataset/parallel_x1/time_2/norm_200000/pre_cti_data.fits similarity index 100% rename from temporal/dataset/parallel_x1/time_2/norm_200000/pre_cti_data.fits rename to legacy/temporal/dataset/parallel_x1/time_2/norm_200000/pre_cti_data.fits diff --git a/temporal/dataset/parallel_x1/time_2/norm_25000/binned_1d/image_parallel_eper.png b/legacy/temporal/dataset/parallel_x1/time_2/norm_25000/binned_1d/image_parallel_eper.png similarity index 100% rename from temporal/dataset/parallel_x1/time_2/norm_25000/binned_1d/image_parallel_eper.png rename to legacy/temporal/dataset/parallel_x1/time_2/norm_25000/binned_1d/image_parallel_eper.png diff --git a/temporal/dataset/parallel_x1/time_2/norm_25000/binned_1d/image_parallel_fpr.png b/legacy/temporal/dataset/parallel_x1/time_2/norm_25000/binned_1d/image_parallel_fpr.png similarity index 100% rename from temporal/dataset/parallel_x1/time_2/norm_25000/binned_1d/image_parallel_fpr.png rename to legacy/temporal/dataset/parallel_x1/time_2/norm_25000/binned_1d/image_parallel_fpr.png diff --git a/temporal/dataset/parallel_x1/time_2/norm_25000/imaging_ci.png b/legacy/temporal/dataset/parallel_x1/time_2/norm_25000/imaging_ci.png similarity index 100% rename from temporal/dataset/parallel_x1/time_2/norm_25000/imaging_ci.png rename to legacy/temporal/dataset/parallel_x1/time_2/norm_25000/imaging_ci.png diff --git a/temporal/dataset/parallel_x1/time_2/norm_25000/noise_map.fits b/legacy/temporal/dataset/parallel_x1/time_2/norm_25000/noise_map.fits similarity index 100% rename from temporal/dataset/parallel_x1/time_2/norm_25000/noise_map.fits rename to legacy/temporal/dataset/parallel_x1/time_2/norm_25000/noise_map.fits diff --git a/temporal/dataset/parallel_x1/time_2/norm_25000/pre_cti_data.fits b/legacy/temporal/dataset/parallel_x1/time_2/norm_25000/pre_cti_data.fits similarity index 100% rename from temporal/dataset/parallel_x1/time_2/norm_25000/pre_cti_data.fits rename to legacy/temporal/dataset/parallel_x1/time_2/norm_25000/pre_cti_data.fits diff --git a/temporal/dataset/parallel_x1/time_2/norm_5000/binned_1d/image_parallel_eper.png b/legacy/temporal/dataset/parallel_x1/time_2/norm_5000/binned_1d/image_parallel_eper.png similarity index 100% rename from temporal/dataset/parallel_x1/time_2/norm_5000/binned_1d/image_parallel_eper.png rename to legacy/temporal/dataset/parallel_x1/time_2/norm_5000/binned_1d/image_parallel_eper.png diff --git a/temporal/dataset/parallel_x1/time_2/norm_5000/binned_1d/image_parallel_fpr.png b/legacy/temporal/dataset/parallel_x1/time_2/norm_5000/binned_1d/image_parallel_fpr.png similarity index 100% rename from temporal/dataset/parallel_x1/time_2/norm_5000/binned_1d/image_parallel_fpr.png rename to legacy/temporal/dataset/parallel_x1/time_2/norm_5000/binned_1d/image_parallel_fpr.png diff --git a/temporal/dataset/parallel_x1/time_2/norm_5000/imaging_ci.png b/legacy/temporal/dataset/parallel_x1/time_2/norm_5000/imaging_ci.png similarity index 100% rename from temporal/dataset/parallel_x1/time_2/norm_5000/imaging_ci.png rename to legacy/temporal/dataset/parallel_x1/time_2/norm_5000/imaging_ci.png diff --git a/temporal/dataset/parallel_x1/time_2/norm_5000/noise_map.fits b/legacy/temporal/dataset/parallel_x1/time_2/norm_5000/noise_map.fits similarity index 100% rename from temporal/dataset/parallel_x1/time_2/norm_5000/noise_map.fits rename to legacy/temporal/dataset/parallel_x1/time_2/norm_5000/noise_map.fits diff --git a/temporal/dataset/parallel_x1/time_2/norm_5000/pre_cti_data.fits b/legacy/temporal/dataset/parallel_x1/time_2/norm_5000/pre_cti_data.fits similarity index 100% rename from temporal/dataset/parallel_x1/time_2/norm_5000/pre_cti_data.fits rename to legacy/temporal/dataset/parallel_x1/time_2/norm_5000/pre_cti_data.fits diff --git a/temporal/model.py b/legacy/temporal/model.py similarity index 100% rename from temporal/model.py rename to legacy/temporal/model.py diff --git a/temporal/simulator.py b/legacy/temporal/simulator.py similarity index 100% rename from temporal/simulator.py rename to legacy/temporal/simulator.py diff --git a/tvac/__init__.py b/legacy/tvac/__init__.py similarity index 100% rename from tvac/__init__.py rename to legacy/tvac/__init__.py diff --git a/tvac/calibrate.py b/legacy/tvac/calibrate.py similarity index 100% rename from tvac/calibrate.py rename to legacy/tvac/calibrate.py diff --git a/tvac/calibrate_file_entry_point.py b/legacy/tvac/calibrate_file_entry_point.py similarity index 100% rename from tvac/calibrate_file_entry_point.py rename to legacy/tvac/calibrate_file_entry_point.py diff --git a/tvac/ci_layout_valid.py b/legacy/tvac/ci_layout_valid.py similarity index 100% rename from tvac/ci_layout_valid.py rename to legacy/tvac/ci_layout_valid.py diff --git a/tvac/cosmic_ray_flagging_iterate.py b/legacy/tvac/cosmic_ray_flagging_iterate.py similarity index 100% rename from tvac/cosmic_ray_flagging_iterate.py rename to legacy/tvac/cosmic_ray_flagging_iterate.py diff --git a/tvac/extract.py b/legacy/tvac/extract.py similarity index 100% rename from tvac/extract.py rename to legacy/tvac/extract.py diff --git a/tvac/fits_extend/add_to_header.py b/legacy/tvac/fits_extend/add_to_header.py similarity index 100% rename from tvac/fits_extend/add_to_header.py rename to legacy/tvac/fits_extend/add_to_header.py diff --git a/tvac/notes b/legacy/tvac/notes similarity index 100% rename from tvac/notes rename to legacy/tvac/notes diff --git a/tvac/pre_cti_visuals.py b/legacy/tvac/pre_cti_visuals.py similarity index 100% rename from tvac/pre_cti_visuals.py rename to legacy/tvac/pre_cti_visuals.py diff --git a/tvac/serial_x1.py b/legacy/tvac/serial_x1.py similarity index 100% rename from tvac/serial_x1.py rename to legacy/tvac/serial_x1.py diff --git a/tvac/unpack.py b/legacy/tvac/unpack.py similarity index 100% rename from tvac/unpack.py rename to legacy/tvac/unpack.py diff --git a/tvac/validate/__init__.py b/legacy/tvac/validate/__init__.py similarity index 100% rename from tvac/validate/__init__.py rename to legacy/tvac/validate/__init__.py diff --git a/tvac/validate/ci_regions.py b/legacy/tvac/validate/ci_regions.py similarity index 100% rename from tvac/validate/ci_regions.py rename to legacy/tvac/validate/ci_regions.py diff --git a/tvac/validate/parallel_overscan.py b/legacy/tvac/validate/parallel_overscan.py similarity index 100% rename from tvac/validate/parallel_overscan.py rename to legacy/tvac/validate/parallel_overscan.py diff --git a/tvac/validate/serial_overscan.py b/legacy/tvac/validate/serial_overscan.py similarity index 100% rename from tvac/validate/serial_overscan.py rename to legacy/tvac/validate/serial_overscan.py diff --git a/tvac/validate/serial_prescan.py b/legacy/tvac/validate/serial_prescan.py similarity index 100% rename from tvac/validate/serial_prescan.py rename to legacy/tvac/validate/serial_prescan.py diff --git a/validation/__init__.py b/legacy/validation/__init__.py similarity index 100% rename from validation/__init__.py rename to legacy/validation/__init__.py diff --git a/validation/no_cti_1d.py b/legacy/validation/no_cti_1d.py similarity index 100% rename from validation/no_cti_1d.py rename to legacy/validation/no_cti_1d.py diff --git a/validation/no_cti_2d.py b/legacy/validation/no_cti_2d.py similarity index 100% rename from validation/no_cti_2d.py rename to legacy/validation/no_cti_2d.py diff --git a/validation/parallel_x1.py b/legacy/validation/parallel_x1.py similarity index 100% rename from validation/parallel_x1.py rename to legacy/validation/parallel_x1.py diff --git a/validation/simulators/__init__.py b/legacy/validation/simulators/__init__.py similarity index 100% rename from validation/simulators/__init__.py rename to legacy/validation/simulators/__init__.py diff --git a/validation/simulators/no_cti.py b/legacy/validation/simulators/no_cti.py similarity index 100% rename from validation/simulators/no_cti.py rename to legacy/validation/simulators/no_cti.py diff --git a/validation/simulators/parallel_x1.py b/legacy/validation/simulators/parallel_x1.py similarity index 100% rename from validation/simulators/parallel_x1.py rename to legacy/validation/simulators/parallel_x1.py diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/dataset_1d/__init__.py b/scripts/dataset_1d/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/dataset_1d/model_fit.py b/scripts/dataset_1d/model_fit.py new file mode 100644 index 0000000..1316174 --- /dev/null +++ b/scripts/dataset_1d/model_fit.py @@ -0,0 +1,133 @@ +""" +Integration: Dataset1D model fit +================================ + +Simulates two 1D CTI datasets at different injection levels, fits them simultaneously via an +`AnalysisFactor` / `FactorGraphModel` model-fit, and round-trips the results through the +aggregator — the core 1D CTI calibration path, end-to-end. + +A single trap species is used so the model has no ordering assertion (which ties at prior +medians under the `PYAUTO_TEST_MODE=2` bypass). +""" + +import os +from os import path + +import autofit as af +import autocti as ac + +""" +__Simulate__ +""" +shape_native = (50,) + +prescan = ac.Region1D(region=(0, 5)) +overscan = ac.Region1D(region=(45, 50)) +region_list = [(5, 25)] + +norm_list = [100.0, 5000.0] + +layout = ac.Layout1D( + shape_1d=shape_native, + region_list=region_list, + prescan=prescan, + overscan=overscan, +) + +clocker = ac.Clocker1D(express=5, roe=ac.ROEChargeInjection()) + +trap = ac.TrapInstantCapture(density=10.0, release_timescale=5.0) +ccd = ac.CCDPhase(well_fill_power=0.5, well_notch_depth=0.0, full_well_depth=200000.0) +cti = ac.CTI1D(trap_list=[trap], ccd=ccd) + +dataset_list = [] + +for norm in norm_list: + simulator = ac.SimulatorDataset1D( + pixel_scales=0.1, + norm=norm, + read_noise=0.01, + add_poisson_noise_to_data=False, + ) + dataset = simulator.via_layout_from(clocker=clocker, layout=layout, cti=cti) + dataset_list.append(dataset) + +""" +__Model + Fit__ +""" +model = af.Collection( + cti=af.Model( + ac.CTI1D, + trap_list=[af.Model(ac.TrapInstantCapture)], + ccd=af.Model(ac.CCDPhase, well_notch_depth=0.0, full_well_depth=200000.0), + ) +) + +search = af.Nautilus( + path_prefix=path.join("dataset_1d"), + name="model_fit", + unique_tag="integration", + n_live=75, +) + +analysis_list = [ + ac.AnalysisDataset1D(dataset=dataset, clocker=clocker) for dataset in dataset_list +] + +analysis_factor_list = [ + af.AnalysisFactor(prior_model=model, analysis=analysis) + for analysis in analysis_list +] + +factor_graph = af.FactorGraphModel(*analysis_factor_list) + +result_list = search.fit(model=factor_graph.global_prior_model, analysis=factor_graph) + +print("max log likelihood:", result_list[0].log_likelihood) +print(result_list[0].max_log_likelihood_instance.cti.trap_list[0].density) + +fit = result_list[0].max_log_likelihood_fit +print("fit chi squared:", fit.chi_squared) + +""" +__Aggregator round-trip__ +""" +from pathlib import Path + +from autoconf import conf +from autoconf.test_mode import with_test_mode_segment + +database_file = "dataset_1d_integration.sqlite" + +if path.exists(database_file): + os.remove(database_file) + +agg = af.Aggregator.from_database( + filename=database_file, completed_only=False, top_level_only=True +) +agg.add_directory( + directory=str(with_test_mode_segment(Path(conf.instance.output_path)) / "dataset_1d") +) + +dataset_agg = ac.agg.Dataset1DAgg(aggregator=agg) + +total_datasets = 0 +for dataset_sublist in dataset_agg.dataset_list_gen_from(): + total_datasets += len(dataset_sublist) + +assert total_datasets >= len(norm_list), ( + f"aggregator returned {total_datasets} datasets, expected >= {len(norm_list)}" +) + +fit_agg = ac.agg.FitDataset1DAgg(aggregator=agg) + +total_fits = 0 +for fit_sublist in fit_agg.max_log_likelihood_gen_from(): + total_fits += len(fit_sublist) + +assert total_fits >= len(norm_list) + +if path.exists(database_file): + os.remove(database_file) + +print("dataset_1d integration: OK") diff --git a/scripts/imaging_ci/__init__.py b/scripts/imaging_ci/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/imaging_ci/model_fit.py b/scripts/imaging_ci/model_fit.py new file mode 100644 index 0000000..11adf16 --- /dev/null +++ b/scripts/imaging_ci/model_fit.py @@ -0,0 +1,89 @@ +""" +Integration: ImagingCI model fit +================================ + +Simulates a small charge injection image with parallel CTI, fits it with a parallel CTI model +via an `AnalysisFactor` / `FactorGraphModel` model-fit, and inspects the result — the core 2D +charge-injection calibration path, end-to-end. + +A single trap species is used so the model has no ordering assertion (which ties at prior +medians under the `PYAUTO_TEST_MODE=2` bypass). +""" + +from os import path + +import autofit as af +import autocti as ac + +""" +__Simulate__ +""" +shape_native = (30, 30) + +parallel_overscan = ac.Region2D((25, 30, 1, 29)) +serial_prescan = ac.Region2D((0, 30, 0, 1)) +serial_overscan = ac.Region2D((0, 25, 29, 30)) + +region_list = [(0, 25, serial_prescan[3], serial_overscan[2])] + +norm = 100.0 + +layout = ac.Layout2DCI( + shape_2d=shape_native, + region_list=region_list, + parallel_overscan=parallel_overscan, + serial_prescan=serial_prescan, + serial_overscan=serial_overscan, +) + +clocker = ac.Clocker2D(parallel_express=5, parallel_roe=ac.ROEChargeInjection()) + +parallel_trap = ac.TrapInstantCapture(density=10.0, release_timescale=5.0) +parallel_ccd = ac.CCDPhase( + well_fill_power=0.5, well_notch_depth=0.0, full_well_depth=200000.0 +) +cti = ac.CTI2D(parallel_trap_list=[parallel_trap], parallel_ccd=parallel_ccd) + +simulator = ac.SimulatorImagingCI(read_noise=0.01, pixel_scales=0.1, norm=norm) + +dataset = simulator.via_layout_from(clocker=clocker, layout=layout, cti=cti) + +""" +__Model + Fit__ +""" +model = af.Collection( + cti=af.Model( + ac.CTI2D, + parallel_trap_list=[af.Model(ac.TrapInstantCapture)], + parallel_ccd=af.Model( + ac.CCDPhase, well_notch_depth=0.0, full_well_depth=200000.0 + ), + ) +) + +search = af.Nautilus( + path_prefix=path.join("imaging_ci"), + name="model_fit", + unique_tag="integration", + n_live=75, +) + +analysis = ac.AnalysisImagingCI(dataset=dataset, clocker=clocker) + +analysis_factor_list = [af.AnalysisFactor(prior_model=model, analysis=analysis)] + +factor_graph = af.FactorGraphModel(*analysis_factor_list) + +result_list = search.fit(model=factor_graph.global_prior_model, analysis=factor_graph) + +result = result_list[0] + +print("max log likelihood:", result.log_likelihood) +print(result.max_log_likelihood_instance.cti.parallel_trap_list[0].density) + +fit = result.max_log_likelihood_fit +print("fit chi squared:", fit.chi_squared) + +assert fit.chi_squared is not None + +print("imaging_ci integration: OK") diff --git a/scripts/plot/__init__.py b/scripts/plot/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/plot/subplots.py b/scripts/plot/subplots.py new file mode 100644 index 0000000..5d8724a --- /dev/null +++ b/scripts/plot/subplots.py @@ -0,0 +1,152 @@ +""" +Integration: plot function API +============================== + +Drives the main `autocti.plot` function surface — dataset and fit subplots, region figures, +combined lists and the binned-data diagnostic — writing every figure to disk, for both a 1D +dataset and a 2D charge injection dataset. +""" + +import shutil +from os import path + +import autocti as ac +import autocti.plot as aplt + +output_root = path.join("output", "plot_integration") + +if path.exists(output_root): + shutil.rmtree(output_root) + +output_path_1d = path.join(output_root, "dataset_1d") +output_path_2d = path.join(output_root, "imaging_ci") + +""" +__Dataset1D__ +""" +layout_1d = ac.Layout1D( + shape_1d=(50,), + region_list=[(5, 25)], + prescan=ac.Region1D(region=(0, 5)), + overscan=ac.Region1D(region=(45, 50)), +) + +clocker_1d = ac.Clocker1D(express=5, roe=ac.ROEChargeInjection()) +cti_1d = ac.CTI1D( + trap_list=[ac.TrapInstantCapture(density=10.0, release_timescale=5.0)], + ccd=ac.CCDPhase(well_fill_power=0.5, well_notch_depth=0.0, full_well_depth=200000.0), +) + +simulator_1d = ac.SimulatorDataset1D(read_noise=0.01, pixel_scales=0.1, norm=100.0) +dataset_1d = simulator_1d.via_layout_from( + clocker=clocker_1d, layout=layout_1d, cti=cti_1d +) + +aplt.figure_dataset_1d_data( + dataset=dataset_1d, output_path=output_path_1d, output_format="png" +) +aplt.figure_dataset_1d_data( + dataset=dataset_1d, region="eper", logy=True, output_path=output_path_1d, output_format="png" +) +aplt.subplot_dataset_1d(dataset=dataset_1d, output_path=output_path_1d, output_format="png") +aplt.subplot_dataset_1d( + dataset=dataset_1d, region="fpr", output_path=output_path_1d, output_format="png" +) +aplt.subplot_dataset_1d_list( + dataset_list=[dataset_1d, dataset_1d], output_path=output_path_1d, output_format="png" +) + +fit_1d = ac.FitDataset1D( + dataset=dataset_1d.apply_mask( + mask=ac.Mask1D.all_false( + shape_slim=dataset_1d.data.shape_slim, + pixel_scales=dataset_1d.data.pixel_scales, + ) + ), + post_cti_data=clocker_1d.add_cti(data=dataset_1d.pre_cti_data, cti=cti_1d), +) + +aplt.subplot_fit_dataset_1d(fit=fit_1d, output_path=output_path_1d, output_format="png") +aplt.figure_fit_dataset_1d( + fit=fit_1d, + quantity="residual_map", + region="eper", + output_path=output_path_1d, + output_format="png", +) +aplt.fits_fit_dataset_1d(fit=fit_1d, output_path=output_path_1d) + +""" +__ImagingCI__ +""" +layout_2d = ac.Layout2DCI( + shape_2d=(30, 30), + region_list=[(0, 25, 1, 29)], + parallel_overscan=ac.Region2D((25, 30, 1, 29)), + serial_prescan=ac.Region2D((0, 30, 0, 1)), + serial_overscan=ac.Region2D((0, 25, 29, 30)), +) + +clocker_2d = ac.Clocker2D(parallel_express=5, parallel_roe=ac.ROEChargeInjection()) +cti_2d = ac.CTI2D( + parallel_trap_list=[ac.TrapInstantCapture(density=10.0, release_timescale=5.0)], + parallel_ccd=ac.CCDPhase( + well_fill_power=0.5, well_notch_depth=0.0, full_well_depth=200000.0 + ), +) + +simulator_2d = ac.SimulatorImagingCI(read_noise=0.01, pixel_scales=0.1, norm=100.0) +dataset_2d = simulator_2d.via_layout_from( + clocker=clocker_2d, layout=layout_2d, cti=cti_2d +) + +aplt.plot_array( + array=dataset_2d.data, + title="Data", + output_path=output_path_2d, + output_filename="data_2d", + output_format="png", +) +aplt.subplot_imaging_ci(dataset=dataset_2d, output_path=output_path_2d, output_format="png") +aplt.subplot_imaging_ci_region( + dataset=dataset_2d, region="parallel_eper", output_path=output_path_2d, output_format="png" +) +aplt.figure_imaging_ci_data_region( + dataset=dataset_2d, + region="parallel_fpr", + output_path=output_path_2d, + output_format="png", +) +aplt.subplot_imaging_ci_data_binned( + dataset=dataset_2d, output_path=output_path_2d, output_format="png" +) + +fit_2d = ac.FitImagingCI( + dataset=dataset_2d.apply_mask( + mask=ac.Mask2D.all_false( + shape_native=dataset_2d.shape_native, pixel_scales=dataset_2d.pixel_scales + ) + ), + post_cti_data=clocker_2d.add_cti(data=dataset_2d.pre_cti_data, cti=cti_2d), +) + +aplt.subplot_fit_ci(fit=fit_2d, output_path=output_path_2d, output_format="png") +aplt.subplot_fit_ci_region( + fit=fit_2d, region="parallel_eper", output_path=output_path_2d, output_format="png" +) +aplt.fits_fit_ci(fit=fit_2d, output_path=output_path_2d) + +""" +__Verify__ +""" +import glob + +n_png_1d = len(glob.glob(path.join(output_path_1d, "*.png"))) +n_png_2d = len(glob.glob(path.join(output_path_2d, "*.png"))) +n_fits = len(glob.glob(path.join(output_root, "*", "*.fits"))) + +assert n_png_1d >= 7, f"expected >= 7 dataset_1d pngs, found {n_png_1d}" +assert n_png_2d >= 7, f"expected >= 7 imaging_ci pngs, found {n_png_2d}" +assert n_fits >= 2, f"expected >= 2 fits files, found {n_fits}" + +print(f"plot integration: OK ({n_png_1d}+{n_png_2d} pngs, {n_fits} fits)") diff --git a/smoke_tests.txt b/smoke_tests.txt new file mode 100644 index 0000000..3a715fc --- /dev/null +++ b/smoke_tests.txt @@ -0,0 +1,3 @@ +dataset_1d/model_fit.py +imaging_ci/model_fit.py +plot/subplots.py