From 15156a66d70541d2e2cd0e45d987ea486a530b74 Mon Sep 17 00:00:00 2001 From: Jammy2211 Date: Fri, 17 Jul 2026 16:32:55 +0100 Subject: [PATCH] feat(skills): author the ac_* CTI calibration skill set (Phase 2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Grow the lightweight seed into a working assistant: nine skills covering the CTI calibration pipeline end-to-end, each grounded in a validated autocti_workspace script rather than API memory, and each verified against the live stack. Skills: ac_setup_environment — install the stack incl. arcticpy (the #1 failure) ac_workspace_navigation — the map: two geometries, shared stage pipeline ac_simulate_dataset_1d — simulate a 1D CTI dataset with a known trap model ac_simulate_imaging_ci — simulate a 2D charge-injection image (Clocker2D) ac_mask_and_extract — mask FPR/EPER; extract regions via layout.extract ac_fit_cti_model — compose + fit, incl. the AnalysisFactor / FactorGraphModel joint fit over several charge lines (the core capability) ac_correct_cti — clocker.remove_cti, for correction + calibration diagnosis ac_plot — the autocti.plot function API (no MatPlot/Output object) ac_load_results — result objects + the aggregator (ac.agg.CTIAgg generators) Wiring: symlinked into .claude/skills/, indexed in a rewritten skills/README.md (the old index was the autolens catalogue with al_->ac_ substitution, listing 45 skills of which only 13 files existed — now an honest CTI catalogue pointing at PENDING.md for the growth queue), matching PENDING entries removed. Grounding surfaced two facts worth recording: the aggregator is ac.agg.CTIAgg (the seed prompt guessed Dataset1DAgg — both exist, CTIAgg is what the scripts use), and the plot scripts live at scripts/plot/, not per-geometry. Validation (audit_skill_apis.py against the live CTI stack): --scope all PASS (0 broken across 33 symbols) --check-citations PASS (0 missing across 27 citations) --lint-idioms PASS --check-provenance PASS --check-version PASS — wrote wiki/core/api_audit_baseline.json, pinning the four-module CTI stack (no autogalaxy) at the current main. Also fixes the pre-existing _style.md lensing-path citation (scripts/imaging/ -> scripts/dataset_1d/modeling/), a seed content leak. Part of #136 (Phase 2). Wiki (Phase 3) and demonstrations (Phase 4) still to come. Co-Authored-By: Claude Opus 4.8 --- .claude/skills/ac_correct_cti.md | 1 + .claude/skills/ac_fit_cti_model.md | 1 + .claude/skills/ac_load_results.md | 1 + .claude/skills/ac_mask_and_extract.md | 1 + .claude/skills/ac_plot.md | 1 + .claude/skills/ac_setup_environment.md | 1 + .claude/skills/ac_simulate_dataset_1d.md | 1 + .claude/skills/ac_simulate_imaging_ci.md | 1 + .claude/skills/ac_workspace_navigation.md | 1 + PENDING.md | 4 - skills/README.md | 234 ++++++---------------- skills/_style.md | 2 +- skills/ac_correct_cti.md | 74 +++++++ skills/ac_fit_cti_model.md | 136 +++++++++++++ skills/ac_load_results.md | 95 +++++++++ skills/ac_mask_and_extract.md | 91 +++++++++ skills/ac_plot.md | 79 ++++++++ skills/ac_setup_environment.md | 127 ++++++++++++ skills/ac_simulate_dataset_1d.md | 117 +++++++++++ skills/ac_simulate_imaging_ci.md | 101 ++++++++++ skills/ac_workspace_navigation.md | 62 ++++++ wiki/core/api_audit_baseline.json | 32 +++ 22 files changed, 989 insertions(+), 174 deletions(-) create mode 120000 .claude/skills/ac_correct_cti.md create mode 120000 .claude/skills/ac_fit_cti_model.md create mode 120000 .claude/skills/ac_load_results.md create mode 120000 .claude/skills/ac_mask_and_extract.md create mode 120000 .claude/skills/ac_plot.md create mode 120000 .claude/skills/ac_setup_environment.md create mode 120000 .claude/skills/ac_simulate_dataset_1d.md create mode 120000 .claude/skills/ac_simulate_imaging_ci.md create mode 120000 .claude/skills/ac_workspace_navigation.md create mode 100644 skills/ac_correct_cti.md create mode 100644 skills/ac_fit_cti_model.md create mode 100644 skills/ac_load_results.md create mode 100644 skills/ac_mask_and_extract.md create mode 100644 skills/ac_plot.md create mode 100644 skills/ac_setup_environment.md create mode 100644 skills/ac_simulate_dataset_1d.md create mode 100644 skills/ac_simulate_imaging_ci.md create mode 100644 skills/ac_workspace_navigation.md create mode 100644 wiki/core/api_audit_baseline.json diff --git a/.claude/skills/ac_correct_cti.md b/.claude/skills/ac_correct_cti.md new file mode 120000 index 0000000..ba878d9 --- /dev/null +++ b/.claude/skills/ac_correct_cti.md @@ -0,0 +1 @@ +../../skills/ac_correct_cti.md \ No newline at end of file diff --git a/.claude/skills/ac_fit_cti_model.md b/.claude/skills/ac_fit_cti_model.md new file mode 120000 index 0000000..db0a0c0 --- /dev/null +++ b/.claude/skills/ac_fit_cti_model.md @@ -0,0 +1 @@ +../../skills/ac_fit_cti_model.md \ No newline at end of file diff --git a/.claude/skills/ac_load_results.md b/.claude/skills/ac_load_results.md new file mode 120000 index 0000000..3f0e653 --- /dev/null +++ b/.claude/skills/ac_load_results.md @@ -0,0 +1 @@ +../../skills/ac_load_results.md \ No newline at end of file diff --git a/.claude/skills/ac_mask_and_extract.md b/.claude/skills/ac_mask_and_extract.md new file mode 120000 index 0000000..c71d0c2 --- /dev/null +++ b/.claude/skills/ac_mask_and_extract.md @@ -0,0 +1 @@ +../../skills/ac_mask_and_extract.md \ No newline at end of file diff --git a/.claude/skills/ac_plot.md b/.claude/skills/ac_plot.md new file mode 120000 index 0000000..9c3e182 --- /dev/null +++ b/.claude/skills/ac_plot.md @@ -0,0 +1 @@ +../../skills/ac_plot.md \ No newline at end of file diff --git a/.claude/skills/ac_setup_environment.md b/.claude/skills/ac_setup_environment.md new file mode 120000 index 0000000..9b88203 --- /dev/null +++ b/.claude/skills/ac_setup_environment.md @@ -0,0 +1 @@ +../../skills/ac_setup_environment.md \ No newline at end of file diff --git a/.claude/skills/ac_simulate_dataset_1d.md b/.claude/skills/ac_simulate_dataset_1d.md new file mode 120000 index 0000000..65ecfdc --- /dev/null +++ b/.claude/skills/ac_simulate_dataset_1d.md @@ -0,0 +1 @@ +../../skills/ac_simulate_dataset_1d.md \ No newline at end of file diff --git a/.claude/skills/ac_simulate_imaging_ci.md b/.claude/skills/ac_simulate_imaging_ci.md new file mode 120000 index 0000000..a48bf72 --- /dev/null +++ b/.claude/skills/ac_simulate_imaging_ci.md @@ -0,0 +1 @@ +../../skills/ac_simulate_imaging_ci.md \ No newline at end of file diff --git a/.claude/skills/ac_workspace_navigation.md b/.claude/skills/ac_workspace_navigation.md new file mode 120000 index 0000000..4bd5ae3 --- /dev/null +++ b/.claude/skills/ac_workspace_navigation.md @@ -0,0 +1 @@ +../../skills/ac_workspace_navigation.md \ No newline at end of file diff --git a/PENDING.md b/PENDING.md index 32e273c..92fd654 100644 --- a/PENDING.md +++ b/PENDING.md @@ -21,7 +21,6 @@ wiki page or dataset, in the reference's shape. - [ ] `.claude/skills/ac_hierarchicac_inference.md` — regenerate for this domain (reference: `.claude/skills/al_hierarchical_inference.md`) - [ ] `.claude/skills/ac_inspect_results_mcp.md` — regenerate for this domain (reference: `.claude/skills/al_inspect_results_mcp.md`) - [ ] `.claude/skills/ac_inspect_source_reconstruction.md` — regenerate for this domain (reference: `.claude/skills/al_inspect_source_reconstruction.md`) -- [ ] `.claude/skills/ac_load_results.md` — regenerate for this domain (reference: `.claude/skills/al_load_results.md`) - [ ] `.claude/skills/ac_mge_decomposition.md` — regenerate for this domain (reference: `.claude/skills/al_mge_decomposition.md`) - [ ] `.claude/skills/ac_multi_dataset.md` — regenerate for this domain (reference: `.claude/skills/al_multi_dataset.md`) - [ ] `.claude/skills/ac_plot_fit_residuals.md` — regenerate for this domain (reference: `.claude/skills/al_plot_fit_residuals.md`) @@ -32,7 +31,6 @@ wiki page or dataset, in the reference's shape. - [ ] `.claude/skills/ac_run_search.md` — regenerate for this domain (reference: `.claude/skills/al_run_search.md`) - [ ] `.claude/skills/ac_run_slam_pipeline.md` — regenerate for this domain (reference: `.claude/skills/al_run_slam_pipeline.md`) - [ ] `.claude/skills/ac_sensitivity_mapping.md` — regenerate for this domain (reference: `.claude/skills/al_sensitivity_mapping.md`) -- [ ] `.claude/skills/ac_setup_environment.md` — regenerate for this domain (reference: `.claude/skills/al_setup_environment.md`) - [ ] `.claude/skills/ac_simulate_dataset.md` — regenerate for this domain (reference: `.claude/skills/al_simulate_dataset.md`) - [ ] `.claude/skills/ac_subhalo_detect.md` — regenerate for this domain (reference: `.claude/skills/al_subhalo_detect.md`) - [ ] `.claude/skills/ac_time_delay_cosmography.md` — regenerate for this domain (reference: `.claude/skills/al_time_delay_cosmography.md`) @@ -113,7 +111,6 @@ wiki page or dataset, in the reference's shape. - [ ] `skills/ac_ingest_paper.md` — regenerate for this domain (reference: `skills/al_ingest_paper.md`) - [ ] `skills/ac_inspect_results_mcp.md` — regenerate for this domain (reference: `skills/al_inspect_results_mcp.md`) - [ ] `skills/ac_inspect_source_reconstruction.md` — regenerate for this domain (reference: `skills/al_inspect_source_reconstruction.md`) -- [ ] `skills/ac_load_results.md` — regenerate for this domain (reference: `skills/al_load_results.md`) - [ ] `skills/ac_mge_decomposition.md` — regenerate for this domain (reference: `skills/al_mge_decomposition.md`) - [ ] `skills/ac_multi_dataset.md` — regenerate for this domain (reference: `skills/al_multi_dataset.md`) - [ ] `skills/ac_plot_fit_residuals.md` — regenerate for this domain (reference: `skills/al_plot_fit_residuals.md`) @@ -124,7 +121,6 @@ wiki page or dataset, in the reference's shape. - [ ] `skills/ac_run_search.md` — regenerate for this domain (reference: `skills/al_run_search.md`) - [ ] `skills/ac_run_slam_pipeline.md` — regenerate for this domain (reference: `skills/al_run_slam_pipeline.md`) - [ ] `skills/ac_sensitivity_mapping.md` — regenerate for this domain (reference: `skills/al_sensitivity_mapping.md`) -- [ ] `skills/ac_setup_environment.md` — regenerate for this domain (reference: `skills/al_setup_environment.md`) - [ ] `skills/ac_simulate_dataset.md` — regenerate for this domain (reference: `skills/al_simulate_dataset.md`) - [ ] `skills/ac_subhalo_detect.md` — regenerate for this domain (reference: `skills/al_subhalo_detect.md`) - [ ] `skills/ac_time_delay_cosmography.md` — regenerate for this domain (reference: `skills/al_time_delay_cosmography.md`) diff --git a/skills/README.md b/skills/README.md index c69e1cf..417c04b 100644 --- a/skills/README.md +++ b/skills/README.md @@ -1,185 +1,81 @@ # skills/ -Procedural how-to-do-X skills for the PyAuto\* lensing stack. Each skill is a single -Markdown file with YAML frontmatter; the body teaches an agent (and through them, the -user) how to write Python that accomplishes one lensing task. +Procedural how-to-do-X skills for the PyAutoCTI stack. Each skill is a single +Markdown file with YAML frontmatter; the body teaches an agent (and through them, +the user) how to write Python that accomplishes one CTI calibration task. -Skills are also exposed at `.claude/skills/` (Claude Code) and `~/.codex/skills/` (when -configured) via symlinks; the canonical files live here. +Skills are also exposed at `.claude/skills/` (Claude Code) and `~/.codex/skills/` +(when configured) via symlinks; the canonical files live here. ## Conventions -- File names use the `ac_` convention for lensing-API skills, e.g. `ac_run_search.md`. -- Project-workflow skills (repo-level operations, template manipulation) use a plain - kebab-case name, e.g. `init-slam.md`, `start-new-project.md`. -- Domain-mode skills that pair an external pipeline repo to the assistant use that - domain as prefix — `euclid_.md` for the Euclid pipeline (euclid mode). +- File names use the `ac_` convention for CTI-API skills, e.g. + `ac_fit_cti_model.md`. +- Project-workflow skills (repo-level operations, template manipulation) use a + plain kebab-case name, e.g. `start-new-project.md`. - Meta-skills (writing guide, bootstrap protocol) start with `_`. -- Every lensing-API skill is **python-first**: the deliverable is a runnable `.py` script - + the understanding to evolve it. Project-workflow skills may instead drive `rsync`, - `cp`, or other repo-level operations. -- Source citations use the project-name + repo-relative-path form, - e.g. `PyAutoFit:autofit/non_linear/search/nest/nautilus/`, resolved via - [`../sources.yaml`](../sources.yaml). -- Wiki references use workspace-relative paths, - e.g. `wiki/core/concepts/non_linear_search.md`. +- Every CTI-API skill is **python-first**: the deliverable is a runnable `.py` + script + the understanding to evolve it, grounded in a validated + `autocti_workspace` example — never API memory. +- Source citations use the project-name + repo-relative-path form, e.g. + `PyAutoCTI:autocti/clocker/`, resolved via [`../sources.yaml`](../sources.yaml). +- Wiki references use workspace-relative paths, e.g. + `wiki/core/concepts/charge_transfer_inefficiency.md`. ## Index -Every skill below is a **complete recipe** unless marked `(stub)` — the stubs are gathered -under "Pending — stubbed" at the end, with a queue of catalogued-but-unstubbed topics after -them. +The recipes below are the assistant's current CTI capability. The full growth +queue — skills, wiki pages and datasets still to be authored — is +[`../PENDING.md`](../PENDING.md); this cell was born as a lightweight seed and is +grown in use. ### Meta -- [`_style.md`](./_style.md) — writing guide every skill is authored against. Read first - before adding or editing any skill. -- [`_bootstrap_skill.md`](./_bootstrap_skill.md) — protocol for authoring a new skill on - demand when a user requests a capability not yet covered. - -### Setup & maintenance - -- [`ac_setup_environment.md`](./ac_setup_environment.md) — detect absent or broken PyAuto\* - environments, install via pip or editable clones when needed, configure caches, verify imports. -- [`ac_update_wiki.md`](./ac_update_wiki.md) — refresh `wiki/core/` pages whose pinned - source commits have moved; surface new public APIs for review. -- [`ac_audit_skill_apis.md`](./ac_audit_skill_apis.md) — verify every PyAuto\* symbol - cited in `skills/` and `wiki/core/api+stack/` resolves in the installed stack; - report stale references with suggested replacements. -- [`ac_refresh_api_docs.md`](./ac_refresh_api_docs.md) — orchestrate a full maintenance - sweep across skill recipes, wiki API pages, and pinned-source drift after a PyAuto\* - upgrade or source refresh. -- [`ac_ingest_paper.md`](./ac_ingest_paper.md) — add a strong-lensing paper (local PDF - or arxiv URL): project-local `wiki/project/bibliography.md` by default inside a science - project; shared `wiki/literature/` in the assistant clone or on explicit promotion. +- [`_style.md`](./_style.md) — writing guide every skill is authored against. + Read first before adding or editing any skill. +- [`_bootstrap_skill.md`](./_bootstrap_skill.md) — protocol for authoring a new + skill on demand when a user requests a capability not yet covered. + +### Setup + +- [`ac_setup_environment.md`](./ac_setup_environment.md) — install and verify the + PyAutoCTI stack, **including arcticpy** (the C++ arctic clocking code `import + autocti` needs but pip won't install). The #1 setup failure; covers the + libgsl-dev + numpy/cython + arcticpy recipe and the numpy-downgrade trap. + +### The CTI calibration pipeline + +Grounded per-skill in the matching `autocti_workspace` stage; navigate the whole +catalogue with `ac_workspace_navigation`. + +- [`ac_workspace_navigation.md`](./ac_workspace_navigation.md) — the map: the two + geometries (`dataset_1d`, `imaging_ci`), their shared stage pipeline, and which + `start_here.py` to read for any task. +- [`ac_simulate_dataset_1d.md`](./ac_simulate_dataset_1d.md) — simulate a 1D CTI + dataset (charge lines + a known trap model) to generate ground truth. +- [`ac_simulate_imaging_ci.md`](./ac_simulate_imaging_ci.md) — simulate a 2D + charge-injection image (the Euclid VIS / HST ACS geometry; parallel/serial CTI, + `Clocker2D`). +- [`ac_mask_and_extract.md`](./ac_mask_and_extract.md) — mask the FPR/EPER before + fitting, and extract FPR/EPER regions as arrays (`layout.extract.fpr/eper`). +- [`ac_fit_cti_model.md`](./ac_fit_cti_model.md) — compose a CTI model and fit it + with a non-linear search, including the **factor graph** that fits several + charge lines jointly. The core capability. +- [`ac_correct_cti.md`](./ac_correct_cti.md) — remove CTI from data with + `clocker.remove_cti`, to correct science data or diagnose a calibration. +- [`ac_load_results.md`](./ac_load_results.md) — inspect a fit's `result`, and + aggregate across many fits (`Aggregator`, `ac.agg.CTIAgg` and its generators). + +### Plotting + +- [`ac_plot.md`](./ac_plot.md) — the `autocti.plot` **function API** + (the `subplot_*` / `figure_*` families, `aplt.plot_yx`, `aplt.plot_array`, `aplt.plot_cti_1d`, the + `*_list` combined subplots). No MatPlot/Output/Visuals object. ### Project workflow -- [`start-new-project.md`](./start-new-project.md) — the single bridge to a standalone - **science project** and its full lifecycle (Create → Work → Collaborate → Publish): - scaffold a lean repo that copies the reproducible science and refers back to the assistant - for skills/wiki, run modelling with reproducibility manifests + the `wiki/project/` journal, - build collaborator summaries, and harden for an open-science release (CITATION/license/Zenodo). - Optional HPC folder. -- [`contribute-upstream.md`](./contribute-upstream.md) — prepare a scoped change, - push it either to your collaborator branch on `PyAutoLabs/autocti_assistant` - or to your fork, and open a draft PR into `PyAutoLabs/autocti_assistant`. -- [`init-slam.md`](./init-slam.md) — populate an empty `scripts/` folder with SLaM - pipeline script(s) copied from `autocti_workspace` and tailored to the chosen data - type. - -### Euclid mode (pipeline-paired) - -Skills pairing the collaboration's -[`euclid_strong_lens_modeling_pipeline`](https://github.com/PyAutoLabs/euclid_strong_lens_modeling_pipeline) -to the assistant; the paired literature context is the dedicated -[`wiki/euclid/`](../wiki/euclid/index.md) sub-wiki. - -- [`euclid_setup_pipeline.md`](./euclid_setup_pipeline.md) — clone/install the pipeline, - the `dataset///` layout, and the black-box `start_here.py` run. -- [`euclid_prepare_data.md`](./euclid_prepare_data.md) — segmentation validation, binary - mask tuning, extra-galaxy masks/centres via the GUI tools, hand-assembling a dataset - folder. -- [`euclid_model_lens.md`](./euclid_model_lens.md) — choose and run the staged - pipelines: initial MGE+SIE, Sersic photometry, lens-only subtraction, multi-waveband, - full SLaM. -- [`euclid_workflow_products.md`](./euclid_workflow_products.md) — aggregate many fits - into .csv catalogues, .fits stacks, and one-line .png summaries via `workflow/`. -- [`euclid_hpc_runs.md`](./euclid_hpc_runs.md) — `hpc/sync` configuration and the - SLURM batch-array templates for sample-scale runs. - -### Data preparation - -- [`ac_prepare_imaging_data.md`](./ac_prepare_imaging_data.md) — load and preprocess - FITS imaging, decide masking for real data, measure noise, prepare PSF. -- [`ac_simulate_dataset.md`](./ac_simulate_dataset.md) — synthesise a lens dataset - (imaging or interferometer) from a ground-truth model. - -### Model building - -- [`ac_build_imaging_model.md`](./ac_build_imaging_model.md) — compose a `Tracer` from - light + mass profiles and wrap it in an `AnalysisImaging`. -- [`ac_build_interferometer_model.md`](./ac_build_interferometer_model.md) — same, but - for visibility-plane data. -- [`ac_custom_profile.md`](./ac_custom_profile.md) — write a new light or mass profile - subclass and register it for use in models. - -### Fitting - -- [`ac_configure_search.md`](./ac_configure_search.md) — pick and tune a non-linear - search (Nautilus, Dynesty, Emcee, Zeus, …) for your problem. -- [`ac_run_search.md`](./ac_run_search.md) — execute `search.fit(model=..., analysis=...)` - and monitor convergence. -- [`ac_chain_searches.md`](./ac_chain_searches.md) — sequence searches so a later phase - inherits priors from an earlier one. -- [`ac_run_slam_pipeline.md`](./ac_run_slam_pipeline.md) — run a Source-Light-Mass - pipeline (the canonical automated lensing workflow). -- [`ac_debug_fit_failure.md`](./ac_debug_fit_failure.md) — diagnose a fit that didn't - converge or produced unphysical results. - -### Results & visualisation - -- [`ac_load_results.md`](./ac_load_results.md) — load a completed fit's `Tracer`, - `Samples`, dataset and FITS products from its output folder. -- [`ac_inspect_results_mcp.md`](./ac_inspect_results_mcp.md) — the read-only - results-inspector MCP server: browse fits, summaries, result images and bulk - subplot/FITS extraction from chat harnesses without code execution (Claude - Desktop first). -- [`ac_plot_tracer.md`](./ac_plot_tracer.md) — plot ray tracing, critical curves, - caustics, magnification maps. -- [`ac_plot_fit_residuals.md`](./ac_plot_fit_residuals.md) — plot model image, - residuals, normalised residuals, chi-squared map. -- [`ac_inspect_source_reconstruction.md`](./ac_inspect_source_reconstruction.md) — - inspect a pixelised inversion: regularisation, source-plane image, reconstruction - diagnostics. -- [`ac_to_notebook.md`](./ac_to_notebook.md) — convert a generated narrative-docstring - script to a Jupyter notebook (docstrings → markdown cells, code → code cells) via the - stdlib-only `autoassistant/to_notebook.py`. - -### Pending — stubbed (need full recipes) - -Drafted as scaffolds during the 2026-05-22 coverage audit against -`autocti_workspace/scripts/`. Each has frontmatter + Orient/Ask/Branch/Combine -structure + `Further reading`; the `Branch` recipes are TODO markers. Fill in -one at a time, paired with their companion wiki/core stub. - -**Data types and regimes** - -- [`ac_point_source.md`](./ac_point_source.md) (stub) — quasar / multi-image - position fits, flux ratios, point-source deblending. -- [`ac_time_delay_cosmography.md`](./ac_time_delay_cosmography.md) (stub) — H0 - from time-delay strong lenses. -- [`ac_group_lensing.md`](./ac_group_lensing.md) (stub) — extra galaxies, - scaling-relation members. -- [`ac_cluster_csv_api.md`](./ac_cluster_csv_api.md) (stub) — cluster-scale - CSV-driven model composition. -- [`ac_multi_dataset.md`](./ac_multi_dataset.md) (stub) — joint imaging + - interferometer, multi-band, wavelength-dependent sources. -- [`ac_weak_lensing.md`](./ac_weak_lensing.md) (stub) — shear catalogue fits - (`WeakDataset` / `AnalysisWeak`). -- [`ac_datacube_modeling.md`](./ac_datacube_modeling.md) (stub) — interferometer - spectral cubes. - -**Dark-matter substructure** - -- [`ac_subhalo_detect.md`](./ac_subhalo_detect.md) (stub) — Bayesian-evidence - grid search for perturbing subhaloes. -- [`ac_sensitivity_mapping.md`](./ac_sensitivity_mapping.md) (stub) — quantitative - detectability calibration. - -**Advanced techniques** - -- [`ac_hierarchicac_inference.md`](./ac_hierarchicac_inference.md) (stub) — - population-level / graphical models, expectation propagation. -- [`ac_aggregator_bulk_analysis.md`](./ac_aggregator_bulk_analysis.md) (stub) — - bulk operations across many completed fits, optional result database. -- [`ac_adaptive_pixelization.md`](./ac_adaptive_pixelization.md) (stub) — adaptive - mesh + adaptive regularisation source reconstructions. -- [`ac_mge_decomposition.md`](./ac_mge_decomposition.md) (stub) — Multi-Gaussian - Expansion workflows for lens / source. -- [`ac_custom_analysis.md`](./ac_custom_analysis.md) (stub) — subclassing - `Analysis` to add custom likelihood terms. - -**Queue (catalogued, not yet stubbed):** `ac_multi_plane`, `ac_los_halos`, -`ac_over_sampling`, `ac_workflow_outputs`, `ac_data_prep_interactive`, -`ac_bayesian_model_comparison`. +- [`start-new-project.md`](./start-new-project.md) — bridge to a standalone + **science project** repo and its lifecycle (Create → Work → Collaborate → + Publish). +- [`contribute-upstream.md`](./contribute-upstream.md) — prepare a scoped change + and open a draft PR into `PyAutoLabs/autocti_assistant`. diff --git a/skills/_style.md b/skills/_style.md index da3cc44..6e364d7 100644 --- a/skills/_style.md +++ b/skills/_style.md @@ -67,7 +67,7 @@ This is the rule that distinguishes the workspace from a tutorial workspace. Every Python script the agent saves — whether to `scripts/` or `scripts/scratch/` — follows the PyAutoCTI **workspace** style, not ad-hoc banner comments. It is the same style used by every script in `autocti_workspace/scripts/` (canonical example: -`autocti_workspace:scripts/imaging/start_here.py`), and it exists for two reasons: it +`autocti_workspace:scripts/dataset_1d/modeling/start_here.py`), and it exists for two reasons: it keeps the science and inference narrative inline with the code, and it makes the script mechanically convertible to a Jupyter notebook — each top-level `"""..."""` block becomes a markdown cell and the code between blocks becomes a code cell. diff --git a/skills/ac_correct_cti.md b/skills/ac_correct_cti.md new file mode 100644 index 0000000..e55d008 --- /dev/null +++ b/skills/ac_correct_cti.md @@ -0,0 +1,74 @@ +--- +name: ac_correct_cti +description: Remove CTI from a dataset given a trap model — the inverse-clocking operation clocker.remove_cti, used to correct science data or as a diagnostic of a calibrated CTI model's quality. Writes a runnable script in scripts/. +--- + +# Correcting CTI + +Correction is arctic run in reverse: given a CTI model, `remove_cti` un-clocks a +dataset, pushing the trailed electrons back toward where they were captured. Two +uses: + +1. **Correct science data** with a CTI model calibrated elsewhere. +2. **Diagnose a calibration** — correct the *calibration* data with the fitted + model and check the trails vanish. Residual trailing means the model is + imperfect; over-correction means it's too strong. It's a direct, visual + quality check on a fit. + +Grounded in `autocti_workspace:scripts/dataset_1d/correction/start_here.py` +(the 2D equivalent is `imaging_ci/correction/start_here.py`). + +## Ask + +- *"Where does the CTI model come from — a fit, or known input values?"* — a fit + gives you `result.max_log_likelihood_instance.cti`; known values you build + directly (below). +- *"Correcting to check a calibration, or to clean science data?"* — same call, + different intent; the diagnostic use compares corrected-vs-original trails. + +## Build the model and correct + +The CTI model is built with concrete values (not `af.Model` — this is an +instance, not something being fitted). If it came from a fit, use +`result.max_log_likelihood_instance.cti` instead of constructing it: + +```python +import autocti as ac + +clocker = ac.Clocker1D(express=5) + +trap_0 = ac.TrapInstantCapture(density=0.13, release_timescale=1.25) +trap_1 = ac.TrapInstantCapture(density=0.25, release_timescale=4.4) +ccd = ac.CCDPhase(well_fill_power=0.58, well_notch_depth=0.0, full_well_depth=200000.0) +cti = ac.CTI1D(trap_list=[trap_0, trap_1], ccd=ccd) +``` + +`remove_cti` takes the data and the model and returns the corrected data: + +```python +data_corrected_list = [ + clocker.remove_cti(data=dataset.data, cti=cti) for dataset in dataset_list +] + +for data_corrected, norm in zip(data_corrected_list, norm_list): + aplt.plot_yx(y=data_corrected, output_path=..., output_filename="data_corrected", + output_format="png") +``` + +For 2D charge-injection data the call is identical with a `Clocker2D`, `CTI2D` +and an `ImagingCI` dataset. + +## Read it as a diagnostic + +The most useful thing to show a user is the *before and after*: plot the +original trailed data and the corrected data side by side (or the residual +between them). If EPER trails remain after correction, the trap model +under-fits; if the injected charge develops a dip, it over-corrects. This is +often more legible than staring at posterior contours. + +## Further reading + +- `autocti_workspace:scripts/dataset_1d/correction/start_here.py` +- `autocti_workspace:scripts/imaging_ci/correction/start_here.py` +- [`ac_fit_cti_model`](./ac_fit_cti_model.md) — where the calibrated model + comes from. diff --git a/skills/ac_fit_cti_model.md b/skills/ac_fit_cti_model.md new file mode 100644 index 0000000..1e44c57 --- /dev/null +++ b/skills/ac_fit_cti_model.md @@ -0,0 +1,136 @@ +--- +name: ac_fit_cti_model +description: Compose a CTI model (trap species + CCD) and fit it to one or more calibration datasets with a non-linear search, recovering the trap densities and release timescales. Covers the model cookbook, the AnalysisDataset1D wiring, and — crucially — the factor-graph structure that fits several charge lines jointly. Writes a runnable script in scripts/. +--- + +# Fitting a CTI model + +Calibration is the inverse of simulation: given data with CTI trails, recover +the trap model that produced them. You compose a model with the trap +densities/timescales and CCD parameters *free*, hand it to a non-linear search +with an `Analysis`, and read the recovered values off the result. + +A CTI calibration dataset is almost always **several charge lines at different +normalisations**, fitted **jointly** — one shared trap model, one likelihood per +line, summed. That joint fit is a **factor graph**, and it is the part most +worth getting right. The canonical example is +`autocti_workspace:scripts/dataset_1d/modeling/start_here.py`. + +## Ask + +- *"How many trap species in the model?"* — must match (or bracket) what you + believe is in the data. Two is standard. +- *"Fitting one dataset or several charge lines jointly?"* — several is the + norm, and means a factor graph (below). One is the simplifying case. +- *"Which parameters free vs fixed?"* — e.g. fix `well_notch_depth=0.0` and + `full_well_depth`, leave densities and release timescales free. +- *"Real or simulated data?"* — simulated has a saved `cti.json` truth to check + the recovery against; real does not. + +## Compose the model + +The model uses `af.Model` and `af.Collection` (the model cookbook). Each trap +and the CCD is a `Model`; they combine into a `CTI1D`: + +```python +import autofit as af +import autocti as ac + +trap_0 = af.Model(ac.TrapInstantCapture) +trap_1 = af.Model(ac.TrapInstantCapture) +trap_list = [trap_0, trap_1] + +ccd = af.Model(ac.CCDPhase) +ccd.well_notch_depth = 0.0 # fix a parameter by assigning it +ccd.full_well_depth = 200000.0 + +model = af.Collection(cti=af.Model(ac.CTI1D, trap_list=trap_list, ccd=ccd)) +``` + +Ordered trap species can otherwise swap identities during the search; assert an +ordering to break the degeneracy: + +```python +trap_0.add_assertion(trap_0.release_timescale < trap_1.release_timescale) +``` + +## The clocker and the analysis + +The `Clocker1D` used for the fit should match the simulation (`express=5`). +Each dataset gets its own `AnalysisDataset1D`, which pairs the data with the +clocker and defines the likelihood: + +```python +clocker = ac.Clocker1D(express=5) + +analysis_list = [ + ac.AnalysisDataset1D(dataset=dataset, clocker=clocker) + for dataset in dataset_list +] +``` + +## The factor graph — fitting several charge lines jointly + +This is the structural heart of a CTI fit and the API changed in the modern +stack: **analysis objects are no longer combined with the `+` operator** — the +addition that used to sum per-dataset log-likelihoods is gone. Instead each +analysis is wrapped in an `AnalysisFactor` that pairs it with the (shared) +model, and the factors combine into a `FactorGraphModel`: + +```python +analysis_factor_list = [ + af.AnalysisFactor(prior_model=model, analysis=analysis) + for analysis in analysis_list +] + +factor_graph = af.FactorGraphModel(*analysis_factor_list) +``` + +The factor graph's log likelihood is the sum of the per-line likelihoods, and +its structure keeps each line's outputs and visualisation separate on disk. + +## Run the search and read the result + +Fit the factor graph's `global_prior_model` with its analysis. The result comes +back as a **list** — one entry per factor (per charge line): + +```python +search = af.Nautilus( + path_prefix=path.join("dataset_1d", dataset_name), name="species[x2]", n_live=100 +) + +result_list = search.fit(model=factor_graph.global_prior_model, analysis=factor_graph) + +print(result_list[0].max_log_likelihood_instance.cti.trap_list[0].density) +aplt.subplot_fit_dataset_1d(fit=result_list[0].max_log_likelihood_fit) +``` + +For a single dataset, wire one `AnalysisDataset1D` into a one-element factor +graph the same way — the factor-graph path is uniform, so scripts don't +special-case the single-line case. + +## Check the recovery + +If the data was simulated, load the input `cti.json` truth and compare the +recovered densities/timescales against it — this is the calibration working. +See [`ac_simulate_dataset_1d`](./ac_simulate_dataset_1d.md) for the saved truth +and [`ac_load_results`](./ac_load_results.md) for pulling values across many +fits via the aggregator. + +## Test mode + +Under `PYAUTO_TEST_MODE=2` the non-linear search is bypassed (it returns quickly +at the prior medians) so a script runs end-to-end for smoke testing. A known +artifact: identical-prior *ordered* trap species tie at the prior median under +the bypass and trip their own ordering assertion — that is a test-mode artifact, +not a real-fit problem. `PYAUTOFIT_TEST_MODE` does **not** exist; the knob is +`PYAUTO_TEST_MODE`. + +## Further reading + +- `autocti_workspace:scripts/dataset_1d/modeling/` — `start_here.py`, plus + `customize/priors.py` and `features/species_x3.py`. +- `autocti_workspace:scripts/imaging_ci/modeling/` — the 2D charge-injection + equivalent (parallel + serial CTI, `Clocker2D`). +- [`ac_mask_and_extract`](./ac_mask_and_extract.md) — masking the FPR/EPER + before the fit. diff --git a/skills/ac_load_results.md b/skills/ac_load_results.md new file mode 100644 index 0000000..90e6301 --- /dev/null +++ b/skills/ac_load_results.md @@ -0,0 +1,95 @@ +--- +name: ac_load_results +description: Load and inspect CTI fit results — from a single search's result object, or across many fits with the aggregator (Aggregator.from_directory, ac.agg.CTIAgg / Dataset1DAgg / FitDataset1DAgg and their max_log_likelihood / weighted generators). Writes a runnable script in scripts/. +--- + +# Loading CTI results + +Two ways to get at a fit's output: + +1. **The `result` object** returned by `search.fit(...)` — immediate, in-memory, + the fastest path when you still have it. For a factor-graph fit it's a + `result_list` (one per charge line). +2. **The aggregator** — reloads results from disk, so you can inspect a run from + a fresh session or sweep over *many* fits at once. This is how calibration + campaigns (hundreds of datasets) are analysed. + +Grounded in `autocti_workspace:scripts/dataset_1d/results/start_here.py` and +`.../results/examples/` (plus the `advanced/database/` examples). + +## From the `result` object + +```python +result = result_list[0] # factor-graph fits return a list + +instance = result.max_log_likelihood_instance +print(instance.cti.trap_list[0].density) +print(instance.cti.trap_list[0].release_timescale) + +fit = result.max_log_likelihood_fit # a Fit object for plotting +samples = result.samples # the full posterior +``` + +The `samples_summary` gives medians and confidence intervals *without* +recomputing over the full sample list — prefer it when the value you want is +there, and fall back to `samples` only when it isn't. + +## With the aggregator + +Build an aggregator from the search's output directory (or a database), then use +`values(name)` to get a memory-efficient **generator** of any stored object. +Generators are single-use — remake them at the point of use rather than storing +them: + +```python +from autofit.aggregator.aggregator import Aggregator + +agg = Aggregator.from_directory(directory=path.join("output", "results_folder")) + +for samples in agg.values("samples"): + print(samples.parameter_lists[0]) +``` + +## CTI-aware aggregation + +Raw `values("samples")` gives you PyAutoFit objects. For the *CTI* objects +themselves, `autocti.agg` wraps the aggregator and rebuilds `CTI`, `Dataset` and +`Fit` instances per result: + +```python +import autocti as ac + +cti_agg = ac.agg.CTIAgg(aggregator=agg) + +# the max-likelihood CTI model of every fit: +for cti in cti_agg.max_log_likelihood_gen_from(): + print(cti.trap_list[0].density) + +# posterior-weighted draws, for propagating uncertainty: +for cti_list in cti_agg.all_above_weight_gen_from(minimum_weight=1e-4): + ... +for weights in cti_agg.weights_above_gen_from(minimum_weight=1e-4): + ... +for cti_list in cti_agg.randomly_drawn_via_pdf_gen_from(total_samples=2): + ... +``` + +The sibling aggregators follow the same shape: `ac.agg.Dataset1DAgg` and +`ac.agg.FitDataset1DAgg` for 1D, `ac.agg.ImagingCIAgg` and +`ac.agg.FitImagingCIAgg` for 2D charge-injection. Each stores its objects in the +consolidated per-fit format and yields generators, not lists — the memory +discipline matters once you're aggregating hundreds of calibration fits. + +## Checking recovery against the truth + +For simulated data, load the input `cti.json` (saved by +[`ac_simulate_dataset_1d`](./ac_simulate_dataset_1d.md)) and compare the +recovered trap densities/timescales against it — a table of input-vs-recovered +per species is the clearest evidence the calibration works. + +## Further reading + +- `autocti_workspace:scripts/dataset_1d/results/` — `start_here.py` + + `examples/{cti,fits,samples}.py`. +- `autocti_workspace:scripts/dataset_1d/advanced/database/` — the full + aggregator/database cookbook and every `*_gen_from` generator. diff --git a/skills/ac_mask_and_extract.md b/skills/ac_mask_and_extract.md new file mode 100644 index 0000000..292258e --- /dev/null +++ b/skills/ac_mask_and_extract.md @@ -0,0 +1,91 @@ +--- +name: ac_mask_and_extract +description: Mask a CTI dataset to isolate the FPR or EPER before fitting, and extract FPR/EPER regions as arrays for inspection or diagnostics. Covers Mask1D/Mask2D.masked_fpr_and_eper_from and the layout.extract.fpr / layout.extract.eper API. Writes a runnable script in scripts/. +--- + +# Masking and extracting FPR / EPER regions + +A CTI dataset has structure a fit cares about: the injected charge (**FPR**, +First Pixel Response) and the trail arctic leaves behind it (**EPER**, Extended +Pixel Edge Response). Two related operations exploit that structure — *masking* +(hide the FPR so the fit sees only the trails, or vice versa) and *extracting* +(pull a region out as a plain array to plot or diagnose). + +Grounded in `autocti_workspace:scripts/dataset_1d/modeling/start_here.py` +(masking) and `autocti_workspace:scripts/dataset_1d/extract.py` (extraction). + +## Ask + +- *"Fit the trails or the injected charge?"* — masking the FPR leaves the EPER + trails (the usual calibration target); masking the EPER leaves the FPR. +- *"1D dataset or 2D charge-injection image?"* — `Mask1D` / `Layout1D` vs + `Mask2D` / `Layout2DCI`; the extract region strings differ (below). + +## Masking before a fit + +Start from an all-false mask (nothing masked) and add the FPR/EPER masking. The +`fpr_pixels` setting says how many pixels of the FPR to mask: + +```python +import autocti as ac + +mask = ac.Mask1D.all_false( + shape_slim=dataset_list[0].shape_slim, + pixel_scales=dataset_list[0].pixel_scales, +) + +mask = ac.Mask1D.masked_fpr_and_eper_from( + mask=mask, + layout=dataset_list[0].layout, + settings=ac.SettingsMask1D(fpr_pixels=(0, 10)), + pixel_scales=dataset_list[0].pixel_scales, +) + +dataset_list = [dataset.apply_mask(mask=mask) for dataset in dataset_list] +``` + +Plotting the masked dataset now shows only the EPER trails — the FPR is hidden. +This masked dataset is what you hand to [`ac_fit_cti_model`](./ac_fit_cti_model.md). + +For 2D charge-injection imaging the shape is the same but the objects are 2D and +the settings name the *direction* — `ac.Mask2D`, +`ac.SettingsMask2D(parallel_fpr_pixels=(0, 200))`, over a `Layout2DCI`. + +## Extracting a region as an array + +A `Layout` exposes `extract.fpr` and `extract.eper`, each with three shapes of +getter. `SettingsExtract(pixels=(0, 10))` selects the pixel window relative to +the region (negative starts reach into the pixels *before* it — useful for the +overscan side of an EPER): + +```python +layout_1d = ac.Layout1D(shape_1d=(200,), region_list=[(10, 20)], + prescan=ac.Region1D((0, 10)), overscan=ac.Region1D((190, 200))) +data_1d = ac.Array1D.from_fits(file_path=..., pixel_scales=0.1) + +# just the pixel ranges each region occupies: +eper_regions = layout_1d.extract.eper.region_list_from( + settings=ac.SettingsExtract(pixels=(0, 10))) + +# the data in each region, one array per charge region: +eper_arrays = layout_1d.extract.eper.array_1d_list_from( + array=data_1d, settings=ac.SettingsExtract(pixels=(0, 10))) +aplt.plot_yx(y=eper_arrays[0]) + +# all regions stacked into one averaged array (boosts the trail S/N): +eper_stacked = layout_1d.extract.eper.stacked_array_1d_from( + array=data_1d, settings=ac.SettingsExtract(pixels=(0, 10))) +aplt.plot_yx(y=eper_stacked) +``` + +`layout.extract.fpr.*` mirrors this exactly for the injected-charge region. + +The 2D region vocabulary is richer — instead of `fpr`/`eper`, a `Layout2DCI` +exposes direction-qualified regions (`parallel_fpr`, `parallel_eper`, and the +serial equivalents) reachable through the same `extract` interface. + +## Further reading + +- `autocti_workspace:scripts/dataset_1d/extract.py` — every extract getter. +- `autocti_workspace:scripts/imaging_ci/extract.py` — the 2D region strings. +- [`ac_plot`](./ac_plot.md) — plotting extracted arrays with `aplt.plot_yx`. diff --git a/skills/ac_plot.md b/skills/ac_plot.md new file mode 100644 index 0000000..f7e6b6d --- /dev/null +++ b/skills/ac_plot.md @@ -0,0 +1,79 @@ +--- +name: ac_plot +description: Plot CTI datasets, fits and arrays using the autocti.plot function API — aplt.subplot_dataset_1d, aplt.subplot_imaging_ci, aplt.subplot_fit_dataset_1d, aplt.plot_yx, aplt.plot_array, aplt.plot_cti_1d and the *_list combined subplots. There is no MatPlot/Output/Visuals object; output paths are passed straight to the call. +--- + +# Plotting CTI data and fits + +PyAutoCTI's plotting is a **matplotlib function API**: you call `aplt.` +directly and pass the output path to the call. There is deliberately no +`Output` / `MatPlot2D` / `Visuals` object to construct and thread through (that +is the lensing stack's pattern, not this one). Follow the workspace scripts, +which use functions throughout. + +Grounded in `autocti_workspace:scripts/plot/start_here.py` and the plotting +calls across the simulator/modeling/results scripts. + +## The functions you'll actually use + +**Whole datasets and fits** — the high-level subplots: + +```python +import autocti.plot as aplt + +aplt.subplot_dataset_1d(dataset=dataset) # a 1D dataset +aplt.subplot_imaging_ci(dataset=imaging_ci) # a 2D charge-injection image +aplt.subplot_fit_dataset_1d(fit=result.max_log_likelihood_fit) # a 1D fit +aplt.subplot_fit_ci(fit=fit) # a 2D charge-injection fit +``` + +**Combined subplots over a list** — one figure showing every charge line, via +the `*_list` variants: + +```python +aplt.subplot_dataset_1d_list(dataset_list=dataset_list) +aplt.subplot_fit_dataset_1d_list(fit_list=fit_list) +aplt.subplot_imaging_ci_list(dataset_list=imaging_ci_list) +``` + +**Single figures and raw arrays** — for one panel or an extracted region: + +```python +aplt.figure_dataset_1d_data(dataset=dataset) # one named figure +aplt.plot_yx(y=array_1d) # a 1D array (e.g. an extracted EPER) +aplt.plot_array(array=array_2d) # a 2D array +aplt.plot_cti_1d(cti=cti) # the CTI model itself +``` + +## Output paths — passed to the call, then announced + +There is no separate output object: pass `output_path`, `output_filename` and +`output_format` straight to the plotting function. + +```python +aplt.subplot_fit_dataset_1d( + fit=fit, + output_path="scripts/scratch/fit/", + output_filename="fit_species_x2", + output_format="png", +) +``` + +After running, **print the absolute path and quote it back to the user**, then +offer to open it (`xdg-open` on Linux, `open` on macOS, `explorer.exe` / +`wslview` from WSL). Don't just say "plot saved" — one offer per plot. The full +convention is in `AGENTS.md` "Plot path announcement". + +## Note on Plotter objects + +`aplt.PlotterDataset1D` and `aplt.PlotterImagingCI` do exist in the namespace, +but the workspace scripts do **not** use them — the function API above is the +taught, validated surface. Prefer the functions; reach for a Plotter only if a +user has an existing script built on one. + +## Further reading + +- `autocti_workspace:scripts/plot/start_here.py` — the plotting catalogue + (`ccd/`, `diagnostics/`, `plotters/` subfolders for specific figures). +- Run `python -c "import autocti.plot as aplt; print(dir(aplt))"` for the full + current function list before writing an unfamiliar plot call. diff --git a/skills/ac_setup_environment.md b/skills/ac_setup_environment.md new file mode 100644 index 0000000..8ea31d0 --- /dev/null +++ b/skills/ac_setup_environment.md @@ -0,0 +1,127 @@ +--- +name: ac_setup_environment +description: Install and verify the PyAutoCTI stack, including arcticpy — the C++ arctic clocking code that `import autocti` requires but pip will not install for you. The #1 setup failure. Covers the libgsl-dev + numpy/cython + arcticpy recipe, the numpy-downgrade trap, a no-root header workaround, and the environment/version drift check. +--- + +# Setting up the PyAutoCTI environment + +Before any simulate/fit/correct work runs, the interpreter needs a working CTI +stack: `autoconf`, `autoarray`, `autofit`, `autocti`, and — the part that trips +almost everyone — **arcticpy**, the C++ *arctic* clocking code that `Clocker1D` +and `Clocker2D` wrap. `import autocti` fails without it, yet `pip install +autocti` does **not** pull it. This skill is how you get past that cleanly. + +If `python autoassistant/audit_skill_apis.py --check-version` already exits 0, +the stack is healthy — skip to the bottom. Run this skill when that check exits +2/3, when `import autocti` raises `ModuleNotFoundError: No module named +'arcticpy'`, or when a user is installing for the first time. + +## Ask + +- *"Do you want a fresh virtual environment, or install into an existing one?"* — + a project-local `.venv` is the default (`activate.sh` looks for it); a shared + or HPC checkout points `PYAUTO_HPC_BASE` at a `PyAuto/` venv instead. +- *"Do you have root / sudo on this machine?"* — it decides how the GSL headers + arctic needs are obtained (system package vs. local extraction). +- *"Python version?"* — 3.12 or 3.13 is first-class; 3.11 works but warns. + +## Why arcticpy is special + +`arcticpy` (pinned to **2.6**) is a hard import of autocti but deliberately not +a pip dependency, for two reasons that both bite if you let pip resolve it +normally: + +- Its PyPI sdist is **source-only C++** — it needs the GSL headers + (`libgsl-dev`) and a C++ toolchain to build. There is no wheel. +- Its own requirements **downgrade numpy below 2.0**, quietly breaking a modern + autoarray/autofit stack that expects numpy ≥ 2. + +So arcticpy is installed *after* numpy is already in place, with dependency +resolution and build isolation both turned off. + +## The recipe + +The order matters. numpy and cython must exist before arctic builds, and +arctic must be installed `--no-deps` so it cannot drag numpy back down. + +```bash +# 1. GSL headers + a C++ toolchain (system-wide; needs sudo). +sudo apt-get update && sudo apt-get install -y libgsl-dev + +# 2. numpy + cython first, so arctic has what it needs to build against. +pip install numpy cython + +# 3. arctic itself — no build isolation (uses the numpy just installed), +# no deps (so it cannot pull an old numpy). +pip install arcticpy==2.6 --no-build-isolation --no-deps + +# 4. the PyAutoCTI stack. +pip install autocti +``` + +Then confirm: + +```bash +python -c "import arcticpy, autocti; print('arctic', arcticpy.__version__, '| autocti', autocti.__version__)" +``` + +The CI-hardened form of exactly this recipe lives in +`autocti_workspace_test:.github/scripts/smoke_install.sh` — cite it when a user +wants the canonical, tested sequence. + +## No root? Extract the GSL headers locally + +If `libgsl-dev` can't be installed system-wide, fetch and unpack it into your +home directory, then point the compiler at it for step 3: + +```bash +apt-get download libgsl-dev # downloads the .deb, no install +dpkg -x libgsl-dev*.deb "$HOME/gsl" # extracts headers/libs under ~/gsl +export CPPFLAGS="-I$HOME/gsl/usr/include" +export LDFLAGS="-L$HOME/gsl/usr/lib/x86_64-linux-gnu" +pip install arcticpy==2.6 --no-build-isolation --no-deps +``` + +The full note is in `PyAutoCTI:AGENTS.md` under "arcticpy (read before +installing)". + +## Developer install (editable source checkouts) + +Running against the `main` source checkouts instead of a PyPI release (what +`activate.sh`'s developer block and the `wiki-currency` CI do): install the +same arctic prefix (steps 1–3), then the local repos in dependency order, and +pin autoconf last so a re-resolution can't replace it with a stale wheel: + +```bash +pip install ./PyAutoConf ./PyAutoFit ./PyAutoArray ./PyAutoCTI +pip install "./PyAutoArray[optional]" +pip install --force-reinstall --no-deps ./PyAutoConf +``` + +Note the CTI stack is **autoconf/autoarray/autofit/autocti** — it does *not* +include autogalaxy (that is the lensing stack). Don't install or expect it. + +## Verify + +Once installed, the environment + API drift check is the single source of truth: + +```bash +python autoassistant/audit_skill_apis.py --check-version +``` + +- **exit 0** — the documented API matches the installed stack; you're ready. +- **exit 1** — genuine drift (a version moved); recommend the pinned version or + an audit (see [`ac_audit_skill_apis`](./ac_audit_skill_apis.md)). +- **exit 2 / 3** — the stack is absent or `import autocti` failed; this is + almost always the missing-arcticpy case above. Re-run the recipe; do **not** + install into an unrelated system Python. + +The sandbox cache variables (`NUMBA_CACHE_DIR`, `MPLCONFIGDIR`) and test-mode +(`PYAUTO_TEST_MODE`) conventions are in +[`wiki/core/operations/sandbox.md`](../wiki/core/operations/sandbox.md). + +## Further reading + +- `PyAutoCTI:AGENTS.md` — the arcticpy note and the dependency direction. +- `autocti_workspace_test:.github/scripts/smoke_install.sh` — the tested recipe. +- [`wiki/core/operations/installation.md`](../wiki/core/operations/installation.md) diff --git a/skills/ac_simulate_dataset_1d.md b/skills/ac_simulate_dataset_1d.md new file mode 100644 index 0000000..5491568 --- /dev/null +++ b/skills/ac_simulate_dataset_1d.md @@ -0,0 +1,117 @@ +--- +name: ac_simulate_dataset_1d +description: Simulate a 1D CTI calibration dataset — charge lines with a known trap model clocked through arctic, so a later fit can be checked against the input truth. Writes a runnable script in scripts/. The 1D dataset is the simplest CTI calibration geometry; for 2D charge-injection imaging see ac_simulate_imaging_ci. +--- + +# Simulating a 1D CTI dataset + +A 1D CTI dataset is the simplest calibration geometry: a line of pixels with an +injected charge signal (the **FPR**, First Pixel Response) that arctic clocks +through a trap model, leaving a trail of captured-and-released electrons behind +it (the **EPER**, Extended Pixel Edge Response). Simulating one with a *known* +trap model is how you generate ground truth — the input densities and release +timescales a later fit must recover. + +The canonical example is +`autocti_workspace:scripts/dataset_1d/simulators/start_here.py`. This skill +writes the equivalent for the user's chosen trap model and geometry. + +## Ask + +- *"How many trap species, and what densities / release timescales?"* — the + input truth. Two species is the standard starting point. +- *"What charge normalisations?"* — a calibration dataset is several charge + lines at different injection levels; the list length is how many 1D datasets + are simulated. +- *"Image length and region layout?"* — where the prescan, overscan and FPR + sit. The defaults below (200 px, 10-px prescan/overscan, FPR at 10–20) match + the fiducial workspace dataset. + +## The script + +Standard imports and geometry. `Region1D` tuples are `(x0, x1)` pixel ranges. + +```python +import autocti as ac +import autocti.plot as aplt + +shape_native = (200,) +prescan = ac.Region1D((0, 10)) +overscan = ac.Region1D((190, 200)) +region_list = [(10, 20)] # the FPR — charge injected here +norm_list = [100, 5000, 25000, 200000] # one 1D dataset per normalisation +``` + +The `Layout1D` ties the geometry together — it knows where the FPR, EPERs and +overscans are, and the simulator uses it to place the initial charge: + +```python +layout_list = [ + ac.Layout1D( + shape_1d=shape_native, + region_list=region_list, + prescan=prescan, + overscan=overscan, + ) + for norm in norm_list +] +``` + +The CTI model — this is the ground truth. `Clocker1D` wraps arctic; +`express` trades speed for accuracy (5 is a good balance). Traps are +`TrapInstantCapture(density, release_timescale)`; the `CCDPhase` sets the +volume-filling behaviour: + +```python +clocker = ac.Clocker1D(express=5) + +trap_0 = ac.TrapInstantCapture(density=0.13, release_timescale=1.25) +trap_1 = ac.TrapInstantCapture(density=0.25, release_timescale=4.4) +trap_list = [trap_0, trap_1] + +ccd = ac.CCDPhase(well_fill_power=0.58, well_notch_depth=0.0, full_well_depth=200000.0) + +cti = ac.CTI1D(trap_list=trap_list, ccd=ccd) +``` + +> **Clocker kwargs are geometry-specific.** 1D uses `Clocker1D(express=, roe=)`. +> The 2D charge-injection clocker is `Clocker2D(parallel_express=, +> parallel_roe=, ...)` — different keyword names. Don't carry 1D kwargs into a +> 2D script (see [`ac_simulate_imaging_ci`](./ac_simulate_imaging_ci.md)). + +Simulate one dataset per normalisation and plot: + +```python +simulator_list = [ + ac.SimulatorDataset1D(read_noise=0.01, pixel_scales=0.1, norm=norm) + for norm in norm_list +] +dataset_list = [ + simulator.via_layout_from(clocker=clocker, layout=layout, cti=cti) + for simulator, layout in zip(simulator_list, layout_list) +] + +aplt.subplot_dataset_1d(dataset=dataset_list[0]) +``` + +Save the input `Clocker1D` and `CTI1D` alongside the data as JSON, so the fit +can be checked against the truth later (`ac.from_json()` reloads them): + +```python +ac.output_to_json(obj=cti, file_path=path.join(dataset_path, "cti.json")) +ac.output_to_json(obj=clocker, file_path=path.join(dataset_path, "clocker.json")) +``` + +## Verify the truth is recoverable + +The point of a simulated dataset is that a fit recovers its input trap model. +After simulating, the natural next step is +[`ac_fit_cti_model`](./ac_fit_cti_model.md) — compose a model with the trap +densities/timescales free and confirm the search returns the values above. + +## Further reading + +- `autocti_workspace:scripts/dataset_1d/simulators/` — `start_here.py` plus + `examples/` (multi-species, continuum, temporal variants). +- [`ac_plot`](./ac_plot.md) — the `aplt.*` function plotting API. +- `wiki/core/` — trap physics and the FPR/EPER anatomy (once populated). diff --git a/skills/ac_simulate_imaging_ci.md b/skills/ac_simulate_imaging_ci.md new file mode 100644 index 0000000..32406ae --- /dev/null +++ b/skills/ac_simulate_imaging_ci.md @@ -0,0 +1,101 @@ +--- +name: ac_simulate_imaging_ci +description: Simulate a 2D charge-injection (CI) imaging dataset — injected charge regions clocked through a parallel (and/or serial) trap model with Clocker2D, the Euclid VIS / HST ACS calibration geometry. Writes a runnable script in scripts/. For the simpler 1D geometry see ac_simulate_dataset_1d. +--- + +# Simulating a charge-injection (CI) image + +Charge-injection imaging is the 2D calibration geometry used by real +instruments (Euclid VIS, HST ACS): known charge is injected into rows of the +CCD, then clocked out through the trap field, and the trails it leaves calibrate +the CTI model. It is the 2D analogue of the 1D dataset — same trap physics, but +now with a **parallel** clocking direction (and optionally **serial**), 2D +regions, and `Clocker2D`. + +Grounded in `autocti_workspace:scripts/imaging_ci/simulators/start_here.py`. + +## Ask + +- *"Parallel CTI only, or parallel + serial?"* — parallel-only is the standard + starting point; serial adds a second clocking direction and a second trap set. +- *"Injection normalisations?"* — as in 1D, one image per charge level. +- *"CCD geometry — image shape, prescan/overscan regions?"* — the defaults below + match the fiducial Euclid-like VIS layout. + +## Geometry — 2D regions and layout + +`Region2D` tuples are `(y0, y1, x0, x1)`. A `Layout2DCI` names the injection +regions and the parallel/serial overscans and prescans: + +```python +import autocti as ac + +shape_native = (2000, 100) +parallel_overscan = ac.Region2D((1980, 2000, 5, 95)) +serial_prescan = ac.Region2D((0, 2000, 0, 5)) +serial_overscan = ac.Region2D((0, 1980, 95, 100)) +region_list = [(0, 200, 5, 95), (400, 600, 5, 95)] # injected-charge rows +norm_list = [100, 5000, 25000, 200000] + +layout_list = [ + ac.Layout2DCI( + shape_2d=shape_native, + region_list=region_list, + parallel_overscan=parallel_overscan, + serial_prescan=serial_prescan, + serial_overscan=serial_overscan, + ) + for norm in norm_list +] +``` + +## The 2D clocker and CTI model + +This is where the 1D/2D difference bites. `Clocker2D` takes +**direction-prefixed** kwargs — `parallel_express`, `parallel_roe`, +`parallel_fast_mode` — not the bare `express`/`roe` of `Clocker1D`. The +`ROEChargeInjection` read-out electronics model is the charge-injection default: + +```python +clocker = ac.Clocker2D( + parallel_express=5, + parallel_roe=ac.ROEChargeInjection(), + parallel_fast_mode=True, # only clocks unique columns for uniform charge — a big speedup +) + +parallel_trap_0 = ac.TrapInstantCapture(density=0.13, release_timescale=1.25) +parallel_trap_1 = ac.TrapInstantCapture(density=0.25, release_timescale=4.4) +parallel_ccd = ac.CCDPhase(well_fill_power=0.58, well_notch_depth=0.0, full_well_depth=200000.0) + +cti = ac.CTI2D(parallel_trap_list=[parallel_trap_0, parallel_trap_1], parallel_ccd=parallel_ccd) +``` + +Adding serial CTI means a `serial_trap_list` / `serial_ccd` on the `CTI2D` and +`serial_express` / `serial_roe` on the `Clocker2D` — see the `serial_cti.py` +example. + +## Simulate and save the truth + +The simulator call is identical in shape to 1D — `via_layout_from`: + +```python +simulator_list = [ + ac.SimulatorImagingCI(read_noise=4.0, pixel_scales=0.1, norm=norm) + for norm in norm_list +] +dataset_list = [ + simulator.via_layout_from(clocker=clocker, layout=layout, cti=cti) + for simulator, layout in zip(simulator_list, layout_list) +] + +aplt.subplot_imaging_ci(dataset=dataset_list[0]) +ac.output_to_json(obj=cti, file_path=path.join(dataset_path, "cti.json")) +``` + +## Further reading + +- `autocti_workspace:scripts/imaging_ci/simulators/` — `start_here.py` plus + `examples/` (serial CTI, cosmic rays, non-uniform injection, Poisson traps). +- [`ac_fit_cti_model`](./ac_fit_cti_model.md) — the fit uses the same factor + graph; the 2D masking uses `Mask2D` / `SettingsMask2D(parallel_fpr_pixels=...)`. +- Euclid VIS / HST ACS heritage — `wiki/core/` (once populated). diff --git a/skills/ac_workspace_navigation.md b/skills/ac_workspace_navigation.md new file mode 100644 index 0000000..9e4db0a --- /dev/null +++ b/skills/ac_workspace_navigation.md @@ -0,0 +1,62 @@ +--- +name: ac_workspace_navigation +description: Navigate the autocti_workspace script catalogue — the two dataset geometries (dataset_1d, imaging_ci), the per-geometry pipeline (simulators → modeling → correction → results), the overview tour, and where to find the validated example for any CTI task. Read this to locate the right start_here.py to adapt. +--- + +# Navigating autocti_workspace + +Every CTI capability the assistant teaches is grounded in a **validated +workspace script**, not API memory. This skill is the map: given a task, it +tells you which script is the canonical example to read and adapt. When you're +about to write CTI code, check here (or the workspace's `llms.txt` / index) +first — an existing `start_here.py` almost always covers it. + +`autocti_workspace` (clone into gitignored `sources/` if not installed) is +organised by **dataset geometry**, each with the same pipeline of stages. + +## The two geometries + +- **`scripts/dataset_1d/`** — 1D CTI data: a line of pixels with an injected + charge signal. The simplest calibration geometry; start here to learn the API. +- **`scripts/imaging_ci/`** — 2D charge-injection imaging: the real-instrument + geometry (Euclid VIS, HST ACS), with parallel and serial clocking directions. + +## The per-geometry pipeline + +Both geometries share the same stage folders, each with a `start_here.py`: + +| Stage | Folder | Skill | +|-------|--------|-------| +| Simulate a dataset with a known trap model | `simulators/` | [`ac_simulate_dataset_1d`](./ac_simulate_dataset_1d.md) · [`ac_simulate_imaging_ci`](./ac_simulate_imaging_ci.md) | +| Compose + fit a CTI model | `modeling/` | [`ac_fit_cti_model`](./ac_fit_cti_model.md) | +| Correct CTI from data | `correction/` | [`ac_correct_cti`](./ac_correct_cti.md) | +| Inspect results / aggregate | `results/` | [`ac_load_results`](./ac_load_results.md) | +| Extract FPR/EPER regions | `extract.py` | [`ac_mask_and_extract`](./ac_mask_and_extract.md) | + +Within `modeling/` and `simulators/`, an `examples/` (or `features/`) subfolder +holds variants: multi-species models, non-uniform injection, cosmic rays, +serial CTI, temporal variation. `advanced/` holds chaining pipelines and the +database/aggregator cookbook. + +## Where to start reading + +- **New to CTI?** `scripts/overview/overview_1_what_is_cti.py` through + `overview_6_cti_calibration.py` are a narrated tour — what CTI is, parallel vs + serial, charge-injection data, fitting, and calibration end-to-end. +- **A specific task?** Go straight to that stage's `start_here.py` in the + matching geometry, then its `examples/` for the closest variant. +- **Plotting?** `scripts/plot/start_here.py` and [`ac_plot`](./ac_plot.md). + +## Adapting a script + +The workspace scripts are written in the PyAutoCTI **workspace style** (a +title-underlined docstring, `"""__Section__"""` dividers with physics framing, +`:` citations). When you generate a script for a user, match that +style and cite the workspace example you adapted — see +[`_style.md`](./_style.md) "Generated script style". + +## Further reading + +- `autocti_workspace:README.rst` and `scripts/overview/` — the guided entry. +- `PyAutoCTI:AGENTS.md` — the library's own map of `Clocker`, `Fit*`, + `Analysis*`, `extract/`, and the instrument heritage. diff --git a/wiki/core/api_audit_baseline.json b/wiki/core/api_audit_baseline.json new file mode 100644 index 0000000..b3fb523 --- /dev/null +++ b/wiki/core/api_audit_baseline.json @@ -0,0 +1,32 @@ +{ + "_comment": "API baseline for autocti_assistant - see autoassistant/audit_skill_apis.py and skills/ac_audit_skill_apis.md. Regenerate with --write-baseline.", + "generated": "2026-07-17", + "versions": { + "autoconf": "2026.7.9.1", + "autoarray": "2026.7.9.1", + "autofit": "2026.7.9.1", + "autocti": "2024.11.13.2" + }, + "api_surface": { + "autoconf": { + "hash": "7bd5c3b0830f6a87158e90563cd2b65d3484482df75c125376506c160ce1074e", + "n_symbols": 32 + }, + "autoarray": { + "hash": "dfd1365821a3df8bd3c94b225378a2bef399ae0f18c11c6cd286e990fd416216", + "n_symbols": 107 + }, + "autofit": { + "hash": "141cb9b8a13b9074957c5570086b761e033aa7993609a55a3bcd21cc77ea3d64", + "n_symbols": 128 + }, + "autocti": { + "hash": "0ef66b5833a4bb1ae96653546dcac3a6bae4a2f16312012f485ba65f7480ef28", + "n_symbols": 87 + }, + "autocti.plot": { + "hash": "a589671b0d00039c63e4dcb6fb00465857bf090e5d86a2f142fe6ab3a188f0e4", + "n_symbols": 32 + } + } +}