fix(howtolens): should_simulate every dataset load; run under SMALL_DATASETS

config/build/env_vars.yaml

@@ -30,15 +30,10 @@ overrides:
     unset: [PYAUTO_SMALL_DATASETS]
   - pattern: "multi/start_here"
     unset: [PYAUTO_SMALL_DATASETS]
-  # Non-simulator scripts that load committed FITS data need full-size
-  # datasets to avoid shape mismatch with pre-existing 100x100 data.
-  # Simulators run first and regenerate data at the capped size, but
-  # scripts using the old `if not dataset_path.exists()` pattern skip
-  # re-simulation when data already exists.
-  - pattern: "howtolens/"
-    unset: [PYAUTO_SMALL_DATASETS]
-  - pattern: "guides/"
-    unset: [PYAUTO_SMALL_DATASETS]
+  # (Removed the dead `howtolens/` + `guides/` PYAUTO_SMALL_DATASETS-unset
+  # workarounds: they matched no files here, and the tutorials now use
+  # `al.util.dataset.should_simulate` which deletes + re-simulates stale
+  # data under the cap, so the chapters run correctly at 16x16.)
   # guides/results/start_here.py must produce real samples so the example
   # scripts that read from `output/results_folder` afterwards
   # (data_fitting, queries, models, samples_via_aggregator) find a

notebooks/chapter_1_introduction/tutorial_0_visualization.ipynb

@@ -116,6 +116,32 @@
    "outputs": [],
    "execution_count": null
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "__Dataset Auto-Simulation__\n",
+    "\n",
+    "If the dataset does not already exist on your system, it is created by running the corresponding\n",
+    "simulator script. This ensures every example script can be run without manually simulating data first."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "metadata": {},
+   "source": [
+    "if al.util.dataset.should_simulate(str(dataset_path)):\n",
+    "    import subprocess\n",
+    "    import sys\n",
+    "\n",
+    "    subprocess.run(\n",
+    "        [sys.executable, \"scripts/simulator/no_lens_light.py\"],\n",
+    "        check=True,\n",
+    "    )"
+   ],
+   "outputs": [],
+   "execution_count": null
+  },
   {
    "cell_type": "markdown",
    "metadata": {},

notebooks/chapter_1_introduction/tutorial_7_fitting.ipynb

@@ -129,20 +129,21 @@
    "source": [
     "__Dataset Auto-Simulation__\n",
     "\n",
-    "If the dataset does not already exist on your system, it will be created by running the corresponding\n",
-    "simulator script. This ensures that all example scripts can be run without manually simulating data first."
+    "The `howtolens` dataset is the one built up and saved in tutorial 6 (`tutorial_6_data.py`). If it does\n",
+    "not already exist on your system, it is created by running that script. This ensures every example\n",
+    "script can be run without manually simulating data first."
    ]
   },
   {
    "cell_type": "code",
    "metadata": {},
    "source": [
-    "if not dataset_path.exists():\n",
+    "if al.util.dataset.should_simulate(str(dataset_path)):\n",
     "    import subprocess\n",
     "    import sys\n",
     "\n",
     "    subprocess.run(\n",
-    "        [sys.executable, \"scripts/simulator/no_lens_light__mass_sis.py\"],\n",
+    "        [sys.executable, \"scripts/chapter_1_introduction/tutorial_6_data.py\"],\n",
     "        check=True,\n",
     "    )\n",
     "\n",
@@ -1025,6 +1026,15 @@
     "dataset_name = \"simple__no_lens_light__mass_sis\"\n",
     "dataset_path = Path(\"dataset\") / \"imaging\" / dataset_name\n",
     "\n",
+    "if al.util.dataset.should_simulate(str(dataset_path)):\n",
+    "    import subprocess\n",
+    "    import sys\n",
+    "\n",
+    "    subprocess.run(\n",
+    "        [sys.executable, \"scripts/simulator/no_lens_light__mass_sis.py\"],\n",
+    "        check=True,\n",
+    "    )\n",
+    "\n",
     "dataset = al.Imaging.from_fits(\n",
     "    data_path=dataset_path / \"data.fits\",\n",
     "    psf_path=dataset_path / \"psf.fits\",\n",

notebooks/chapter_2_lens_modeling/tutorial_1_non_linear_search.ipynb

@@ -281,7 +281,7 @@
    "cell_type": "code",
    "metadata": {},
    "source": [
-    "if not dataset_path.exists():\n",
+    "if al.util.dataset.should_simulate(str(dataset_path)):\n",
     "    import subprocess\n",
     "    import sys\n",
     "\n",

notebooks/chapter_2_lens_modeling/tutorial_2_practicalities.ipynb

@@ -191,7 +191,7 @@
    "cell_type": "code",
    "metadata": {},
    "source": [
-    "if not dataset_path.exists():\n",
+    "if al.util.dataset.should_simulate(str(dataset_path)):\n",
     "    import subprocess\n",
     "    import sys\n",
     "\n",

notebooks/chapter_2_lens_modeling/tutorial_3_realism_and_complexity.ipynb

@@ -129,7 +129,7 @@
    "cell_type": "code",
    "metadata": {},
    "source": [
-    "if not dataset_path.exists():\n",
+    "if al.util.dataset.should_simulate(str(dataset_path)):\n",
     "    import subprocess\n",
     "    import sys\n",
     "\n",

notebooks/chapter_2_lens_modeling/tutorial_4_dealing_with_failure.ipynb

@@ -125,7 +125,7 @@
    "cell_type": "code",
    "metadata": {},
    "source": [
-    "if not dataset_path.exists():\n",
+    "if al.util.dataset.should_simulate(str(dataset_path)):\n",
     "    import subprocess\n",
     "    import sys\n",
     "\n",

notebooks/chapter_2_lens_modeling/tutorial_5_linear_profiles.ipynb

@@ -139,7 +139,7 @@
    "cell_type": "code",
    "metadata": {},
    "source": [
-    "if not dataset_path.exists():\n",
+    "if al.util.dataset.should_simulate(str(dataset_path)):\n",
     "    import subprocess\n",
     "    import sys\n",
     "\n",

notebooks/chapter_2_lens_modeling/tutorial_6_masking_and_positions.ipynb

@@ -112,7 +112,7 @@
    "cell_type": "code",
    "metadata": {},
    "source": [
-    "if not dataset_path.exists():\n",
+    "if al.util.dataset.should_simulate(str(dataset_path)):\n",
     "    import subprocess\n",
     "    import sys\n",
     "\n",

notebooks/chapter_2_lens_modeling/tutorial_7_results.ipynb

@@ -107,7 +107,7 @@
    "cell_type": "code",
    "metadata": {},
    "source": [
-    "if not dataset_path.exists():\n",
+    "if al.util.dataset.should_simulate(str(dataset_path)):\n",
     "    import subprocess\n",
     "    import sys\n",
     "\n",

notebooks/chapter_3_search_chaining/tutorial_1_search_chaining.ipynb

@@ -146,7 +146,7 @@
    "cell_type": "code",
    "metadata": {},
    "source": [
-    "if not dataset_path.exists():\n",
+    "if al.util.dataset.should_simulate(str(dataset_path)):\n",
     "    import subprocess\n",
     "    import sys\n",
     "\n",

notebooks/chapter_3_search_chaining/tutorial_2_prior_passing.ipynb

@@ -121,7 +121,7 @@
    "cell_type": "code",
    "metadata": {},
    "source": [
-    "if not dataset_path.exists():\n",
+    "if al.util.dataset.should_simulate(str(dataset_path)):\n",
     "    import subprocess\n",
     "    import sys\n",
     "\n",

notebooks/chapter_3_search_chaining/tutorial_3_lens_and_source.ipynb

@@ -148,7 +148,7 @@
    "cell_type": "code",
    "metadata": {},
    "source": [
-    "if not dataset_path.exists():\n",
+    "if al.util.dataset.should_simulate(str(dataset_path)):\n",
     "    import subprocess\n",
     "    import sys\n",
     "\n",

notebooks/chapter_3_search_chaining/tutorial_4_x2_lens_galaxies.ipynb

@@ -120,7 +120,7 @@
    "cell_type": "code",
    "metadata": {},
    "source": [
-    "if not dataset_path.exists():\n",
+    "if al.util.dataset.should_simulate(str(dataset_path)):\n",
     "    import subprocess\n",
     "    import sys\n",
     "\n",

notebooks/chapter_3_search_chaining/tutorial_5_complex_source.ipynb

@@ -117,7 +117,7 @@
    "cell_type": "code",
    "metadata": {},
    "source": [
-    "if not dataset_path.exists():\n",
+    "if al.util.dataset.should_simulate(str(dataset_path)):\n",
     "    import subprocess\n",
     "    import sys\n",
     "\n",

notebooks/chapter_4_pixelizations/tutorial_10_brightness_adaption.ipynb

@@ -121,7 +121,7 @@
    "cell_type": "code",
    "metadata": {},
    "source": [
-    "if not dataset_path.exists():\n",
+    "if al.util.dataset.should_simulate(str(dataset_path)):\n",
     "    import subprocess\n",
     "    import sys\n",
     "\n",

notebooks/chapter_4_pixelizations/tutorial_11_adaptive_regularization.ipynb

@@ -113,7 +113,7 @@
    "cell_type": "code",
    "metadata": {},
    "source": [
-    "if not dataset_path.exists():\n",
+    "if al.util.dataset.should_simulate(str(dataset_path)):\n",
     "    import subprocess\n",
     "    import sys\n",
     "\n",

notebooks/chapter_4_pixelizations/tutorial_2_mappers.ipynb

@@ -112,7 +112,7 @@
    "cell_type": "code",
    "metadata": {},
    "source": [
-    "if not dataset_path.exists():\n",
+    "if al.util.dataset.should_simulate(str(dataset_path)):\n",
     "    import subprocess\n",
     "    import sys\n",
     "\n",

notebooks/chapter_4_pixelizations/tutorial_3_inversions.ipynb

@@ -114,7 +114,7 @@
    "cell_type": "code",
    "metadata": {},
    "source": [
-    "if not dataset_path.exists():\n",
+    "if al.util.dataset.should_simulate(str(dataset_path)):\n",
     "    import subprocess\n",
     "    import sys\n",
     "\n",

notebooks/chapter_4_pixelizations/tutorial_4_bayesian_regularization.ipynb

@@ -117,7 +117,7 @@
    "cell_type": "code",
    "metadata": {},
    "source": [
-    "if not dataset_path.exists():\n",
+    "if al.util.dataset.should_simulate(str(dataset_path)):\n",
     "    import subprocess\n",
     "    import sys\n",
     "\n",

notebooks/chapter_4_pixelizations/tutorial_5_borders.ipynb

@@ -111,7 +111,7 @@
    "cell_type": "code",
    "metadata": {},
    "source": [
-    "if not dataset_path.exists():\n",
+    "if al.util.dataset.should_simulate(str(dataset_path)):\n",
     "    import subprocess\n",
     "    import sys\n",
     "\n",

notebooks/chapter_4_pixelizations/tutorial_6_lens_modeling.ipynb

@@ -118,7 +118,7 @@
    "cell_type": "code",
    "metadata": {},
    "source": [
-    "if not dataset_path.exists():\n",
+    "if al.util.dataset.should_simulate(str(dataset_path)):\n",
     "    import subprocess\n",
     "    import sys\n",
     "\n",
@@ -319,6 +319,15 @@
     "dataset_name = \"lens_sersic\"\n",
     "dataset_path = Path(\"dataset\") / \"imaging\" / dataset_name\n",
     "\n",
+    "if al.util.dataset.should_simulate(str(dataset_path)):\n",
+    "    import subprocess\n",
+    "    import sys\n",
+    "\n",
+    "    subprocess.run(\n",
+    "        [sys.executable, \"scripts/simulator/lens_sersic.py\"],\n",
+    "        check=True,\n",
+    "    )\n",
+    "\n",
     "dataset = al.Imaging.from_fits(\n",
     "    data_path=dataset_path / \"data.fits\",\n",
     "    noise_map_path=dataset_path / \"noise_map.fits\",\n",

notebooks/chapter_4_pixelizations/tutorial_7_adaptive_pixelization.ipynb

@@ -113,7 +113,7 @@
    "cell_type": "code",
    "metadata": {},
    "source": [
-    "if not dataset_path.exists():\n",
+    "if al.util.dataset.should_simulate(str(dataset_path)):\n",
     "    import subprocess\n",
     "    import sys\n",
     "\n",

notebooks/chapter_optional/tutorial_searches.ipynb

@@ -112,7 +112,7 @@
    "cell_type": "code",
    "metadata": {},
    "source": [
-    "if not dataset_path.exists():\n",
+    "if al.util.dataset.should_simulate(str(dataset_path)):\n",
     "    import subprocess\n",
     "    import sys\n",
     "\n",

scripts/chapter_1_introduction/tutorial_0_visualization.py

@@ -45,6 +45,21 @@
 
 dataset_path = Path("dataset") / "imaging" / "simple__no_lens_light"
 
+"""
+__Dataset Auto-Simulation__
+
+If the dataset does not already exist on your system, it is created by running the corresponding
+simulator script. This ensures every example script can be run without manually simulating data first.
+"""
+if al.util.dataset.should_simulate(str(dataset_path)):
+    import subprocess
+    import sys
+
+    subprocess.run(
+        [sys.executable, "scripts/simulator/no_lens_light.py"],
+        check=True,
+    )
+
 """
 We now load this dataset from .fits files and create an instance of an `Imaging` object.
 """

scripts/chapter_1_introduction/tutorial_7_fitting.py

@@ -66,15 +66,16 @@
 """
 __Dataset Auto-Simulation__
 
-If the dataset does not already exist on your system, it will be created by running the corresponding
-simulator script. This ensures that all example scripts can be run without manually simulating data first.
+The `howtolens` dataset is the one built up and saved in tutorial 6 (`tutorial_6_data.py`). If it does
+not already exist on your system, it is created by running that script. This ensures every example
+script can be run without manually simulating data first.
 """
-if not dataset_path.exists():
+if al.util.dataset.should_simulate(str(dataset_path)):
     import subprocess
     import sys
 
     subprocess.run(
-        [sys.executable, "scripts/simulator/no_lens_light__mass_sis.py"],
+        [sys.executable, "scripts/chapter_1_introduction/tutorial_6_data.py"],
         check=True,
     )
 
@@ -616,6 +617,15 @@
 dataset_name = "simple__no_lens_light__mass_sis"
 dataset_path = Path("dataset") / "imaging" / dataset_name
 
+if al.util.dataset.should_simulate(str(dataset_path)):
+    import subprocess
+    import sys
+
+    subprocess.run(
+        [sys.executable, "scripts/simulator/no_lens_light__mass_sis.py"],
+        check=True,
+    )
+
 dataset = al.Imaging.from_fits(
     data_path=dataset_path / "data.fits",
     psf_path=dataset_path / "psf.fits",

scripts/chapter_2_lens_modeling/tutorial_1_non_linear_search.py

@@ -201,7 +201,7 @@
 If the dataset does not already exist on your system, it will be created by running the corresponding
 simulator script. This ensures that all example scripts can be run without manually simulating data first.
 """
-if not dataset_path.exists():
+if al.util.dataset.should_simulate(str(dataset_path)):
     import subprocess
     import sys
 

scripts/chapter_2_lens_modeling/tutorial_2_practicalities.py

@@ -111,7 +111,7 @@
 If the dataset does not already exist on your system, it will be created by running the corresponding
 simulator script. This ensures that all example scripts can be run without manually simulating data first.
 """
-if not dataset_path.exists():
+if al.util.dataset.should_simulate(str(dataset_path)):
     import subprocess
     import sys
 

scripts/chapter_2_lens_modeling/tutorial_3_realism_and_complexity.py

@@ -61,7 +61,7 @@
 If the dataset does not already exist on your system, it will be created by running the corresponding
 simulator script. This ensures that all example scripts can be run without manually simulating data first.
 """
-if not dataset_path.exists():
+if al.util.dataset.should_simulate(str(dataset_path)):
     import subprocess
     import sys
 

scripts/chapter_2_lens_modeling/tutorial_4_dealing_with_failure.py

@@ -57,7 +57,7 @@
 If the dataset does not already exist on your system, it will be created by running the corresponding
 simulator script. This ensures that all example scripts can be run without manually simulating data first.
 """
-if not dataset_path.exists():
+if al.util.dataset.should_simulate(str(dataset_path)):
     import subprocess
     import sys