fix(howtogalaxy): should_simulate every dataset load; run under SMALL_DATASETS

config/build/env_vars.yaml

@@ -21,12 +21,10 @@ defaults:
   MPLCONFIGDIR: "/tmp/matplotlib"       # Writable config dir for matplotlib
 
 overrides:
-  # Non-simulator scripts that load committed FITS data need full-size
-  # datasets to avoid shape mismatch with pre-existing 100x100 data.
-  - pattern: "howtogalaxy/"
-    unset: [PYAUTO_SMALL_DATASETS]
-  - pattern: "guides/"
-    unset: [PYAUTO_SMALL_DATASETS]
+  # (Removed the dead `howtogalaxy/` + `guides/` PYAUTO_SMALL_DATASETS-unset
+  # workarounds: they matched no files here, and the tutorials now use
+  # `ag.util.dataset.should_simulate` which deletes + re-simulates stale data
+  # under the cap, so the chapters run correctly at 16x16.)
   # guides/results/start_here.py must produce real samples so the example
   # scripts that read from `output/results_folder` afterwards
   # (data_fitting, queries, models, samples_via_aggregator) find a

notebooks/chapter_1_introduction/tutorial_0_visualization.ipynb

@@ -128,7 +128,7 @@
    "cell_type": "code",
    "metadata": {},
    "source": [
-    "if not dataset_path.exists():\n",
+    "if ag.util.dataset.should_simulate(str(dataset_path)):\n",
     "    import subprocess\n",
     "    import sys\n",
     "\n",

notebooks/chapter_1_introduction/tutorial_3_fitting.ipynb

@@ -118,20 +118,21 @@
    "source": [
     "__Dataset Auto-Simulation__\n",
     "\n",
-    "If the dataset does not already exist on your system, it will be created by running the corresponding\n",
-    "simulator script. This ensures that all example scripts can be run without manually simulating data first."
+    "The `howtogalaxy` dataset is the one built up and saved in tutorial 2 (`tutorial_2_data.py`). If it\n",
+    "does not already exist on your system, it is created by running that script. This ensures every\n",
+    "example script can be run without manually simulating data first."
    ]
   },
   {
    "cell_type": "code",
    "metadata": {},
    "source": [
-    "if not dataset_path.exists():\n",
+    "if ag.util.dataset.should_simulate(str(dataset_path)):\n",
     "    import subprocess\n",
     "    import sys\n",
     "\n",
     "    subprocess.run(\n",
-    "        [sys.executable, \"scripts/simulators/simple.py\"],\n",
+    "        [sys.executable, \"scripts/chapter_1_introduction/tutorial_2_data.py\"],\n",
     "        check=True,\n",
     "    )\n",
     "\n",
@@ -984,6 +985,15 @@
     "dataset_name = \"simple\"\n",
     "dataset_path = Path(\"dataset\") / \"imaging\" / dataset_name\n",
     "\n",
+    "if ag.util.dataset.should_simulate(str(dataset_path)):\n",
+    "    import subprocess\n",
+    "    import sys\n",
+    "\n",
+    "    subprocess.run(\n",
+    "        [sys.executable, \"scripts/simulators/simple.py\"],\n",
+    "        check=True,\n",
+    "    )\n",
+    "\n",
     "dataset = ag.Imaging.from_fits(\n",
     "    data_path=dataset_path / \"data.fits\",\n",
     "    psf_path=dataset_path / \"psf.fits\",\n",

notebooks/chapter_2_modeling/tutorial_1_non_linear_search.ipynb

@@ -284,7 +284,7 @@
    "cell_type": "code",
    "metadata": {},
    "source": [
-    "if not dataset_path.exists():\n",
+    "if ag.util.dataset.should_simulate(str(dataset_path)):\n",
     "    import subprocess\n",
     "    import sys\n",
     "\n",

notebooks/chapter_2_modeling/tutorial_2_practicalities.ipynb

@@ -152,7 +152,7 @@
    "cell_type": "code",
    "metadata": {},
    "source": [
-    "if not dataset_path.exists():\n",
+    "if ag.util.dataset.should_simulate(str(dataset_path)):\n",
     "    import subprocess\n",
     "    import sys\n",
     "\n",

notebooks/chapter_2_modeling/tutorial_3_realism_and_complexity.ipynb

@@ -119,7 +119,7 @@
    "cell_type": "code",
    "metadata": {},
    "source": [
-    "if not dataset_path.exists():\n",
+    "if ag.util.dataset.should_simulate(str(dataset_path)):\n",
     "    import subprocess\n",
     "    import sys\n",
     "\n",

notebooks/chapter_2_modeling/tutorial_4_dealing_with_failure.ipynb

@@ -120,7 +120,7 @@
    "cell_type": "code",
    "metadata": {},
    "source": [
-    "if not dataset_path.exists():\n",
+    "if ag.util.dataset.should_simulate(str(dataset_path)):\n",
     "    import subprocess\n",
     "    import sys\n",
     "\n",

notebooks/chapter_2_modeling/tutorial_5_linear_profiles.ipynb

@@ -134,7 +134,7 @@
    "cell_type": "code",
    "metadata": {},
    "source": [
-    "if not dataset_path.exists():\n",
+    "if ag.util.dataset.should_simulate(str(dataset_path)):\n",
     "    import subprocess\n",
     "    import sys\n",
     "\n",

notebooks/chapter_2_modeling/tutorial_6_masking.ipynb

@@ -107,7 +107,7 @@
    "cell_type": "code",
    "metadata": {},
    "source": [
-    "if not dataset_path.exists():\n",
+    "if ag.util.dataset.should_simulate(str(dataset_path)):\n",
     "    import subprocess\n",
     "    import sys\n",
     "\n",

notebooks/chapter_2_modeling/tutorial_7_results.ipynb

@@ -105,7 +105,7 @@
    "cell_type": "code",
    "metadata": {},
    "source": [
-    "if not dataset_path.exists():\n",
+    "if ag.util.dataset.should_simulate(str(dataset_path)):\n",
     "    import subprocess\n",
     "    import sys\n",
     "\n",

notebooks/chapter_3_search_chaining/tutorial_1_search_chaining.ipynb

@@ -140,7 +140,7 @@
    "cell_type": "code",
    "metadata": {},
    "source": [
-    "if not dataset_path.exists():\n",
+    "if ag.util.dataset.should_simulate(str(dataset_path)):\n",
     "    import subprocess\n",
     "    import sys\n",
     "\n",

notebooks/chapter_3_search_chaining/tutorial_2_prior_passing.ipynb

@@ -119,7 +119,7 @@
    "cell_type": "code",
    "metadata": {},
    "source": [
-    "if not dataset_path.exists():\n",
+    "if ag.util.dataset.should_simulate(str(dataset_path)):\n",
     "    import subprocess\n",
     "    import sys\n",
     "\n",

notebooks/chapter_3_search_chaining/tutorial_3_x2_galaxies.ipynb

@@ -122,7 +122,7 @@
    "cell_type": "code",
    "metadata": {},
    "source": [
-    "if not dataset_path.exists():\n",
+    "if ag.util.dataset.should_simulate(str(dataset_path)):\n",
     "    import subprocess\n",
     "    import sys\n",
     "\n",

notebooks/chapter_4_pixelizations/tutorial_2_mappers.ipynb

@@ -109,7 +109,7 @@
    "cell_type": "code",
    "metadata": {},
    "source": [
-    "if not dataset_path.exists():\n",
+    "if ag.util.dataset.should_simulate(str(dataset_path)):\n",
     "    import subprocess\n",
     "    import sys\n",
     "\n",

notebooks/chapter_4_pixelizations/tutorial_3_inversions.ipynb

@@ -113,7 +113,7 @@
    "cell_type": "code",
    "metadata": {},
    "source": [
-    "if not dataset_path.exists():\n",
+    "if ag.util.dataset.should_simulate(str(dataset_path)):\n",
     "    import subprocess\n",
     "    import sys\n",
     "\n",

notebooks/chapter_4_pixelizations/tutorial_4_bayesian_regularization.ipynb

@@ -117,7 +117,7 @@
    "cell_type": "code",
    "metadata": {},
    "source": [
-    "if not dataset_path.exists():\n",
+    "if ag.util.dataset.should_simulate(str(dataset_path)):\n",
     "    import subprocess\n",
     "    import sys\n",
     "\n",

notebooks/chapter_4_pixelizations/tutorial_5_model_fit.ipynb

@@ -123,7 +123,7 @@
    "cell_type": "code",
    "metadata": {},
    "source": [
-    "if not dataset_path.exists():\n",
+    "if ag.util.dataset.should_simulate(str(dataset_path)):\n",
     "    import subprocess\n",
     "    import sys\n",
     "\n",

notebooks/chapter_optional/tutorial_searches.ipynb

@@ -110,7 +110,7 @@
    "cell_type": "code",
    "metadata": {},
    "source": [
-    "if not dataset_path.exists():\n",
+    "if ag.util.dataset.should_simulate(str(dataset_path)):\n",
     "    import subprocess\n",
     "    import sys\n",
     "\n",

scripts/chapter_1_introduction/tutorial_0_visualization.py

@@ -48,7 +48,7 @@
 If the dataset does not already exist on your system, it will be created by running the corresponding
 simulator script. This ensures that all example scripts can be run without manually simulating data first.
 """
-if not dataset_path.exists():
+if ag.util.dataset.should_simulate(str(dataset_path)):
     import subprocess
     import sys
 

scripts/chapter_1_introduction/tutorial_3_fitting.py

@@ -54,15 +54,16 @@
 """
 __Dataset Auto-Simulation__
 
-If the dataset does not already exist on your system, it will be created by running the corresponding
-simulator script. This ensures that all example scripts can be run without manually simulating data first.
+The `howtogalaxy` dataset is the one built up and saved in tutorial 2 (`tutorial_2_data.py`). If it
+does not already exist on your system, it is created by running that script. This ensures every
+example script can be run without manually simulating data first.
 """
-if not dataset_path.exists():
+if ag.util.dataset.should_simulate(str(dataset_path)):
     import subprocess
     import sys
 
     subprocess.run(
-        [sys.executable, "scripts/simulators/simple.py"],
+        [sys.executable, "scripts/chapter_1_introduction/tutorial_2_data.py"],
         check=True,
     )
 
@@ -574,6 +575,15 @@
 dataset_name = "simple"
 dataset_path = Path("dataset") / "imaging" / dataset_name
 
+if ag.util.dataset.should_simulate(str(dataset_path)):
+    import subprocess
+    import sys
+
+    subprocess.run(
+        [sys.executable, "scripts/simulators/simple.py"],
+        check=True,
+    )
+
 dataset = ag.Imaging.from_fits(
     data_path=dataset_path / "data.fits",
     psf_path=dataset_path / "psf.fits",

scripts/chapter_2_modeling/tutorial_1_non_linear_search.py

@@ -204,7 +204,7 @@
 If the dataset does not already exist on your system, it will be created by running the corresponding
 simulator script. This ensures that all example scripts can be run without manually simulating data first.
 """
-if not dataset_path.exists():
+if ag.util.dataset.should_simulate(str(dataset_path)):
     import subprocess
     import sys
 

scripts/chapter_2_modeling/tutorial_2_practicalities.py

@@ -72,7 +72,7 @@
 If the dataset does not already exist on your system, it will be created by running the corresponding
 simulator script. This ensures that all example scripts can be run without manually simulating data first.
 """
-if not dataset_path.exists():
+if ag.util.dataset.should_simulate(str(dataset_path)):
     import subprocess
     import sys
 

scripts/chapter_2_modeling/tutorial_3_realism_and_complexity.py

@@ -50,7 +50,7 @@
 If the dataset does not already exist on your system, it will be created by running the corresponding
 simulator script. This ensures that all example scripts can be run without manually simulating data first.
 """
-if not dataset_path.exists():
+if ag.util.dataset.should_simulate(str(dataset_path)):
     import subprocess
     import sys
 

scripts/chapter_2_modeling/tutorial_4_dealing_with_failure.py

@@ -51,7 +51,7 @@
 If the dataset does not already exist on your system, it will be created by running the corresponding
 simulator script. This ensures that all example scripts can be run without manually simulating data first.
 """
-if not dataset_path.exists():
+if ag.util.dataset.should_simulate(str(dataset_path)):
     import subprocess
     import sys
 

scripts/chapter_2_modeling/tutorial_5_linear_profiles.py

@@ -65,7 +65,7 @@
 If the dataset does not already exist on your system, it will be created by running the corresponding
 simulator script. This ensures that all example scripts can be run without manually simulating data first.
 """
-if not dataset_path.exists():
+if ag.util.dataset.should_simulate(str(dataset_path)):
     import subprocess
     import sys
 

scripts/chapter_2_modeling/tutorial_6_masking.py

@@ -38,7 +38,7 @@
 If the dataset does not already exist on your system, it will be created by running the corresponding
 simulator script. This ensures that all example scripts can be run without manually simulating data first.
 """
-if not dataset_path.exists():
+if ag.util.dataset.should_simulate(str(dataset_path)):
     import subprocess
     import sys
 

scripts/chapter_2_modeling/tutorial_7_results.py

@@ -36,7 +36,7 @@
 If the dataset does not already exist on your system, it will be created by running the corresponding
 simulator script. This ensures that all example scripts can be run without manually simulating data first.
 """
-if not dataset_path.exists():
+if ag.util.dataset.should_simulate(str(dataset_path)):
     import subprocess
     import sys
 

scripts/chapter_3_search_chaining/tutorial_1_search_chaining.py

@@ -71,7 +71,7 @@
 If the dataset does not already exist on your system, it will be created by running the corresponding
 simulator script. This ensures that all example scripts can be run without manually simulating data first.
 """
-if not dataset_path.exists():
+if ag.util.dataset.should_simulate(str(dataset_path)):
     import subprocess
     import sys
 

scripts/chapter_3_search_chaining/tutorial_2_prior_passing.py

@@ -50,7 +50,7 @@
 If the dataset does not already exist on your system, it will be created by running the corresponding
 simulator script. This ensures that all example scripts can be run without manually simulating data first.
 """
-if not dataset_path.exists():
+if ag.util.dataset.should_simulate(str(dataset_path)):
     import subprocess
     import sys
 

scripts/chapter_3_search_chaining/tutorial_3_x2_galaxies.py

@@ -53,7 +53,7 @@
 If the dataset does not already exist on your system, it will be created by running the corresponding
 simulator script. This ensures that all example scripts can be run without manually simulating data first.
 """
-if not dataset_path.exists():
+if ag.util.dataset.should_simulate(str(dataset_path)):
     import subprocess
     import sys
 

scripts/chapter_4_pixelizations/tutorial_2_mappers.py

@@ -40,7 +40,7 @@
 If the dataset does not already exist on your system, it will be created by running the corresponding
 simulator script. This ensures that all example scripts can be run without manually simulating data first.
 """
-if not dataset_path.exists():
+if ag.util.dataset.should_simulate(str(dataset_path)):
     import subprocess
     import sys
 

scripts/chapter_4_pixelizations/tutorial_3_inversions.py

@@ -44,7 +44,7 @@
 If the dataset does not already exist on your system, it will be created by running the corresponding
 simulator script. This ensures that all example scripts can be run without manually simulating data first.
 """
-if not dataset_path.exists():
+if ag.util.dataset.should_simulate(str(dataset_path)):
     import subprocess
     import sys
 

scripts/chapter_4_pixelizations/tutorial_4_bayesian_regularization.py

@@ -48,7 +48,7 @@
 If the dataset does not already exist on your system, it will be created by running the corresponding
 simulator script. This ensures that all example scripts can be run without manually simulating data first.
 """
-if not dataset_path.exists():
+if ag.util.dataset.should_simulate(str(dataset_path)):
     import subprocess
     import sys
 

scripts/chapter_4_pixelizations/tutorial_5_model_fit.py

@@ -54,7 +54,7 @@
 If the dataset does not already exist on your system, it will be created by running the corresponding
 simulator script. This ensures that all example scripts can be run without manually simulating data first.
 """
-if not dataset_path.exists():
+if ag.util.dataset.should_simulate(str(dataset_path)):
     import subprocess
     import sys
 

scripts/chapter_optional/tutorial_searches.py

@@ -41,7 +41,7 @@
 If the dataset does not already exist on your system, it will be created by running the corresponding
 simulator script. This ensures that all example scripts can be run without manually simulating data first.
 """
-if not dataset_path.exists():
+if ag.util.dataset.should_simulate(str(dataset_path)):
     import subprocess
     import sys
 

workspace_index.json

@@ -58,7 +58,9 @@
       "Model Fitting",
       "Wrap Up"
     ],
-    "cross_refs": [],
+    "cross_refs": [
+      "tutorial_2_data.py"
+    ],
     "notebook": "notebooks/chapter_1_introduction/tutorial_3_fitting.ipynb",
     "path": "scripts/chapter_1_introduction/tutorial_3_fitting.py",
     "summary": "In previous tutorials, we used light profiles to create simulated images of galaxies and visualized how these images would appear when captured by a CCD detector on a telescope like the Hubble Space Telescope.",