refactor: simplify workflow and remove obsolete code paths

CHANGELOG.md

@@ -95,6 +95,9 @@ introduce breaking changes to configuration and outputs.
   re-stamps a set without solving, for when a code change provably cannot move
   the artefacts.
 
+- `config/default.yaml` now has the canonical name `default`. All shipped
+  configuration fields are validated as required; workflow code no longer
+  supplies hidden fallback values for missing keys.
 - `planning_horizon` now defaults to 2020, matching `baseline_year`, so an
   unmodified run solves the observed year the calibration artefacts are fit
   against. Configs that previously relied on the 2030 default (including
@@ -249,6 +252,12 @@ introduce breaking changes to configuration and outputs.
 
 - The MARS surrogate method; supported surrogates are now `pce`, `rf`, `xgb`
   and `mlp`.
+- Unused configuration keys `health.ssb_sugar_g_per_100g`,
+  `data.gaez.climate_model_ensemble`, and
+  `sensitivity_analysis.default_surrogate`. Surrogate methods are selected
+  explicitly in target and bundle names. Sensitivity scenarios must use the
+  separate `food_loss` and `food_waste` factors instead of the removed
+  `food_loss_waste` convenience key.
 
 ### Fixed
 

config/default.yaml

@@ -3,6 +3,8 @@
 # SPDX-License-Identifier: CC-BY-4.0
 
 # Default configuration - override in project-specific configs
+name: "default"
+
 # --- section: scenario_metadata ---
 scenarios:
   # Each key represents a named scenario that can be activated via the
@@ -1396,7 +1398,6 @@ health:
   region_clusters: 30
   breakpoint_rel_tol: 0.05  # Max Stage 1 PWL deviation as a fraction of each RR curve's amplitude (5%)
   log_rr_points: 15
-  ssb_sugar_g_per_100g: 5.7  # ~=50 kcal per 226.8 g sugar-sweetened beverage (SSB) implies ~5.7 g sugar per 100 g
   value_per_yll: 50000  # USD_2024 per year of life lost
   intake_age_min: 11  # GDD adult band starts at 11; set to 11 to retain adult intake data. Note however that GDB chronic disease risk factors are for adults of >=25 years.
   # Dietary risk factors to consider (must match GBD risk-factor items)
@@ -1746,7 +1747,6 @@ data:
     # GAEZ v5 parameters
     # Note: RES05 (yields/suitability) has ENSEMBLE, but RES02 (growing season) only has individual GCMs
     climate_model: "GFDL-ESM4" # Specific GCMs: "GFDL-ESM4", "IPSL-CM6A-LR", "MPI-ESM1-2-HR", "MRI-ESM2-0", "UKESM1-0-LL"
-    climate_model_ensemble: "ENSEMBLE" # Multi-model mean (only available for RES05, not RES02)
     period: "FP2140" # Future: "FP2140" (2021-2040), "FP4160" (2041-2060), "FP6180" (2061-2080), "FP8100" (2081-2100); Historical: "HP0120" (2001-2020), "HP8100" (1981-2000)
     climate_scenario: "SSP126" # "SSP126" (low emissions), "SSP370" (medium, ~RCP4.5), "SSP585" (high), "HIST" (historical)
     input_level: "H" # "H" (High), "L" (Low)
@@ -1948,9 +1948,6 @@ remote_solve:
 sensitivity_analysis:
   holdout_fraction: 0.15
   threads: 6
-  # Method downstream consumers (uncertainty-band plots, notebooks) use when
-  # no explicit choice is given. Must match a key under ``methods``.
-  default_surrogate: mlp
   # When false (default), the surrogate-fit rule declares every Sobol scenario
   # the generator promises so Snakemake drives the full solve→analyse→
   # surrogate chain in a single invocation (the canonical Snakemake idiom).

config/gsa.yaml

@@ -125,7 +125,8 @@ scenarios:
             ch4: "{ch4_factor}"
             n2o: "{n2o_factor}"
             luc: "{luc_factor}"
-          food_loss_waste: "{flw_factor}"
+          food_loss: "{flw_factor}"
+          food_waste: "{flw_factor}"
           feed_conversion: "{fcr_factor}"
           health_relative_risk:
             protective: "{rr_protective}"
@@ -147,7 +148,6 @@ scenarios:
 sensitivity_analysis:
   holdout_fraction: 0.15
   threads: 6
-  default_surrogate: mlp
   # GSA solves run outside Snakemake on the cluster (tools/batch-solve), so the
   # surrogate fit reads scenarios from disk rather than declaring every output.
   discover_scenarios_on_disk: true

config/gsa_fixed_diet.yaml

@@ -99,7 +99,8 @@ scenarios:
             ch4: "{ch4_factor}"
             n2o: "{n2o_factor}"
             luc: "{luc_factor}"
-          food_loss_waste: "{flw_factor}"
+          food_loss: "{flw_factor}"
+          food_waste: "{flw_factor}"
           feed_conversion: "{fcr_factor}"
         land:
           reforestation_cap:

config/schemas/config.schema.yaml

@@ -9,6 +9,8 @@ description: |
   Based on config/default.yaml structure.
 type: object
 required:
+  - name
+  - scenarios
   - planning_horizon
   - baseline_year
   - currency_base_year
@@ -62,14 +64,16 @@ required:
   - deviation_penalty
   - solving
   - remote_solve
+  - sensitivity
+  - sensitivity_analysis
   - plotting
 additionalProperties: false
 
 properties:
   name:
     type: string
     pattern: "^[a-zA-Z0-9_-]+$"
-    description: "Optional config name identifier (overrides default)"
+    description: "Config name used for processing and result directories"
 
   scenarios:
     type: object
@@ -171,6 +175,7 @@ properties:
       - min_crop_yield_t_per_ha
       - min_grassland_yield_t_per_ha
       - min_link_area_mha
+      - min_water_capacity_mm3
       - min_water_requirement_m3_per_ha
       - min_co2_coefficient_tco2_per_ha
       - min_cost_correction_bnusd
@@ -1141,6 +1146,8 @@ properties:
       - source
       - baseline_age
       - anchor_groups_to_gbd
+      - fbs_override_foods
+      - nhanes
       - source_basis
       - fbs
       - gdd_ia
@@ -1318,13 +1325,14 @@ properties:
       - region_clusters
       - breakpoint_rel_tol
       - log_rr_points
-      - ssb_sugar_g_per_100g
+      - intake_age_min
       - value_per_yll
       - risk_factors
       - causes
       - risk_cause_map
       - gbd_rei_id
       - gbd_cause_id
+      - alternative_rr
       - clustering
     additionalProperties: false
     properties:
@@ -1355,10 +1363,6 @@ properties:
         type: integer
         minimum: 0
         description: "Lower bound (inclusive) of age buckets used for intake weighting"
-      ssb_sugar_g_per_100g:
-        type: number
-        minimum: 0
-        description: "g sugar per 100g sugar-sweetened beverage"
       value_per_yll:
         type: number
         minimum: 0
@@ -1504,7 +1508,6 @@ properties:
         type: object
         required:
           - climate_model
-          - climate_model_ensemble
           - period
           - climate_scenario
           - input_level
@@ -1516,9 +1519,6 @@ properties:
           climate_model:
             type: string
             description: "GAEZ v5 climate model (specific GCM)"
-          climate_model_ensemble:
-            type: string
-            description: "GAEZ v5 ensemble model"
           period:
             type: string
             pattern: "^(FP|HP)[0-9]{4}$"
@@ -2083,7 +2083,7 @@ properties:
 
   solving:
     type: object
-    required: [solver, io_api, threads, calculate_fixed_duals, options_gurobi, options_highs, time_limit, runtime, mem_mb, inline_analysis]
+    required: [solver, io_api, threads, calculate_fixed_duals, options_gurobi, options_highs, export_for_tuning, time_limit, runtime, mem_mb, inline_analysis]
     additionalProperties: false
     properties:
       solver:
@@ -2209,7 +2209,7 @@ properties:
   sensitivity_analysis:
     type: object
     description: "Configuration for surrogate-based sensitivity analysis methods"
-    required: [outputs, holdout_fraction, threads, default_surrogate, methods, sobol, discover_scenarios_on_disk]
+    required: [outputs, holdout_fraction, threads, methods, sobol, discover_scenarios_on_disk]
     additionalProperties: false
     properties:
       outputs:
@@ -2250,9 +2250,6 @@ properties:
         type: integer
         minimum: 1
         description: "Number of threads for sensitivity analysis (controls sklearn n_jobs and BLAS)"
-      default_surrogate:
-        type: string
-        description: "Surrogate method downstream consumers (e.g. uncertainty-band plots, notebooks) pick when no method is specified explicitly.  Must match a key under ``methods``."
       discover_scenarios_on_disk:
         type: boolean
         description: "If false (default), the surrogate-fit rule declares every scenario the generator promises so Snakemake drives the full solve→analyse→surrogate chain.  If true, the rule scans the analysis directory and fits the surrogate on whatever scenarios have complete outputs on disk; intended for cluster sweeps where solves run outside Snakemake.  Errors out if more than 50% of scenarios are missing."
@@ -2423,10 +2420,6 @@ properties:
             type: number
             minimum: 0
             description: "Factor for CO2 from land-use change"
-      food_loss_waste:
-        type: number
-        minimum: 0
-        description: "Bundle convenience key: applies the same multiplicative factor to both food_loss and food_waste. Per-component keys take precedence when both are set. factor=1.5 raises the underlying loss/waste fractions by 50%; factor=1.0 is a no-op."
       food_loss:
         type: number
         minimum: 0

data/curated/calibration/default/provenance.yaml

@@ -199,6 +199,7 @@ structural_config:
   - rendered-fat
   biomass.enforce_baseline_demand: true
   biomass.enforce_fiber_demand: true
+  biomass.marginal_values_usd_per_tonne: 0
   byproducts:
   - beet-pulp
   - wheat-bran
@@ -639,7 +640,6 @@ structural_config:
   data.faostat.fbs_production_element_code: 5511
   data.faostat.qcl_production_element_code: 5510
   data.gaez.climate_model: GFDL-ESM4
-  data.gaez.climate_model_ensemble: ENSEMBLE
   data.gaez.climate_scenario: SSP126
   data.gaez.input_level: H
   data.gaez.period: FP2140
@@ -903,6 +903,21 @@ structural_config:
   - sugar
   - stimulants
   - animal_fat
+  food_groups.max_per_capita.animal_fat: 50
+  food_groups.max_per_capita.dairy: 2865
+  food_groups.max_per_capita.eggs: 213
+  food_groups.max_per_capita.fruits: 658
+  food_groups.max_per_capita.grain: 1403
+  food_groups.max_per_capita.legumes: 300
+  food_groups.max_per_capita.nuts_seeds: 79
+  food_groups.max_per_capita.oil: 155
+  food_groups.max_per_capita.poultry: 241
+  food_groups.max_per_capita.red_meat: 285
+  food_groups.max_per_capita.starchy_vegetable: 1221
+  food_groups.max_per_capita.stimulants: 50
+  food_groups.max_per_capita.sugar: 133
+  food_groups.max_per_capita.vegetables: 785
+  food_groups.max_per_capita.whole_grains: 300
   food_loss_waste_calibration.food_groups:
   - vegetables
   - fruits
@@ -969,7 +984,6 @@ structural_config:
   - legumes
   - red_meat
   - whole_grains
-  health.ssb_sugar_g_per_100g: 5.7
   irrigation.irrigated_crops: all
   land.conversion_cost_forest_usd_per_ha: 8000
   land.conversion_cost_nonforest_usd_per_ha: 2000

data/curated/calibration/gbd-anchored/provenance.yaml

@@ -199,6 +199,7 @@ structural_config:
   - rendered-fat
   biomass.enforce_baseline_demand: true
   biomass.enforce_fiber_demand: true
+  biomass.marginal_values_usd_per_tonne: 0
   byproducts:
   - beet-pulp
   - wheat-bran
@@ -639,7 +640,6 @@ structural_config:
   data.faostat.fbs_production_element_code: 5511
   data.faostat.qcl_production_element_code: 5510
   data.gaez.climate_model: GFDL-ESM4
-  data.gaez.climate_model_ensemble: ENSEMBLE
   data.gaez.climate_scenario: SSP126
   data.gaez.input_level: H
   data.gaez.period: FP2140
@@ -903,6 +903,21 @@ structural_config:
   - sugar
   - stimulants
   - animal_fat
+  food_groups.max_per_capita.animal_fat: 50
+  food_groups.max_per_capita.dairy: 2865
+  food_groups.max_per_capita.eggs: 213
+  food_groups.max_per_capita.fruits: 658
+  food_groups.max_per_capita.grain: 1403
+  food_groups.max_per_capita.legumes: 300
+  food_groups.max_per_capita.nuts_seeds: 79
+  food_groups.max_per_capita.oil: 155
+  food_groups.max_per_capita.poultry: 241
+  food_groups.max_per_capita.red_meat: 285
+  food_groups.max_per_capita.starchy_vegetable: 1221
+  food_groups.max_per_capita.stimulants: 50
+  food_groups.max_per_capita.sugar: 133
+  food_groups.max_per_capita.vegetables: 785
+  food_groups.max_per_capita.whole_grains: 300
   food_loss_waste_calibration.food_groups:
   - vegetables
   - fruits
@@ -969,7 +984,6 @@ structural_config:
   - legumes
   - red_meat
   - whole_grains
-  health.ssb_sugar_g_per_100g: 5.7
   irrigation.irrigated_crops: all
   land.conversion_cost_forest_usd_per_ha: 8000
   land.conversion_cost_nonforest_usd_per_ha: 2000

docs/cluster_execution.rst

@@ -188,10 +188,8 @@ flag at its default ``false``.
         --allowed-rules build_surrogate \
         -- results/gsa/surrogates/surrogate_gsa_xgb.pkl
 
-The method (``xgb``/``pce``/``rf``/``mlp``) is a wildcard of the rule;
-``sensitivity_analysis.default_surrogate`` in the config picks the
-default when downstream consumers (uncertainty plots, notebooks) load
-a bundle without specifying a method.
+The method (``xgb``/``pce``/``rf``/``mlp``) is a wildcard of the rule and is
+always explicit in downstream bundle and plot targets.
 
 Repeat the command for each ``{group}`` needed (e.g., ``gsa-l1-low``,
 ``gsa-l1-high``) and for alternative surrogate types if you want to

docs/configuration.rst

@@ -335,16 +335,13 @@ Validation Options
 Set ``validation.enforce_baseline_diet`` to ``true`` to force the optimizer to match
 baseline consumption derived from the estimated baseline diet. When this flag is active,
 the ``diet.baseline_age`` and ``baseline_year`` settings determine which
-cohort/year is enforced. Use ``validation.food_group_slack_marginal_cost`` to set the
-penalty (USD\ :sub:`2024` per Mt) for the slack generators that backstop those fixed
-food-group loads. Keep the value high so slack only activates when recorded production
-cannot meet the enforced demand targets.
+cohort/year is enforced. ``validation.slack_marginal_cost`` sets the common
+penalty in bn USD per Mt or Mha for validation slack. Keep the value high so
+slack only activates when recorded production cannot meet the enforced
+demand targets.
 
 Set ``validation.enforce_baseline_feed`` to ``true`` to fix animal feed use to
-GLEAM-derived baseline levels (see :ref:`gleam-feed-baseline`). The baseline is
-scaled from GLEAM 2.0 (2010) to the reference year and calibrated against the
-known GLEAM 3.0 global total using ``validation.gleam_calibration_year`` and
-``validation.gleam_calibration_total_gt_dm``.
+GLEAM-derived baseline levels (see :ref:`gleam-feed-baseline`).
 
 See :doc:`validation` for a detailed walkthrough of the validation workflow and
 diagnostic figures.
@@ -361,7 +358,8 @@ the objective:
   curve per ``(food, country)`` pair.
 
 When ``food_utility_piecewise.enabled`` is ``true``, the workflow always reads
-utility blocks from ``results/{name}/consumer_values/utility_blocks.csv``.
+utility blocks from
+``results/{name}/consumer_values/{baseline_scenario}/utility_blocks.csv``.
 These blocks are generated by ``calibrate_food_utility_blocks`` from:
 
 * baseline dual values extracted by ``extract_consumer_values``; and
@@ -421,8 +419,11 @@ across components.
 * ``deviation_penalty.deviation_type``: ``absolute`` or ``relative``.
 * ``deviation_penalty.quadratic_cost``: shared coefficient for quadratic mode.
 * ``deviation_penalty.land.enabled`` plus per-component switches
-  ``land.crops.enabled``, ``land.grassland.enabled``, ``feed.enabled``,
-  ``diet.enabled``. ``land.land_conversion.enabled`` (default ``false``)
+  ``deviation_penalty.land.crops.enabled``,
+  ``deviation_penalty.land.grassland.enabled``,
+  ``deviation_penalty.feed.enabled``, and
+  ``deviation_penalty.diet.enabled``.
+  ``deviation_penalty.land.land_conversion.enabled`` (default ``false``)
   would additionally penalise land-use transitions, but is kept off
   because those carriers include sparing -- the penalty would tax
   reforestation from a zero baseline.
@@ -434,8 +435,8 @@ across components.
   applied after sentinel resolution; lets scenarios scan around the
   calibrated central value without hard-coding absolute numbers.
 * ``deviation_penalty.land.crops.max_relative_deviation`` /
-  ``land.grassland.max_relative_deviation`` /
-  ``feed.max_relative_deviation``: hard-mode bounds.
+  ``deviation_penalty.land.grassland.max_relative_deviation`` /
+  ``deviation_penalty.feed.max_relative_deviation``: hard-mode bounds.
 
 **Behavior notes**: