Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -24,21 +24,21 @@ repos:

# Python file formatting
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.15.4
rev: v0.15.9
hooks:
- id: ruff
args: [--fix, --exit-non-zero-on-fix]
- id: ruff-format

# Snakemake file formatting
- repo: https://github.com/snakemake/snakefmt
rev: v0.11.4
rev: v1.0.0
hooks:
- id: snakefmt

# Spelling
- repo: https://github.com/codespell-project/codespell
rev: v2.4.1
rev: v2.4.2
hooks:
- id: codespell
files: .*\.(py|smk|md)$|^Snakefile$
Expand Down
4 changes: 2 additions & 2 deletions tests/integration/Snakefile
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,11 @@ CATEGORIES = [

# Request something from the module
rule all:
message:
"A generic test case for your module."
default_target: True
input:
expand(
"results/integration_test/results/MNE/aggregated/adjusted/{category}.parquet",
category=CATEGORIES,
),
message:
"A generic test case for your module."
4 changes: 2 additions & 2 deletions workflow/Snakefile
Original file line number Diff line number Diff line change
Expand Up @@ -37,14 +37,14 @@ workflow.source_path("scripts/_utils.py")


rule all:
message:
"ERROR: Invalid `rule all:` call"
default_target: True
output:
"INVALID",
log:
stderr="logs/all.stderr",
conda:
"envs/shell.yaml"
message:
"ERROR: Invalid `rule all:` call"
shell:
'echo "This workflow must be called as a snakemake module." > {log.stderr}'
12 changes: 6 additions & 6 deletions workflow/rules/aggregate.smk
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,6 @@ ADJUSTMENTS = ("adjusted", "unadjusted")


rule aggregate_capacity:
message:
"Aggregating capacity for {wildcards.shapes}-{wildcards.adjustment}-{wildcards.category}."
params:
year=config["imputation"]["adjustment_year"],
input:
powerplants="results/{shapes}/disaggregated/{adjustment}/{category}.parquet",
shapes="resources/user/{shapes}/shapes.parquet",
Expand All @@ -19,12 +15,16 @@ rule aggregate_capacity:
category="Powerplants module",
subcategory="{category}",
),
log:
"logs/aggregate_capacity_{shapes}_{adjustment}_{category}.log",
wildcard_constraints:
adjustment="|".join(ADJUSTMENTS),
category="|".join(IMPUTED_CAT),
log:
"logs/aggregate_capacity_{shapes}_{adjustment}_{category}.log",
conda:
"../envs/shapes.yaml"
params:
year=config["imputation"]["adjustment_year"],
message:
"Aggregating capacity for {wildcards.shapes}-{wildcards.adjustment}-{wildcards.category}."
script:
"../scripts/aggregate_capacity.py"
32 changes: 16 additions & 16 deletions workflow/rules/automatic.smk
Original file line number Diff line number Diff line change
Expand Up @@ -2,67 +2,67 @@


rule download_eia:
message:
"Download the EIA International energy statistics in bulk."
params:
url=internal["resources"]["automatic"]["EIA"],
output:
path="resources/automatic/downloads/EIA-INTL.txt",
log:
"logs/download_eia.log",
conda:
"../envs/shell.yaml"
params:
url=internal["resources"]["automatic"]["EIA"],
message:
"Download the EIA International energy statistics in bulk."
shell:
"""
curl -sSLo {output.path:q} {params.url:q}
"""


rule download_tz_sam:
message:
"Download the Transition Zero - Solar Asset Mapper dataset."
params:
url=internal["resources"]["automatic"]["TZ-SAM"],
output:
path="resources/automatic/downloads/TZ-SAM.gpkg",
log:
"logs/download_tz_sam.log",
conda:
"../envs/shell.yaml"
params:
url=internal["resources"]["automatic"]["TZ-SAM"],
message:
"Download the Transition Zero - Solar Asset Mapper dataset."
shell:
"""
curl -sSLo {output.path:q} {params.url:q}
"""


rule download_glohydrores:
message:
"Download the GloHydroRes dataset."
params:
url=internal["resources"]["automatic"]["GloHydroRes"],
output:
path="resources/automatic/downloads/GloHydroRes.csv",
log:
"logs/download_glohydrores.log",
conda:
"../envs/shell.yaml"
params:
url=internal["resources"]["automatic"]["GloHydroRes"],
message:
"Download the GloHydroRes dataset."
shell:
"""
curl -sSLo {output.path:q} {params.url:q}
"""


rule download_gem:
message:
"Download the GEM {wildcards.dataset} dataset."
params:
url=lambda wc: internal["resources"]["automatic"]["GEM"][wc.dataset],
output:
path="resources/automatic/downloads/GEM_{dataset}.xlsx",
log:
"logs/download_gem_{dataset}.log",
conda:
"../envs/shell.yaml"
params:
url=lambda wc: internal["resources"]["automatic"]["GEM"][wc.dataset],
message:
"Download the GEM {wildcards.dataset} dataset."
shell:
"""
curl -sSLo {output.path:q} {params.url:q}
Expand Down
42 changes: 21 additions & 21 deletions workflow/rules/impute.smk
Original file line number Diff line number Diff line change
Expand Up @@ -14,34 +14,29 @@ IMPUTED_CAT_WITHOUT_ADJUSTMENT = {"large_solar"}


rule impute_years:
message:
"National-level imputation of missing years for all powerplants in {wildcards.shapes}-{wildcards.dataset} dataset."
params:
imputation=config["imputation"],
projected_crs=config["projected_crs"],
tech_map=lambda wc: get_technology_mapping(wc.dataset),
input:
prepared="resources/automatic/prepared/{dataset}.parquet",
shapes="resources/user/{shapes}/shapes.parquet",
output:
imputed="resources/automatic/{shapes}/imputed/{dataset}.parquet",
plot="resources/automatic/{shapes}/imputed/{dataset}.pdf",
wildcard_constraints:
dataset="|".join(PREPARED_PLANT_CAT),
log:
"logs/impute_years_{shapes}_{dataset}.log",
wildcard_constraints:
dataset="|".join(PREPARED_PLANT_CAT),
conda:
"../envs/shapes.yaml"
params:
imputation=config["imputation"],
projected_crs=config["projected_crs"],
tech_map=lambda wc: get_technology_mapping(wc.dataset),
message:
"National-level imputation of missing years for all powerplants in {wildcards.shapes}-{wildcards.dataset} dataset."
script:
"../scripts/impute_years.py"


rule impute_category_combination:
message:
"National-level imputation of user-configured inclusions and exclusions for {wildcards.shapes}-{wildcards.category}."
params:
tech_map=lambda wc: get_technology_mapping(f"{wc.category}"),
excluded=lambda wc: get_excluded_powerplant_ids(f"{wc.category}"),
input:
to_combine=lambda wc: get_files_to_combine(wc.shapes, wc.category),
output:
Expand All @@ -58,21 +53,22 @@ rule impute_category_combination:
category="Powerplants module",
subcategory="{category}",
),
wildcard_constraints:
category="|".join(IMPUTED_CAT),
log:
"logs/impute_category_combination_{shapes}_{category}.log",
wildcard_constraints:
category="|".join(IMPUTED_CAT),
conda:
"../envs/shapes.yaml"
params:
tech_map=lambda wc: get_technology_mapping(f"{wc.category}"),
excluded=lambda wc: get_excluded_powerplant_ids(f"{wc.category}"),
message:
"National-level imputation of user-configured inclusions and exclusions for {wildcards.shapes}-{wildcards.category}."
script:
"../scripts/impute_category_combination.py"


rule impute_capacity_adjustment:
message:
"National-level adjustment of powerplant capacity in {wildcards.shapes}-{wildcards.category} to {params.year} statistics."
params:
year=config["imputation"]["adjustment_year"],
input:
unadjusted="results/{shapes}/disaggregated/unadjusted/{category}.parquet",
stats="results/{shapes}/statistics/category_capacity.parquet",
Expand All @@ -84,11 +80,15 @@ rule impute_capacity_adjustment:
category="Powerplants module",
subcategory="{category}",
),
wildcard_constraints:
category="|".join(IMPUTED_CAT - IMPUTED_CAT_WITHOUT_ADJUSTMENT),
log:
"logs/impute_capacity_adjustment_{shapes}_{category}.log",
wildcard_constraints:
category="|".join(IMPUTED_CAT - IMPUTED_CAT_WITHOUT_ADJUSTMENT),
conda:
"../envs/shapes.yaml"
params:
year=config["imputation"]["adjustment_year"],
message:
"National-level adjustment of powerplant capacity in {wildcards.shapes}-{wildcards.category} to {params.year} statistics."
script:
"../scripts/impute_capacity_adjustment.py"
Loading