diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index d0be6252..74c3fa18 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -72,7 +72,7 @@ Include additional illustrative plots describing input data, methods, testing, a - [ ] No large data file(s) added/modified - [ ] No substantive impact on runtime for full-US reference case - [ ] No substantive impact on folder size for full-US reference case -- [ ] No change to process flow (runbatch.py, d_solve_iterate.py) +- [ ] No change to process flow (runreeds.py, reeds/core/solve/solve.py) - [ ] No change to code organization - [ ] No change to package requirements (environment.yml or Project.toml) diff --git a/.github/labeler.yaml b/.github/labeler.yaml index e0558a5b..5268fb03 100644 --- a/.github/labeler.yaml +++ b/.github/labeler.yaml @@ -9,6 +9,7 @@ dependencies: - "Project.toml" - "Manifest.toml" - ".github/dependabot.yml" + - "instantiate.jl" github_actions: - changed-files: @@ -42,15 +43,8 @@ tests: model_changes: - changed-files: - any-glob-to-any-file: - - "*.gms" - - "**/*.gms" - - "runbatch.py" - - "d_solve*.py" - "reeds/**" - - "ReEDS_Augur/**" - - "Augur.py" - - "instantiate.jl" - - "reeds2pras/**" + - "runreeds.py" data_changes: - changed-files: @@ -60,19 +54,11 @@ data_changes: - "postprocessing/**/inputs/**" - "**/*.csv" - "**/*.h5" - - "sources.csv" - - "sources_documentation.md" hourlize: - changed-files: - any-glob-to-any-file: "hourlize/**" -input_processing: - - changed-files: - - any-glob-to-any-file: - - "input_processing/**" - - "preprocessing/**" - postprocessing: - changed-files: - any-glob-to-any-file: "postprocessing/**" diff --git a/.github/workflows/build-docs.yaml b/.github/workflows/build-docs.yaml index 001cf63b..1ee03d85 100644 --- a/.github/workflows/build-docs.yaml +++ b/.github/workflows/build-docs.yaml @@ -53,10 +53,6 @@ jobs: - name: Set variables for internal github repo run: | echo "BASE_URL=https://github.com/ReEDS-Model/ReEDS" >> "$GITHUB_ENV" - cd "$GITHUB_WORKSPACE/docs/source/documentation_tools/" - sh generate_sources_md_file.sh - cd "$GITHUB_WORKSPACE" - python docs/source/documentation_tools/generate_markdown.py --githubURL "https://github.com/ReEDS-Model/ReEDS/blob/main" --reedsPath "$GITHUB_WORKSPACE" - name: Build Sphinx documentation env: diff --git a/.github/workflows/python-app.yaml b/.github/workflows/python-app.yaml index d2092cff..27dea0cf 100644 --- a/.github/workflows/python-app.yaml +++ b/.github/workflows/python-app.yaml @@ -195,7 +195,7 @@ jobs: - name: Run ReEDS model env: SCENARIO: ${{ matrix.scenario }} - run: python runbatch.py -b "$batch" -c test -s "$SCENARIO" + run: python runreeds.py -b "$batch" -c test -s "$SCENARIO" - name: Print GAMS log if: runner.debug == '1' @@ -290,7 +290,7 @@ jobs: R2X_WEATHER_YEAR: "2012" R2X_SYSTEM_JSON: ${{ format('{0}_system.json', matrix.scenario) }} run: | - uvx --from "r2x-reeds>=0.3.5" python scripts/run_r2x.py \ + uvx --from "r2x-reeds>=0.3.5" python postprocessing/run_r2x.py \ --reeds-run-path "$R2X_REEDS_RUN_PATH" \ --scenario "$R2X_SCENARIO" \ --solve-year "$R2X_SOLVE_YEAR" \ diff --git a/README.md b/README.md index 31242f4a..173e2fa6 100644 --- a/README.md +++ b/README.md @@ -63,10 +63,10 @@ A step-by-step guide for getting started with ReEDS is available [here](https:// These files are downloaded automatically as needed during a ReEDS run, but to finish all the internet-requiring steps up front, you can download them all by running `python reeds/remote.py`. Additional details on remote files and other topics can be found in the [user guide](https://reeds-model.github.io/ReEDS/user_guide.html#large-input-files). 5. Run ReEDS on a test case from the root of the cloned repository: - 1. For interactive setup: `python runbatch.py` - 2. For one-line operation: `python runbatch.py -b v20250314_main -c test`. + 1. For interactive setup: `python runreeds.py` + 2. For one-line operation: `python runreeds.py -b v20250314_main -c test`. In this example, "v20250314_main" is the prefix for this batch of cases, and "test" is the suffix of the cases file, in this case `cases_test.csv`, located in the root of the repository. - Run `python runbatch.py -h` for information on other optional command-line arguments for ReEDS. + Run `python runreeds.py -h` for information on other optional command-line arguments for ReEDS. diff --git a/cases.csv b/cases.csv index 317faeb0..e813a404 100644 --- a/cases.csv +++ b/cases.csv @@ -85,7 +85,7 @@ GSw_BinOM,Turn on/off binned FOM and VOM for each historical binned vintage,0; 1 GSw_Biopower,Turn on/off biopower,0; 1,1, GSw_BioSupply,Multiplier to adjust total biomass supply,N/A,1, GSw_BioTransportCost,Biomass collection and transport costs ($2004 per dry ton),N/A,22, -GSw_calc_powfrac,Switch to compute powfrac in e_report and e_powfrac_calc - dramatically reduces calculation times with hourly resolution,0; 1,0, +GSw_calc_powfrac,Compute powfrac in report.gms and powfrac_calc.gms - dramatically increases calculation times with hourly resolution,0; 1,0, GSw_Canada,"Turn canada off [0], or use flexible (dispatchable-hydro-like) representation [1]",0; 1,1, GSw_CarbTax,Turn on/off CO2 tax,0; 1,0, GSw_CarbTaxOption,Choose the co2_tax input csv file (see inputs\emission_constraints),,default, @@ -296,7 +296,7 @@ GSw_SitingGeo,Specify the Geothermal siting scenario,reference,reference, GSw_SitingUPV,Specify the UPV siting scenario,open; reference; limited,reference, GSw_SitingWindOfs,Specify the offshore wind siting scenario,open; reference; limited,reference, GSw_SitingWindOns,Specify the onshore wind siting scenario,open; reference; limited,reference, -GSw_SkipAugurYear,Last year in which to skip running the Augur module,N/A,2020, +GSw_SkipRAyear,Last year in which to skip running the resource adequacy (RA) calculations,N/A,2020, GSw_SpurCostMult,Multiplier for spur-line costs,float,1, GSw_SpurScen,Spur-line scenario: 0 to include in resource supply curve; 1 to model endoegenously,0; 1,0, GSw_SpurShare,Indicate whether wind-ons and upv are allowed to share spur line capacity,0; 1,0, @@ -355,9 +355,9 @@ cleanup_level,How aggressively to clean up the case folder (see postprocessing/c diagnose,Write A and B matrix of the model [0 = no diagnose ; 1 = diagnose ],0; 1,0, diagnose_year,Year in which to start report diagnose,N/A,2022, dump_alldata,switch to automatically dump data from final solve year into .gdx file,0; 1,0, -file_replacements,"List of files to replace from run folder, e.g: inputs_case/national_gen_frac.csv << //nrelnas01/ReEDS/some proj/national_gen_frac.csv || c_supplymodel.gms << //nrelnas01/ReEDS/some proj/c_supplymodel.gms",N/A,none, +file_replacements,"List of files to replace from run folder, e.g: inputs_case/national_gen_frac.csv << //nrelnas01/ReEDS/some_project/national_gen_frac.csv || c_model.gms << //nrelnas01/ReEDS/some_project/c_model.gms",N/A,none, input_processing_only,Only run input_processing scripts; stop before creating and solving model,0; 1,0, -keep_augur_files,Indicate whether to keep (1) or delete (0) Augur csv and h5 files after Augur finishes. If you plan to run PRAS via ReEDS2PRAS then set this switch to 1.,0; 1,0, +keep_resource_adequacy_files,Indicate whether to keep (1) or delete (0) resource adequacy csv and h5 files after RA calculations finish,0; 1,0, keep_g00_files,Keep (1) or delete (0) .g00 files for completed solve years,0; 1,0, keep_run_terminal,"0=close run terminal, 1=keep run terminal open",0; 1,0, land_use_analysis,switch to turn on/off land-use analysis. Requires the `reeds_to_rev` switch also be activated,0; 1,0, diff --git a/cases_small.csv b/cases_small.csv index 14ec1461..c4812e97 100644 --- a/cases_small.csv +++ b/cases_small.csv @@ -27,7 +27,7 @@ GSw_Refurb,0 GSw_PRM_CapCredit,0 GSw_Retire,0 GSw_RGGI,0 -GSw_SkipAugurYear,2030 +GSw_SkipRAyear,2030 GSw_StateCap,0 GSw_StateRPS,0 GSw_Storage,0 diff --git a/createmodel.gms b/createmodel.gms deleted file mode 100644 index 36ab4603..00000000 --- a/createmodel.gms +++ /dev/null @@ -1,5 +0,0 @@ -$include b_inputs.gms -$include c_supplymodel.gms -$include c_supplyobjective.gms -$include c_mga.gms -$include d_solveprep.gms diff --git a/docs/source/developer_best_practices.md b/docs/source/developer_best_practices.md index 43f1e0cf..2a1b3a4b 100644 --- a/docs/source/developer_best_practices.md +++ b/docs/source/developer_best_practices.md @@ -17,7 +17,7 @@ Since we have not yet adopted strict formatting guidelines, do not make code *fo -------------| --------------- | ------------ | Folders | lowercase | `inputs`, `docs` | Files | typically lowercase with underscores (acronyms often left as uppercase); output files noun first | `battery_ATB_2024_moderate.csv`, `gen_ann` not `ann_gen` | -GAMS Files | letter-underscore prefix by category, alpha-ordered; numbering can help communicate ordering when multiple files share a category | `d1_financials.gms`, `d2_varfix.gms` | +GAMS Files | letter-underscore prefix by category, alpha-ordered; numbering can help communicate ordering when multiple files share a category | `2_financials.gms`, `5_varfix.gms` | Parameters | lowercase with underscores, noun first; costs prefixed with "cost" | `curt_marg` not `marg_curt`, `cost_cap` | Variables | all caps, noun first | `INV`, `INV_TRANS` | Equations (model constraints) | prefixed with `eq_`, lowercase with underscores | `eq_reserve_margin` | @@ -107,7 +107,7 @@ _Some exceptions to this might exist due to number scaling (e.g., emission rates * GAMS functions such as sum, max, smax, etc. should use {}; Example: avg_outage(i) = sum{h,hours(h)*outage(i,h)} / 8760 ; * When including the semicolon on the end of a line there should be a space between the semicolon and the last character of the line (see previous example) * When using `/ /` for a parameter declaration, place the closing semicolon on the same line as the final slash: `/ ;` - * Sums outside of equations (e.g., in e_reports) need not be split over multiple lines if they do not exceed the line limit + * Sums outside of equations (e.g., in `report.gms`) need not be split over multiple lines if they do not exceed the line limit * Do not use hard-coded numbers in equations or calculations. Values should be assigned to an appropriate parameter name that is subsequently used in the code. * Large input data tables should be loaded from individual data files for each table, preferably in *.csv format. Large data tables should not be manually written into the code but can be written dynamically by scripts or inserted with a $include statement. * Compile-time conditionals should always use a tag (period + tag name) to clearly define the relationships between compile-time conditional statements. Failure to do so hurts readability sometimes leads to compilation errors. Example: @@ -162,21 +162,22 @@ _Some exceptions to this might exist due to number scaling (e.g., emission rates * If preprocessing is needed to create an input file that is placed in the ReEDS repository, the preprocessing scripts or workbooks should be included in the [ReEDS_Input_Processing repository](https://github.com/ReEDS-Model/ReEDS_Input_Processing). Data from external sources should be downloaded programmatically when possible. -* Any scripts that preprocess data after a ReEDS run is started should be placed in the input_processing folder. +* Any scripts that preprocess data after a ReEDS run is started should be placed in the `reeds/input_processing` folder. * Input processing scripts should start with a block of descriptive comments describing the purpose and methodology, and internal functions should use docstrings and liberal comments on functionality and assumptions. * Any costs read into b_inputs should already be in 2004$. Cost adjustments in preprocessing scripts should rely on the deflator.csv file rather than have hard-coded conversions. * In general, if inputs require calculations before they are ingested into b_inputs, those calculations should be done in Python rather than in GAMS. GAMS can be used for calculations where the GAMS syntax simplifies the calculation or where upstream dependencies make it challenging for the calculations to happen in Python preprocessing scripts. -* In Python, file paths should be added using os.path.join() rather than writing out the filepath with slashes. +* In Python, file paths should be specified using `from pathlib import Path` and `Path(reeds.io.reeds_path, 'foldername', 'maybe_more_foldernames', 'filename.extension')` instead of writing out the filepath as a string with slashes. +Use absolute filepaths whenever possible. * Data column headers should use the ReEDS set names when practical. * Example: data that include regions should use "r" for the column name rather than "ba", "reeds_ba", or "region". * Preprocessing scripts in input_processing should not change the working directory or use relative filepaths; absolute filepaths should be used wherever possible. -* When feasible, inputs used in the objective function (c_supplyobjective.gms) should be included in tests/objective_function_params.yaml. Inputs included in this .yaml file will be checked for missing values using input_processing/check_inputs.py. +* When feasible, inputs used in the objective function (`d_objective.gms`) should be included in tests/objective_function_params.yaml. Inputs included in this .yaml file will be checked for missing values using input_processing/check_inputs.py. #### Input Data @@ -629,7 +630,7 @@ Additionally, if you want to re-run a given scenario without having to run all o * To avoid the prompts when kicking off a run, you can use the command line arguments: * The following example runs the scenarios in cases_test.csv with the batch name '20240717_test'. The '-r -1' means that all cases will run simultaneously. ``` - python runbatch.py -c test -b 20240717_test -r -1 + python runreeds.py -c test -b 20240717_test -r -1 ``` * All options for command line arguments that can be used: | Flag | | diff --git a/docs/source/model_documentation.md b/docs/source/model_documentation.md index abc6afda..a9a431de 100644 --- a/docs/source/model_documentation.md +++ b/docs/source/model_documentation.md @@ -3751,7 +3751,9 @@ For all ReEDS system cost results, we assume the operational costs for the nonmo The marginal electricity prices in ReEDS are taken as the shadow prices from the constraints in the model that are directly impacted by the need to serve electricity. These include the load balance constraint, the operating reserve requirement, the RPS and CES requirements, and the planning reserve margin requirement (applied either in stress periods or seasonally via capacity credits). Taken together, these values show the total marginal cost of serving electricity in a given region and time slice. Weighted average versions are also calculated to report national annual marginal electricity prices. These marginal prices are most analogous to wholesale electricity prices but within a model that has full coordination and foresight. ```{admonition} Marginal Price Outputs -Marginal electricity price outputs are calculated with `reqt_price` in `e_report.gms` and are outputs in `reqt_price.csv`. The quantity required by the model is reported in the `reqt_quant.csv` output. These two taken together can be used to calculate a $/MWh electricity price, which is done in make of the ReEDS outputs and reported as "Bulk System Electricity Price" in bokehpivot HTML outputs and in other output locations. +Marginal electricity price outputs are recorded in `reqt_price.csv`. +The quantity required by the model is reported in `reqt_quant.csv`. +These two can be combined to calculate a \$/MWh electricity price. ``` diff --git a/docs/source/postprocessing_tools.md b/docs/source/postprocessing_tools.md index 0f2266c9..3c76961c 100644 --- a/docs/source/postprocessing_tools.md +++ b/docs/source/postprocessing_tools.md @@ -103,10 +103,10 @@ The PRAS model is typically run multiple times during each ReEDS case (as long a This script reruns PRAS on a finished ReEDS case (provided by the single required command-line argument) and allows the settings to be changed. For example, to use a different number of samples than are specified by the default `pras_samples` switch, use the `-s/--samples` command-line argument. -### Run a dispatch model: `run_pcm.py` +### Run a dispatch model: `postprocessing/run_pcm.py` This script reruns a completed ReEDS case as a dispatch simulation at higher time resolution. -The operational constraints in `c_supplymodel.gms` are used directly, but the investment and capacity variables are fixed to their previously optimized values; only the operational variables are re-optimized. +The operational constraints in `c_model.gms` are used directly, but the investment and capacity variables are fixed to their previously optimized values; only the operational variables are re-optimized. 365 representative 1-day periods at 1-hour resolution are used by default, but these settings can be changed using the `-s/--switch_mods` switch. This approach is distinct from the [R2X](https://github.com/NatLabRockies/R2X) tool, which formats the results of a ReEDS case as inputs to a separate production cost modeling tool such as [Sienna](https://github.com/NREL-Sienna) or [PLEXOS](https://www.energyexemplar.com/plexos). diff --git a/docs/source/reeds_training_homework.md b/docs/source/reeds_training_homework.md index aa65f230..6630275c 100644 --- a/docs/source/reeds_training_homework.md +++ b/docs/source/reeds_training_homework.md @@ -25,7 +25,7 @@ orphan: true - Open a new terminal and activate the reeds2 environment 7. Start a new run - - `python runbatch.py` + - `python runreeds.py` - when prompted for case file name, enter `examples` - when prompted for how many simultaneous runs you would like to execute, enter 2 - The 'ERCOT_county' and 'ERCOT_BA' runs should start @@ -74,7 +74,7 @@ Create an informal slide deck with the following results: - Open a new terminal and activate the reeds2 environment 7. Start a new run - - `python runbatch.py` + - `python runreeds.py` - when prompted for case file name, enter `examples` - The 'Western_BA_Decarb' run should start diff --git a/docs/source/setup.md b/docs/source/setup.md index 449516d5..e315fcf6 100644 --- a/docs/source/setup.md +++ b/docs/source/setup.md @@ -291,7 +291,7 @@ If you experience issues, try the following: **Quick Start:** 1. Navigate to the ReEDS directory from the command line 2. Activate environment: `conda activate reeds2` -3. Run the model: `python runbatch.py` +3. Run the model: `python runreeds.py` 4. Follow the prompts for batch configuration 5. Check for a successful run: 1. Look for CSV files in `runs/[batchname_scenario]/outputs` (a successful run should have 100+ csv files in the outputs folder) diff --git a/docs/source/user_guide.md b/docs/source/user_guide.md index 0b1a8fb7..b2278011 100644 --- a/docs/source/user_guide.md +++ b/docs/source/user_guide.md @@ -100,38 +100,6 @@ Here is partial list of remotely hosted files used by ReEDS: -## Hourly Resolution - -The model can be run at hourly resolution using the following switch settings: - -- `GSw_Hourly = 1` - - Turn on hourly resolution -- `GSw_Canada = 2` - - Turn on hourly resolution for Canadian imports/exports -- `GSw_AugurCurtailment = 0` - - Turn off the Augur calculation of curtailment -- `GSw_StorageArbitrageMult = 0` - - Turn off the Augur calculation of storage arbitrage value -- `GSw_Storage_in_Min = 0` - - Turn off the Augur calculation of storage charging -- `capcredit_szn_hours = 3` - - The current default hourly representation is 18 representative 5-day weeks. Each representative period is treated as a 'season' and is thus active in the planning-reserve margin constraint. In h17 ReEDS we set `capcredit_szn_hours = 10`, giving 40 total hours considered for planning reserves (the top 10 hours in each of the 4 quarterly seasons). 18 'seasons' with 10 hours each would give 180 hours, so we switch to 3 hours per 'season' (for 54 hours total). - -To further reduce solve time, you can make the following changes: - -- `yearset = 2010_2015_2020_2025_2030_2035_2040_2045_2050` - - Solve in 5-year steps -- `GSw_OpRes = 0` - - Turn off operating reserves -- `GSw_MinLoading = 0` - - Turn off the sliding-window representation of minimum-generation limits -- `GSw_PVB = 0` - - Turn off PV-battery hybrids -- `GSw_calc_powfrac = 0` - - Turn off a post-processing calculation of power flows - - - ## Electricity Demand Profiles ### Switch options for GSw_LoadProfiles @@ -629,7 +597,7 @@ Options are `capacity`, `transmission`, `rasharing`, and `co2`. - `GSw_MGA_SubObjective` (default `fossil`): Technology subset to minimize or maximize the capacity of (only used for `GSw_MGA_Objective = capacity`). Options are the column names in the `inputs/tech-subset-table.csv` file. -Users familiar with GAMS can add alternative objective functions to the `c_mga.gms` file and associated options to the `GSw_MGA_Objective` switch in `cases.csv`. +Users familiar with GAMS can add alternative objective functions to the `d_mga.gms` file and associated options to the `GSw_MGA_Objective` switch in `cases.csv`. @@ -746,7 +714,6 @@ This section provides guidance on identifying and resolving common issues encoun - What to look for: - `1_inputs.lst`: errors will be preceded by `****` - `{batch_prefix}_{case}_{year}i0.lst`: there should be one file for each year of the model run - - `Augur_errors_{year}`: this file will appear in the event that there is an augur-related issue - GAMS Workfiles - Path: `/runs/{batch_prefix}_{case}/g00files/` @@ -762,10 +729,10 @@ This section provides guidance on identifying and resolving common issues encoun - these files should contain data, an error message "GDX file not found" indicates an issue with the reporting script at the end of the model - `reeds-report/` and `reeds-report-reduced/`: if these folders are not present, it can indicate a problem with the post-processing scripts -- Augur Data - - Path: `/runs/{batch_prefix}_{case}/ReEDS_Augur/augur_data/` +- Resource adequacy data + - Path: `/runs/{batch_prefix}_{case}/handoff/reeds_data/` - What to look for: - - `ReEDS_Augur_{year}.gdx`: there should be a file for each year of the model run = + - `ccdata_{year}.gdx`: there should be a file for each year of the model run = - `reeds_data_{year}.gdx`: there should be a file for each year of the model run - Case Inputs diff --git a/sources.csv b/docs/sources.csv similarity index 100% rename from sources.csv rename to docs/sources.csv diff --git a/sources_documentation.md b/docs/sources_documentation.md similarity index 100% rename from sources_documentation.md rename to docs/sources_documentation.md diff --git a/interim_report.py b/helpers/interim_report.py similarity index 100% rename from interim_report.py rename to helpers/interim_report.py diff --git a/interim_report_batch.py b/helpers/interim_report_batch.py similarity index 85% rename from interim_report_batch.py rename to helpers/interim_report_batch.py index 95a62e0e..0f295a9a 100644 --- a/interim_report_batch.py +++ b/helpers/interim_report_batch.py @@ -1,8 +1,8 @@ ''' This script allows for batch re-running of just the reporting -functions (e_report.gms and e_report_dump.py) for a set of jobs +functions (report.gms and report_dump.py) for a set of jobs with a common batch prefix, specified as a command line argument. -It will make new copies of e_report.gms/e_report_dump.py to +It will make new copies of report.gms/report_dump.py to each run folder, and can thus be useful to re-run reporting for a large set of existing runs using new report scripts. @@ -18,6 +18,9 @@ from glob import glob import argparse import sys +from pathlib import Path +sys.path.append(str(Path(__file__).parent.parent)) +import reeds parser = argparse.ArgumentParser() parser.add_argument('--batch_name', '-b', type=str, default='', help='Prefix for batch of runs') @@ -31,7 +34,7 @@ if len(case_list) == 0: sys.exit(f"No cases found with {args.batch_name} prefix.") # list of new report files to copy -report_files = ["e_report.gms", "e_report_dump.py"] +report_files = ["report.gms", "report_dump.py"] # loop over cases to copy files and run 'interim_report.py' for each one for case in case_list: @@ -44,7 +47,10 @@ interim_report = os.path.join(case, "interim_report.py") if os.name=='posix': if hpc: - shutil.copy("srun_template.sh", os.path.join(case, "interim_report.sh")) + shutil.copy( + Path(reeds.io.reeds_path,'reeds','hpc','srun_template.sh'), + os.path.join(case, "interim_report.sh") + ) with open(os.path.join(case, "interim_report.sh"), 'a') as SPATH: #add the name for easy tracking of the case SPATH.writelines("\n#SBATCH --job-name=" + case_name + "_interim_report" + "\n\n") diff --git a/restart_runs.py b/helpers/restart_runs.py similarity index 78% rename from restart_runs.py rename to helpers/restart_runs.py index dee9a85c..f1132e2e 100644 --- a/restart_runs.py +++ b/helpers/restart_runs.py @@ -1,46 +1,55 @@ #%% Imports import os +import sys import shutil import subprocess import argparse -import pandas as pd from glob import glob -from runbatch import submit_slurm_parallel_jobs +from pathlib import Path +sys.path.append(str(Path(__file__).parent.parent)) +import reeds +from runreeds import submit_slurm_parallel_jobs from runstatus import get_run_status #%% Argument inputs parser = argparse.ArgumentParser(description='Restart failed runs on the HPC') parser.add_argument('batch_name', type=str, help='batch name (case prefix) to search for') -parser.add_argument('--copy_cplex', '-c', type=int, default=0, - help='Which cplex.opt file to copy (or 0 for none)') +parser.add_argument( + '--copy_solver_settings', '-c', action='store_true', + help='Copy the solver settings file used by this run from the repo to the run folder', +) parser.add_argument('--copy_srun_template', '-s', action='store_true', help='Copy current srun_template.sh to sbatch file') parser.add_argument('--force', '-f', action='store_true', help='Proceed without double-checking') parser.add_argument('--more_copyfiles', '-m', type=str, default='', - help=',-delimited list of additional files to copy from reeds_path') + help=',-delimited list of additional relative filepaths to copy from reeds_path') +parser.add_argument('--copy_reeds', '-r', action='store_true', + help='Copy the reeds/ model folder from the repo to the run') parser.add_argument('--include_finished', '-i', action='store_true', help='Also restart finished runs (e.g. to redo postprocessing)') args = parser.parse_args() batch_name = args.batch_name -copy_cplex = args.copy_cplex +copy_solver_settings = args.copy_solver_settings copy_srun_template = args.copy_srun_template force = args.force more_copyfiles = [i for i in args.more_copyfiles.split(',') if len(i)] +copy_reeds = args.copy_reeds include_finished = args.include_finished # #%% Inputs for debugging # batch_name = 'v20231113_yamM0' -# copy_cplex = 1 +# copy_solver_settings = True # copy_srun_template = True # force = True -# more_copyfiles = ['e_report.gms'] +# more_copyfiles = ['report.gms'] +# copy_reeds = False # include_finished = False ###### Procedure #%% Shared parameters -reeds_path = os.path.dirname(os.path.abspath(__file__)) +reeds_path = reeds.io.reeds_path #%% Get all runs dictruns = get_run_status(reeds_path, batch_name) @@ -63,18 +72,9 @@ quit() -#%% Get the cplex file to copy -if copy_cplex: - if copy_cplex == 1: - cplex_file = os.path.join(reeds_path,'cplex.opt') - else: - cplex_file = os.path.join(reeds_path,f'cplex.op{copy_cplex}') -else: - cplex_file = None - #%% Copy the header from the srun_template.sh file if desired if copy_srun_template: - srun_template = os.path.join(reeds_path,'srun_template.sh') + srun_template = os.path.join(reeds_path,'reeds','hpc','srun_template.sh') writelines_srun = list() with open(srun_template, 'r') as f: for line in f: @@ -86,13 +86,18 @@ for case in runs_failed: casename = os.path.basename(case) - #%% Copy the cplex file if desired - if copy_cplex: - shutil.copy(cplex_file, os.path.join(case,'')) + #%% Copy the solver settings file if desired + if copy_solver_settings: + fpath_settings = Path( + reeds.io.reeds_path, 'reeds', 'solver', reeds.io.get_optfile(case) + ) + shutil.copy(fpath_settings, os.path.join(case,'')) #%% Copy additional files if desired for f in more_copyfiles: - shutil.copy(os.path.join(reeds_path,f), os.path.join(case,f)) + shutil.copy(f, Path(case, f)) + if copy_reeds: + shutil.copytree(Path(reeds_path, 'reeds'), Path(case, 'reeds'), dirs_exist_ok=True) #%% Make a backup copy of the original bash and sbatch scripts callfile = os.path.join(case,f'call_{casename}.sh') @@ -106,14 +111,16 @@ if any([os.path.basename(i).startswith('report') for i in lstfiles]): restart_tag = '# Output processing' elif len(lstfiles) < 2: - # If there is only 1 lst file, then it is an environment.csv so the run failed during inputs processing + # If there is only 1 lst file, then it is an environment.csv, + # so the run failed during inputs processing restart_tag = '# Input processing' elif len(lstfiles) == 2: - # If there are only 2 lst files, then one of them will be environment.csv and the other will be 1_inputs.lst so the run failed during the model compilation + # If there are only 2 lst files, then one of them will be environment.csv and + # the other will be 1_inputs.lst, so the run failed during the model compilation restart_tag = '# Compile model' else: # Drop environment and inputs .lst files - lstfiles = [l for l in lstfiles if ("environment.csv" not in l) and ('1_Inputs.lst' not in l)] + lstfiles = [i for i in lstfiles if ("environment.csv" not in i) and ('1_Inputs.lst' not in i)] lastfile = lstfiles[-1] restart_year = int(os.path.splitext(lastfile)[0].split('_')[-1].split('i')[0]) restart_tag = f'# Year: {restart_year}' diff --git a/runstatus.py b/helpers/runstatus.py similarity index 96% rename from runstatus.py rename to helpers/runstatus.py index 8e44d134..12a11548 100644 --- a/runstatus.py +++ b/helpers/runstatus.py @@ -1,10 +1,14 @@ #%% Imports import os import re +import sys import datetime import subprocess import argparse from glob import glob +from pathlib import Path +sys.path.append(str(Path(__file__).parent.parent)) +import reeds # #%% Inputs for debugging # batch_name = 'v20250812_mcK0' @@ -91,11 +95,8 @@ def get_run_status(reeds_path, batch_name): include_finished = args.include_finished verbose = args.verbose - #%% Shared parameters - reeds_path = os.path.dirname(os.path.abspath(__file__)) - dictruns = get_run_status(reeds_path, batch_name) - #%%### Loop through categories and runs and report their status + dictruns = get_run_status(reeds.io.reeds_path, batch_name) for key, runs in dictruns.items(): text = f'{key}: {len(runs)}' print(f"\n{text}\n{'-'*len(text)}") @@ -116,7 +117,7 @@ def get_run_status(reeds_path, batch_name): ### Get last .lst file lstfiles = sorted(glob(os.path.join(fullcase,'lstfiles','*'))) if any([os.path.basename(i).startswith('report') for i in lstfiles]): - last_lst = 'e_report.gms' + last_lst = 'report.gms' penultimatefile = None else: if len(lstfiles) > 1: @@ -176,7 +177,7 @@ def get_run_status(reeds_path, batch_name): elif "dual objective limit exceeded" in slurm: errortext = "(hit dual obj. limit)" # check if infeasible - elif "d_solveoneyear.gms failed with return code 3" in slurm: + elif "3_solve_oneyear.gms failed with return code 3" in slurm: errortext = "(infeasible)" else: errortext = "" diff --git a/pyproject.toml b/hourlize/pyproject.toml similarity index 100% rename from pyproject.toml rename to hourlize/pyproject.toml diff --git a/hourlize/qaqc/summarize_supply_curves.py b/hourlize/qaqc/summarize_supply_curves.py index 26a7aa87..4522abcf 100644 --- a/hourlize/qaqc/summarize_supply_curves.py +++ b/hourlize/qaqc/summarize_supply_curves.py @@ -93,7 +93,7 @@ def load_raw_supply_curves(rev_paths): today = datetime.datetime.now().strftime("%Y-%m-%d") site.addsitedir(os.path.join(reeds_path)) - import runbatch as rb + import runreeds ## read list of rev_path files rev_paths = [] @@ -105,7 +105,7 @@ def load_raw_supply_curves(rev_paths): sys.exit(1) else: rev_path = pd.read_csv(rp) - rev_path = rb.get_rev_paths(rev_path, resolution) + rev_path = runreeds.get_rev_paths(rev_path, resolution) # subset to base name for sc_path rev_path['sc_folder'] = rev_path['sc_path'].apply(lambda row: os.path.basename(row)) # subset to relevant columns and techs diff --git a/requirements_dev.txt b/hourlize/requirements_dev.txt similarity index 100% rename from requirements_dev.txt rename to hourlize/requirements_dev.txt diff --git a/inputs/national_generation/README.md b/inputs/national_generation/README.md index 566df849..abae1d86 100644 --- a/inputs/national_generation/README.md +++ b/inputs/national_generation/README.md @@ -43,7 +43,7 @@ For new gas plants, this is the code implementation: 1. `inputs/scalars.csv` - `caa_gas_max_cf` = 0.40. This is the maximum capacity factor that new gas plants (CCs or CTs) can operate at without being regulated under Clean Air Act, Section 111, expressed as a fraction. -2. `c_supplymodel.gms` - `eq_caa_max_cf` enforces the maximum capacity factor for new gas plants. +2. `c_model.gms` - `eq_caa_max_cf` enforces the maximum capacity factor for new gas plants. For existing coal plants, this is the code implementation: @@ -57,7 +57,7 @@ For existing coal plants, this is the code implementation: This is the emissions rate (metric tons CO2 per MWh) equivalent to average emissions from a new coal-CCS plant, assuming 90% capture rate. The emissions rate from a new coal-CCS plant in ReEDS is 0.051956 metric tons CO2 per MWh (see `emit_rate` parameter) which assumes 95% capture. For 90% capture, the emissions rate is double that or 0.1039 metric tons CO2 per MWh, which we use as the standard. -2. `input_processing/WriteHintage.py` +2. `reeds/input_processing/WriteHintage.py` - Coal plants are binned at the unit level if `GSw_Clean_Air_Act=1` so that each coal unit can independently choose to retire or upgrade. - Coal plants maintain their exogenous retirement assumption, except after 2032, when the Clean Air Act regulations begin and coal can retire endogenously. For example, if the NEMS data states that a plant will retire in 2029, we maintain that assumption and that plant will retire in 2029. @@ -68,7 +68,7 @@ For existing coal plants, this is the code implementation: - if `caa_coal_retire_year` is not in the set of years being modeled for this run, then set it to the first year that is modeled after `caa_coal_retire_year`. For example, if running 5 year solves, then instead of enforcing coal retirement in 2032, it will be enforced in 2035. -4. `c_supplymodel.gms` +4. `c_model.gms` - `eq_caa_rate_standard(st,t)` - this constraint enforces the rate-based emissions standard by setting the maximum coal emissions rate per state under Clean Air Act Section 111. ## Assumptions diff --git a/inputs/scalars.csv b/inputs/scalars.csv index 5822d0b8..690fe284 100644 --- a/inputs/scalars.csv +++ b/inputs/scalars.csv @@ -3,7 +3,7 @@ caa_coal_retire_year,2032,"--year-- year in which coal capacity is forced to eit caa_first_year,2024,"--year-- first year in which the emissions requirements under Clean Air Act, Section 111 are active" caa_rate_emis_standard,0.1039,"--metric tons CO2 per MWh-- rate equivalent to average emissions from a new coal-CCS plant, assuming 90% capture rate. Emissions rate from new coal-CCS plant is 0.051956 metric tons CO2 per MWh (see emit_rate) which assumes 95% capture. For 90% capture, the emissions rate is double that or 0.1039 metric tons CO2 per MWh." caa_gas_max_cf,0.40,"--fraction-- maximum cf that new gas plants (CCs or CTs) can operate at without being regulated under Clean Air Act, Section 111" -co2_capture_incentive_length,12,'--years-- length for the co2 captured incentive to be extended in d_solveoneyear if an upgrade occurs +co2_capture_incentive_length,12,'--years-- length for the co2 captured incentive to be extended in 4_post_solve_adjustments.gms if an upgrade occurs co2_capture_incentive_last_year_,2038,'--year-- last year the co2 captured incentive is available co2_emissions_2022,1539.251,"--million metric tons CO2-- 2022 emissions (used for tax credit phaseout)" coal_fom_adj,97.05,"--2004$/MW-year-- FOM cost annual escalation factor for coal. The product of this number and the coal plant's age is added to the plant's FOM cost. See page 17 of Assumptions to the Annual Energy Outlook 2025: Electricity Market Module." diff --git a/postprocessing/air_quality/health_damage_calculations.py b/postprocessing/air_quality/health_damage_calculations.py index 7f4c9a84..01145251 100644 --- a/postprocessing/air_quality/health_damage_calculations.py +++ b/postprocessing/air_quality/health_damage_calculations.py @@ -1,6 +1,6 @@ ''' This script can be run in one of two ways: -1. called automatically from runbatch.py as part of a ReEDS run, +1. called automatically from runreeds.py as part of a ReEDS run, in which case a single case folder is passed to the script 2. call directly to post-process a set of completed runs, with a csv file diff --git a/postprocessing/bokehpivot/in/reeds2/process_style.csv b/postprocessing/bokehpivot/in/reeds2/process_style.csv index 1d12ac8f..61269c26 100644 --- a/postprocessing/bokehpivot/in/reeds2/process_style.csv +++ b/postprocessing/bokehpivot/in/reeds2/process_style.csv @@ -15,16 +15,16 @@ input_processing/transmission.py,#7B4173 input_processing/outage_rates.py,#A55194 input_processing/hourly_repperiods.py,#CE6DBD input_processing/check_inputs.py,#DE9ED6 -createmodel.gms,#31A354 -d_solveoneyear.gms,#843C39 +a_createmodel.gms,#31A354 +3_solve_oneyear.gms,#843C39 solver/barrier,#AD494A solver/crossover,#D6616B solver/remainder,#E7969C -reeds_augur/prep_data.py,#3182BD -reeds_augur/stress_periods.py,#6BAED6 -reeds_augur/capacity_credit.py,#9ECAE1 +ra/prep_data.py,#3182BD +ra/stress_periods.py,#6BAED6 +ra/capacity_credit.py,#9ECAE1 reeds2pras,#E6550D pras,#FD8D3C -e_report.gms,#74C476 -e_report_dump.py,#A1D99B +report.gms,#74C476 +report_dump.py,#A1D99B retail_rate_calculations.py,#C7E9C0 diff --git a/postprocessing/bokehpivot/reeds2.py b/postprocessing/bokehpivot/reeds2.py index 7d8d0da4..28d3a237 100644 --- a/postprocessing/bokehpivot/reeds2.py +++ b/postprocessing/bokehpivot/reeds2.py @@ -1068,14 +1068,14 @@ def rgba2hex(rgba): def pre_runtime(dictin, **kw): """ ### Use the code below to redefine the colormap when new scripts are added - augur_start = df.index.tolist().index('ReEDS_Augur/prep_data.py') + ra_start = df.index.tolist().index('ra/prep_data.py') for i, row in enumerate(df.index): - if i < augur_start: + if i < ra_start: colors[row.lower()] = rgba2hex(plt.cm.tab20b(i)) - elif i < augur_start + 20: - colors[row.lower()] = rgba2hex(plt.cm.tab20c(i-augur_start)) + elif i < ra_start + 20: + colors[row.lower()] = rgba2hex(plt.cm.tab20c(i-ra_start)) else: - colors[row.lower()] = rgba2hex(plt.cm.tab20(i-augur_start-20)) + colors[row.lower()] = rgba2hex(plt.cm.tab20(i-ra_start-20)) """ df = dictin['runtime'].copy() diff --git a/postprocessing/cleanup_files.py b/postprocessing/cleanup_files.py index 0e94b9e6..0ba36c28 100644 --- a/postprocessing/cleanup_files.py +++ b/postprocessing/cleanup_files.py @@ -43,16 +43,15 @@ ], ## Large output files. Can be regenerated without rerunning the case. 3: [ - os.path.join('outputs', 'Augur_plots'), os.path.join('outputs', 'hourly'), os.path.join('outputs', 'figures'), ], ## Largest output files. Would need to rerun the case to regenerate. 4: [ 'g00files', - 'ReEDS_Augur', + 'handoff', ## The following regex matches the rep_{casename}.gdx file written by - ## e_report.gms (which contains the same data as outputs.h5) + ## report.gms (which contains the same data as outputs.h5) os.path.join('outputs', '^rep_.*\.gdx$'), ] } diff --git a/raw_value_streams.py b/postprocessing/raw_value_streams.py similarity index 99% rename from raw_value_streams.py rename to postprocessing/raw_value_streams.py index d2f95961..7ed4b033 100644 --- a/raw_value_streams.py +++ b/postprocessing/raw_value_streams.py @@ -3,8 +3,8 @@ from GAMS gdx solution file to produce value streams for the variables of the model. ''' -import pandas as pd import gdxpds +import pandas as pd import subprocess from datetime import datetime import logging @@ -250,4 +250,4 @@ def get_df_symbols(dfs, symbols): df_syms = pd.concat(df_syms).reset_index(drop=True) for col in ['sym_name','sym_set']: df_syms[col] = df_syms[col].str.lower() - return df_syms \ No newline at end of file + return df_syms diff --git a/postprocessing/reValue/reValue.py b/postprocessing/reValue/reValue.py index b7062584..064376b2 100644 --- a/postprocessing/reValue/reValue.py +++ b/postprocessing/reValue/reValue.py @@ -75,39 +75,39 @@ def get_prices(): df_pq_rm_adj = df_pq_rm.rename(columns={'h':'season'}) df_seas_h_map = df_hmap[['season','h']].drop_duplicates() if res_marg_style == 'max_net_load_2012': - #Read in net_load_2012 of the appropriate ReEDS Augur file. + #Read in net_load_2012 of the appropriate capacity credit file. #The file we want is for the highest year that is lower than r['year'] #TODO: Perhaps we should use the file that has the same year as r['year'], #depending on if it represents the system of that year better. - aug_files = os.listdir(f'{reeds_run_path}/ReEDS_Augur/augur_data') - aug_file_yrs = [f for f in aug_files if 'ReEDS_Augur_' in f] - yrs = [int(f.replace('ReEDS_Augur_','').replace('.gdx','')) for f in aug_file_yrs] + aug_files = os.listdir(f'{reeds_run_path}/handoff/reeds_data') + aug_file_yrs = [f for f in aug_files if 'ccdata_' in f] + yrs = [int(f.replace('ccdata_','').replace('.gdx','')) for f in aug_file_yrs] yrs_less = [y for y in yrs if y < year] max_yr = max(yrs_less) - df_aug = gdxpds.to_dataframe(f'{reeds_run_path}/ReEDS_Augur/augur_data/ReEDS_Augur_{max_yr}.gdx', + df_ra = gdxpds.to_dataframe(f'{reeds_run_path}/handoff/reeds_data/ccdata_{max_yr}.gdx', 'net_load_2012', old_interface=False) - if int(df_aug['t'][0]) != year: - sys.exit(f'ERROR: Augur year ({int(df_aug["t"][0])}) does not match current scenario year ({year})') - df_aug = df_aug.sort_values('Value', ascending=False) - df_aug_top = df_aug.groupby(['ccreg','ccseason'], as_index=False).head(netload_num_hrs).copy() - df_aug_top = df_aug_top.rename(columns={'ccseason':'season'}) + if int(df_ra['t'][0]) != year: + raise ValueError(f'RA year ({int(df_ra["t"][0])}) does not match current scenario year ({year})') + df_ra = df_ra.sort_values('Value', ascending=False) + df_ra_top = df_ra.groupby(['ccreg','ccseason'], as_index=False).head(netload_num_hrs).copy() + df_ra_top = df_ra_top.rename(columns={'ccseason':'season'}) if netload_time_style == 'hour': - df_aug_top = df_aug_top[['ccreg','season','hour']].copy() - df_aug_top['hour'] = df_aug_top['hour'].astype(int) + df_ra_top = df_ra_top[['ccreg','season','hour']].copy() + df_ra_top['hour'] = df_ra_top['hour'].astype(int) #Convert the seasonal price to an hourly price over the set of hours assigned to that season (for that ba) df_pq_rm_adj['price'] = df_pq_rm_adj['price'] / netload_num_hrs #Restrict to prices only and add ccreg column df_p_rm = df_pq_rm_adj[['reeds_ba','season','price']].merge(df_ba_cc_map, on='reeds_ba', how='left') #Merge max net load hours into df_p_rm (which will duplicate rows if there are multiple timeslices in a ba/season) - df_p_rm = df_p_rm.merge(df_aug_top, on=['ccreg','season'], how='left') + df_p_rm = df_p_rm.merge(df_ra_top, on=['ccreg','season'], how='left') #Hour is one-indexed. Re-index to the 2012 set of 8760 hours (43801 to 52560) df_p_rm_h = df_p_rm.pivot_table(index=['hour'], columns='reeds_ba', values='price') df_p_rm_h = df_p_rm_h.reindex(range(43801,52561)).reset_index(drop=True) elif netload_time_style == 'timeslice': #We assign reserve margin prices to the entire timeslice(s) that contain the top net load hour(s) of each season - df_aug_top = df_aug_top[['ccreg','season','h']].drop_duplicates() + df_ra_top = df_ra_top[['ccreg','season','h']].drop_duplicates() #Find number of total hours that we're mapping prices to in each season - df_seas_hrs = df_aug_top.merge(df_h_num_hrs, on=['h'], how='left') + df_seas_hrs = df_ra_top.merge(df_h_num_hrs, on=['h'], how='left') df_seas_hrs = df_seas_hrs.groupby(['ccreg','season'], as_index=False)['num_hrs'].sum() #Add ccreg and num_hrs columns to df_pq_rm_adj df_pq_rm_adj = df_pq_rm_adj.merge(df_ba_cc_map, on='reeds_ba', how='left') @@ -116,7 +116,7 @@ def get_prices(): df_pq_rm_adj['price'] = df_pq_rm_adj['price'] / df_pq_rm_adj['num_hrs'] #Merge in the timeslices to which we'll be mapping prices. This might duplicate rows if there are #multiple timeslices in a ba/season (which is possible if netload_num_hrs is greater than 1) - df_pq_rm_adj = df_pq_rm_adj.merge(df_aug_top, on=['ccreg','season'], how='left') + df_pq_rm_adj = df_pq_rm_adj.merge(df_ra_top, on=['ccreg','season'], how='left') #Isolate prices with h as first column and reeds_ba as other columns df_p_rm = df_pq_rm_adj.pivot_table(index=['h'], columns='reeds_ba', values='price').reset_index() #Merge with df_hmap, duplicating prices across all hours of the chosen timeslices. diff --git a/postprocessing/retail_rate_module/ITC-PTC_expenditures.py b/postprocessing/retail_rate_module/ITC-PTC_expenditures.py index e906b833..3c94bef0 100644 --- a/postprocessing/retail_rate_module/ITC-PTC_expenditures.py +++ b/postprocessing/retail_rate_module/ITC-PTC_expenditures.py @@ -6,9 +6,9 @@ desired ReEDS and {batch}_{case} run directories * Outputs are saved to ReEDS/runs/{batch}_{case}/outputs/ * To use for outputs generated before PR #527 (2cec69214baddf817b276a89e3f4072ab76cf2c9), - add the following lines to the start of e_report.gms and re-run (otherwise geothermal + add the following lines to the start of report.gms and re-run (otherwise geothermal won't be included) - e_report.gms and e_report_dump.py: + report.gms and report_dump.py: cost_cap_fin_mult_noITC(i,r,t)$geo(i) = cost_cap_fin_mult_noITC("geothermal",r,t) cost_cap_fin_mult_no_credits(i,r,t)$geo(i) = cost_cap_fin_mult_no_credits("geothermal",r,t) * Caveats diff --git a/postprocessing/retail_rate_module/retail_rate_calculations.py b/postprocessing/retail_rate_module/retail_rate_calculations.py index c8365b99..f29c6422 100644 --- a/postprocessing/retail_rate_module/retail_rate_calculations.py +++ b/postprocessing/retail_rate_module/retail_rate_calculations.py @@ -4,9 +4,9 @@ import argparse import datetime import itertools +import gdxpds import pandas as pd import numpy as np -import gdxpds import os import sys import urllib @@ -462,7 +462,7 @@ def main(run_dir, inputpath='inputs.csv', write=True, verbose=0): 'r':'receiving_region', 't':'t', 'Value':'expenditure_flow', 'Dim1':'receiving_region','Dim2':'t','Val':'expenditure_flow'}) ) - ### According to e_report.gms, all international flows are load to/from Canada + ### According to report.gms, all international flows are load to/from Canada # (not capacity, reserves, rps, or Mexico) state_international_flows['price_type'] = 'load' state_international_flows['sending_state'] = 'Canadian Imports' diff --git a/run_pcm.py b/postprocessing/run_pcm.py similarity index 93% rename from run_pcm.py rename to postprocessing/run_pcm.py index fe035579..1186f1c1 100644 --- a/run_pcm.py +++ b/postprocessing/run_pcm.py @@ -1,19 +1,18 @@ # %% Imports import os -import sys import subprocess import argparse import json from glob import glob import gdxpds import pandas as pd +from pathlib import Path ## Local imports import reeds -import e_report_dump -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), 'input_processing'))) -import hourly_repperiods -import hourly_writetimeseries +from reeds.input_processing import hourly_repperiods +from reeds.input_processing import hourly_writetimeseries +from reeds.core.terminus.report_dump import write_dfdict # %% Inferred inputs @@ -64,7 +63,7 @@ def solvestring_pcm( savefile = f"pcm_{label}_{batch_case}_{t}i{iteration}" _stress_year = f"{t}i{iteration}" if stress_year in ['keep', 'default'] else stress_year out = ( - "gams d_solvepcm.gms" + f"gams {Path('reeds','core','solve_pcm','solve_pcm.gms')}" + (" license=gamslice.txt" if hpc else '') + f" o={os.path.join('lstfiles', f'{savefile}.lst')}" + f" r={os.path.join('g00files', restartfile)}" @@ -80,7 +79,7 @@ def solvestring_pcm( [ f" --{s}={sw[s]}" for s in [ - 'GSw_SkipAugurYear', + 'GSw_SkipRAyear', 'GSw_HourlyType', 'GSw_HourlyWrapLevel', 'GSw_ClimateWater', @@ -107,7 +106,7 @@ def solvestring_pcm( def pcm_report_string(batch_case, sw, t, iteration=0, hpc=0, label=''): savefile = f"pcm_{label}_{batch_case}_{t}i{iteration}" out = ( - "gams e_report.gms" + f"gams {Path('reeds','core','terminus','report.gms')}" + (' license=gamslice.txt' if hpc else '') + f" o={os.path.join('lstfiles', f'report_pcm_{label}_{t}_{batch_case}.lst')}" + f" r={os.path.join('g00files', savefile)}" @@ -124,11 +123,11 @@ def submit_job(casepath, command_string, jobname='', joblabel='', bigmem=0): """ Create a slurm job submission script for `command_string` at `casepath`, then submit it. - Uses the slurm settings from {reeds_path}/srun_template.sh. + Uses the slurm settings from {reeds_path}/reeds/hpc/srun_template.sh. """ ### Get the SLURM boilerplate commands_header, commands_sbatch, commands_other = [], [], [] - with open(os.path.join(reeds_path, 'srun_template.sh'), 'r') as f: + with open(os.path.join(reeds_path,'reeds','hpc','srun_template.sh'), 'r') as f: for line in f: if bigmem and ('--mem=' in line): line = '#SBATCH --mem=500000' @@ -248,7 +247,7 @@ def main(casepath, t, switch_mods=switch_mods_default, label='', overwrite=False ### Run GAMS LP result = subprocess.run(cmd_gams, shell=True) if result.returncode: - raise Exception(f'd_solvepcm.gms failed with return code {result.returncode}') + raise Exception(f'solve_pcm.gms failed with return code {result.returncode}') # %% Dump results to gdx cmd_report = pcm_report_string( @@ -263,12 +262,12 @@ def main(casepath, t, switch_mods=switch_mods_default, label='', overwrite=False result = subprocess.run(cmd_report, shell=True) if result.returncode: - raise Exception(f'e_report.gms failed with return code {result.returncode}') + raise Exception(f'report.gms failed with return code {result.returncode}') # %% Dump gdx to h5 ## Get new file names if applicable dfparams = pd.read_csv( - os.path.join(casepath, "e_report_params.csv"), + os.path.join(casepath, 'reeds', 'core', 'terminus', 'report_params.csv'), comment="#", index_col="param", ) @@ -284,7 +283,7 @@ def main(casepath, t, switch_mods=switch_mods_default, label='', overwrite=False outputs_path = os.path.join(casepath, 'outputs', f'pcm_{label}_{_t}') os.makedirs(outputs_path, exist_ok=True) - e_report_dump.write_dfdict( + write_dfdict( dfdict=dict_out, outputs_path=outputs_path, rename=rename, diff --git a/scripts/run_r2x.py b/postprocessing/run_r2x.py similarity index 100% rename from scripts/run_r2x.py rename to postprocessing/run_r2x.py diff --git a/postprocessing/run_reeds2pras.py b/postprocessing/run_reeds2pras.py index bdad0b7e..b5f6d87b 100644 --- a/postprocessing/run_reeds2pras.py +++ b/postprocessing/run_reeds2pras.py @@ -37,7 +37,7 @@ def submit_job( jobname = f'PRAS-{os.path.basename(case)}-{samples}' ## Get the SLURM boilerplate commands_header, commands_sbatch, commands_other = [], [], [] - with open(os.path.join(reeds_path,'srun_template.sh'), 'r') as f: + with open(os.path.join(reeds_path,'reeds','hpc','srun_template.sh'), 'r') as f: for line in f: if line.strip().startswith('#!'): commands_header.append(line.strip()) @@ -99,14 +99,11 @@ def main( Run prep_data, ReEDS2PRAS, and PRAS as necessary. If running PRAS, append the number of samples to the filename. """ - ### Import Augur scripts if repo: site.addsitedir(reeds_path) else: site.addsitedir(case) - import Augur import reeds - import ReEDS_Augur.prep_data as prep_data ### Get the switches, overwriting values as necessary sw = reeds.io.get_switches(case) @@ -134,7 +131,7 @@ def main( print(f'Running PRAS for {t}i{iteration}') ### Check if prep_data.py outputs exist; if not, run it - augur_data = os.path.join(case,'ReEDS_Augur','augur_data') + reeds_data = os.path.join(case,'handoff','reeds_data') files_expected = [ f'cap_converter_{t}.csv', f'energy_cap_{t}.csv', @@ -144,13 +141,13 @@ def main( f'pras_vre_gen_{t}.h5', ] if ( - any([not os.path.isfile(os.path.join(augur_data,f)) for f in files_expected]) + any([not os.path.isfile(os.path.join(reeds_data,f)) for f in files_expected]) or overwrite ): - augur_csv, augur_h5 = prep_data.main(t, case) + reeds_csv, reeds_h5 = reeds.resource_adequacy.prep_data.main(t, case) ### Run ReEDS2PRAS - Augur.run_pras( + reeds.resource_adequacy.ra_calcs.run_pras( case, t, iteration=iteration, @@ -189,7 +186,7 @@ def main( parser.add_argument('--samples', '-s', type=int, default=0, help='PRAS samples to run') parser.add_argument('--repo', '-r', action='store_true', - help=('Import Augur scripts from local repo ' + help=('Import RA scripts from local repo ' '(instead of from the case being rerun)')) parser.add_argument('--local', '-l', action='store_true', help='Run locally (not as SLURM job)') @@ -207,7 +204,7 @@ def main( help="Write hourly unit availability by sample from PRAS") parser.add_argument('--switch_mods', '-m', type=json.loads, default=json.dumps({}), help=('Dictionary-formated string of switch arguments for ' - 'Augur.run_pras(). Use single quotes outside the dictionary and ' + 'ra_calcs.run_pras(). Use single quotes outside the dictionary and ' 'double quotes for keys, as in:\n' '`-s \'{"pras_seed":0}\'`')) diff --git a/postprocessing/tableau/tableau_viz_suite.py b/postprocessing/tableau/tableau_viz_suite.py index 96ff771b..36ce1b86 100644 --- a/postprocessing/tableau/tableau_viz_suite.py +++ b/postprocessing/tableau/tableau_viz_suite.py @@ -417,7 +417,7 @@ def calc_peakload( ).rename(columns=int) dictout = {} - level_map = reeds.output_calc.get_level_map(case) + level_map = reeds.results.get_level_map(case) for level in levels: df_level = df.loc[level,years].stack().rename_axis(['r','t']).rename('Value').reset_index().astype({'t':int}).set_index(['r','t']).squeeze().reset_index() df_level['Spatial Resolution'] = level_map[level] @@ -426,7 +426,7 @@ def calc_peakload( df = pd.concat(dictout.values(), axis=0, ignore_index=True) # # convert MW to GW for national data - df = reeds.output_calc.scale_column(df,**{'scale_factor': 1e-3, 'column':'Value'}) + df = reeds.results.scale_column(df,**{'scale_factor': 1e-3, 'column':'Value'}) return df @@ -477,7 +477,7 @@ def calc_transmission_map(case,level='transgrp'): trans_total_new = pd.concat([trans_total_new, tran_new], axis=0) # convert MW to GW - trans_total_new = reeds.output_calc.scale_column(trans_total_new,**{'scale_factor': 1e-3, 'column':'Value'}) + trans_total_new = reeds.results.scale_column(trans_total_new,**{'scale_factor': 1e-3, 'column':'Value'}) return trans_total_new @@ -602,7 +602,7 @@ def export(dictin,output_dir): create_scenarios_csv(output_dir,cases) # Grab clean display names for the levels - level_map = reeds.output_calc.get_level_map(cases[basecase]) + level_map = reeds.results.get_level_map(cases[basecase]) # import some key inputs from ReEDS dictin_sw = {case: reeds.io.get_switches(cases[case]) for case in cases} @@ -630,7 +630,7 @@ def export(dictin,output_dir): dictin_cap = {} metric = 'Capacity (GW)' for case in tqdm(cases, desc=metric): - dictin_cap[case] = reeds.output_calc.calc_cap(cases[case]) + dictin_cap[case] = reeds.results.calc_cap(cases[case]) dictin_cap[case] = reformat(dictin_cap[case],case,metric,years[case]) dictin_cap = set_zero_values(dictin_cap) dictin[metric] = dictin_cap @@ -639,7 +639,7 @@ def export(dictin,output_dir): dictin_gen = {} metric = 'Generation (TWh)' for case in tqdm(cases, desc=metric): - dictin_gen[case] = reeds.output_calc.calc_gen(cases[case]) + dictin_gen[case] = reeds.results.calc_gen(cases[case]) dictin_gen[case] = reformat(dictin_gen[case],case,metric,years[case]) dictin_gen = set_zero_values(dictin_gen) dictin[metric] = dictin_gen @@ -648,7 +648,7 @@ def export(dictin,output_dir): dictin_emissions = {} metric = 'Emissions (million metric tonnes)' for case in tqdm(cases, desc=metric): - dictin_emissions[case] = reeds.output_calc.calc_emissions(cases[case]) + dictin_emissions[case] = reeds.results.calc_emissions(cases[case]) dictin_emissions[case] = dictin_emissions[case].loc[dictin_emissions[case]['e'] != 'H2'] # remove hydrogen emissions from the dataframe dictin_emissions[case] = reformat(dictin_emissions[case],case,metric,years[case]) dictin[metric] = dictin_emissions @@ -665,7 +665,7 @@ def export(dictin,output_dir): dictin_annualload = {} metric = 'Annual End-Use Electricity Demand (TWh)' for case in tqdm(cases, desc=metric): - dictin_annualload[case] = reeds.output_calc.calc_annualload(cases[case],dictin_scalars[case]) + dictin_annualload[case] = reeds.results.calc_annualload(cases[case],dictin_scalars[case]) dictin_annualload[case] = reformat(dictin_annualload[case],case,metric,years[case]) dictin[metric] = dictin_annualload @@ -673,7 +673,7 @@ def export(dictin,output_dir): dictin_systemcost = {} metric = 'System Cost (billion $)' for case in tqdm(cases, desc=metric): - dictin_systemcost[case] = reeds.output_calc.calc_systemcost(cases[case],group_r=False,drop_zeros=False).rename(columns={'year':'t','Discounted Cost (Bil $)':'Value'}) + dictin_systemcost[case] = reeds.results.calc_systemcost(cases[case],group_r=False,drop_zeros=False).rename(columns={'year':'t','Discounted Cost (Bil $)':'Value'}) del dictin_systemcost[case]['Cost (Bil $)'] dictin_systemcost[case].cost_cat = dictin_systemcost[case].cost_cat.map(lambda x: output_formatting['cost_cat_map'].get(x,x)) dictin_systemcost[case] = reformat(dictin_systemcost[case],case,metric,years[case]) @@ -703,7 +703,7 @@ def export(dictin,output_dir): metric = 'Transmission Capacity (TW-miles)' for case in tqdm(cases, desc=metric): # pull the total installed transmission capacity from ReEDS outputs - dictin_trans_total[case], _ = reeds.output_calc.calc_transmission_capacity(cases[case],levels=['transgrp']) + dictin_trans_total[case], _ = reeds.results.calc_transmission_capacity(cases[case],levels=['transgrp']) tran_total = dictin_trans_total[case] tran_total['Measure'] = 'Total installed' # make one df with the new capacity @@ -726,7 +726,7 @@ def export(dictin,output_dir): dictin_h2prod = {} metric = 'Hydrogen Production (million metric tonnes)' for case in tqdm(cases, desc=metric): - dictin_h2prod[case] = reeds.output_calc.calc_h2prod(cases[case]) + dictin_h2prod[case] = reeds.results.calc_h2prod(cases[case]) dictin_h2prod[case] = reformat(dictin_h2prod[case],case,metric,years[case]) dictin[metric] = dictin_h2prod @@ -734,7 +734,7 @@ def export(dictin,output_dir): dictin_sited_load = {} metric = 'Load Site Capacity (MW)' for case in tqdm(cases, desc=metric): - dictin_sited_load[case] = reeds.output_calc.calc_sited_load(cases[case]) + dictin_sited_load[case] = reeds.results.calc_sited_load(cases[case]) dictin_sited_load[case] = reformat(dictin_sited_load[case],case,metric,years[case]) dictin[metric] = dictin_sited_load diff --git a/postprocessing/uncertainty_plots.py b/postprocessing/uncertainty_plots.py index d8e391d7..84f4a0f4 100644 --- a/postprocessing/uncertainty_plots.py +++ b/postprocessing/uncertainty_plots.py @@ -21,7 +21,6 @@ import matplotlib.pyplot as plt from matplotlib.lines import Line2D from matplotlib.ticker import FuncFormatter -import pptx from pptx.util import Inches import io import seaborn as sns @@ -486,7 +485,7 @@ def __init__(self, cases_dict: dict): # --- Fetch methods for output data --- self.OUT_FETCH_METHODS = { - # Naming as in e_report_params.csv + # Naming as in report_params.csv "cap_out": self._fetch_cap_out, "gen_ann": self._fetch_gen_ann, "tran_mi_out_detail": self._fetch_tran_mi_out_detail, @@ -626,7 +625,7 @@ def _fetch_tran_mi_out_detail(self, case: str) -> pd.DataFrame: ) df_tech_trans = ( - reeds.output_calc.calc_reinforcement_spur_capacity_miles(self.cases_dict[case]) + reeds.results.calc_reinforcement_spur_capacity_miles(self.cases_dict[case]) ) df_tech_trans['rr'] = df_tech_trans['r'] # Duplicate region data # Convert "Trans (GW-mi)" to "Trans (TW-mi)" @@ -681,7 +680,7 @@ def _fetch_npv_r(self, case: str) -> pd.DataFrame: Returns: pd.DataFrame: Columns ['year', 'r', 'cost_cat', 'Cost (Bil $)', 'Discounted Cost (Bil $)'] """ - return reeds.output_calc.calc_systemcost(self.cases_dict[case]) + return reeds.results.calc_systemcost(self.cases_dict[case]) ### =========================================================================== diff --git a/valuestreams.py b/postprocessing/valuestreams.py similarity index 100% rename from valuestreams.py rename to postprocessing/valuestreams.py diff --git a/preprocessing/casemaker.py b/preprocessing/casemaker.py index c70b8d72..a793a152 100644 --- a/preprocessing/casemaker.py +++ b/preprocessing/casemaker.py @@ -191,9 +191,9 @@ def main(casematrix_path=None, batchname=None): ### Write it dfcases.to_csv(os.path.join(reeds_path,f'cases_{_batchname}.csv')) - ### Make sure the switch names and values pass the checks in runbatch.py + ### Make sure the switch names and values pass the checks in runreeds.py sep = ';' if os.name == 'posix' else '&&' - cmd = f"cd {reeds_path} {sep} python runbatch.py -b test -c {_batchname} -r 4 -p 1 --dryrun" + cmd = f"cd {reeds_path} {sep} python runreeds.py -b test -c {_batchname} -r 4 -p 1 --dryrun" result = subprocess.run(cmd, shell=True, capture_output=True) err = result.stderr.decode() if len(err): diff --git a/preprocessing/casematrix_example.yaml b/preprocessing/casematrix_example.yaml index 664f8639..bf676e10 100644 --- a/preprocessing/casematrix_example.yaml +++ b/preprocessing/casematrix_example.yaml @@ -4,7 +4,7 @@ ### Shared settings are applied to all cases. ### Any settings not specified use the defaults from cases.csv. shared: - yearset: 2010_2015_2020_2025_2030_2035_2040_2045_2050 + yearset: 2010..2050..5 ################## ### Dimensions ### diff --git a/reeds/__init__.py b/reeds/__init__.py index e6aaa6e1..ff723dfd 100644 --- a/reeds/__init__.py +++ b/reeds/__init__.py @@ -1,15 +1,19 @@ +from . import input_processing as input_processing +from . import core as core +from . import hpc as hpc +from . import resource_adequacy as resource_adequacy + from . import checks as checks from . import financials as financials from . import inputs as inputs from . import io as io from . import log as log -from . import output_calc as output_calc from . import plots as plots from . import prasplots as prasplots -from . import ra as ra from . import reedsplots as reedsplots from . import remote as remote from . import report_utils as report_utils +from . import results as results from . import spatial as spatial from . import techs as techs from . import timeseries as timeseries diff --git a/reeds/core/setup/a_createmodel.gms b/reeds/core/setup/a_createmodel.gms new file mode 100644 index 00000000..9250f30e --- /dev/null +++ b/reeds/core/setup/a_createmodel.gms @@ -0,0 +1,10 @@ +$setglobal ds \ +$ifthen.unix %system.filesys% == UNIX +$setglobal ds / +$endif.unix + +$include reeds%ds%core%ds%setup%ds%b_inputs.gms +$include reeds%ds%core%ds%setup%ds%c_model.gms +$include reeds%ds%core%ds%setup%ds%d_objective.gms +$include reeds%ds%core%ds%setup%ds%d_mga.gms +$include reeds%ds%core%ds%setup%ds%e_solveprep.gms diff --git a/b_inputs.gms b/reeds/core/setup/b_inputs.gms similarity index 99% rename from b_inputs.gms rename to reeds/core/setup/b_inputs.gms index f1b09e54..e2a65430 100644 --- a/b_inputs.gms +++ b/reeds/core/setup/b_inputs.gms @@ -63,7 +63,7 @@ alias(dummy,adummy) ; * Following are scalars used to turn on or off various components of the model. * For binary switches, [0] is off and [1] is on. -* These switches are generated from the cases file in runbatch.py. +* These switches are generated from the cases file in runreeds.py. $include inputs_case%ds%gswitches.txt * Extra switches that are defined based on other switches @@ -77,7 +77,7 @@ parameter Sw_Timetype(timetype) "Switch that specifies the type of time method u Sw_Timetype("%timetype%") = 1 ; -* Sw_PCM is always 0 except when running d_solvepcm.gms, where it's set to 1 +* Sw_PCM is always 0 except when running solve_pcm.gms, where it's set to 1 scalar Sw_PCM "Internal switch used when running PCM mode" / 0 / ; * Sw_MGA is always 0 except when running the optimization a second time for MGA, where it's set to 1 scalar Sw_MGA "Internal switch used when running MGA mode" / 0 / ; @@ -91,7 +91,7 @@ scalar retireyear "first year to allow capacity to start retiring" /%GSw_Retire upgradeyear "first year to allow capacity to upgrade" /%GSw_Upgradeyear%/ climateyear "first year to apply climate impacts" /%GSw_ClimateStartYear%/ ; -*** Scalars: copied from inputs/scalars.csv to inputs_case/scalars.txt in runbatch.py +*** Scalars: copied from inputs/scalars.csv to inputs_case/scalars.txt in runreeds.py $include inputs_case%ds%scalars.txt *========================== @@ -233,7 +233,7 @@ $include inputs_case%ds%val_hurdlereg.csv ; * Written by copy_files.py -$include b_sets.gms +$include autocode%ds%b_sets.gms sets *The following two sets: @@ -1425,7 +1425,7 @@ ivt(i,newv,t)$[ord(newv) = ivt_num(i,t)] = yes ; ivt(i,v,t)$[i_water_cooling(i)$Sw_WaterMain] = sum{ii$ctt_i_ii(i,ii), ivt(ii,v,t) } ; -*Also expand ivt_num to water techs for use in Augur +*Also expand ivt_num to water techs for use in resource adequacy calculations ivt_num(i,t)$[i_water_cooling(i)$Sw_WaterMain] = sum{ii$ctt_i_ii(i,ii), ivt_num(ii,t) } ; @@ -1481,7 +1481,7 @@ $onlisting /, * pvf_capital and pvf_onm here are for intertemporal mode. These parameters -* are overwritten for sequential mode in d_solveprep.gms. +* are overwritten for sequential mode in e_solveprep.gms. pvf_capital(t) "--unitless-- present value factor for overnight capital costs" / $offlisting @@ -1512,7 +1512,7 @@ $onlisting /, h2_ptc(i,v,r,allt) "--2004$/kg h2 produced -- incentive on hydrogen production by electrolyzers which purchase Energy Attribute Credits" - h2_ptc_in(i,v,allt) "--2004$/kg h2 produced -- incentive on hydrogen production by electrolyzers which purchase Energy Attribute Credits, this parameter is used to build h2_ptc and is produced in input_processing/calc_financial_inputs.py" + h2_ptc_in(i,v,allt) "--2004$/kg h2 produced -- incentive on hydrogen production by electrolyzers which purchase Energy Attribute Credits, this parameter is used to build h2_ptc" / $offlisting $ondelim @@ -1611,8 +1611,8 @@ $offdelim $onlisting / ; -*created by /input_processing/writecapdat.py -table capnonrsc(i,r,*) "--MW-- raw power capacity data for non-RSC tech created by .\input_processing\writecapdat.py" +*created by reeds/input_processing/writecapdat.py +table capnonrsc(i,r,*) "--MW-- raw power capacity data for non-RSC tech" $offlisting $ondelim $include inputs_case%ds%capnonrsc.csv @@ -1620,9 +1620,9 @@ $offdelim $onlisting ; -*created by /input_processing/writecapdat.py +*created by reeds/input_processing/writecapdat.py $onempty -table capnonrsc_energy(i,r,*) "--MWh-- raw energy capacity data for battery tech created by .\input_processing\writecapdat.py" +table capnonrsc_energy(i,r,*) "--MWh-- raw energy capacity data for battery tech" $offlisting $ondelim $include inputs_case%ds%capnonrsc_energy.csv @@ -1631,9 +1631,9 @@ $onlisting ; $offempty -*created by /input_processing/writecapdat.py +*created by reeds/input_processing/writecapdat.py $onempty -table caprsc(pcat,r,*) "--MW-- raw RSC capacity data, created by .\writecapdat.py" +table caprsc(pcat,r,*) "--MW-- raw RSC capacity data" $offlisting $ondelim $include inputs_case%ds%caprsc.csv @@ -1642,10 +1642,10 @@ $onlisting ; $offempty -*created by /input_processing/writecapdat.py +*created by reeds/input_processing/writecapdat.py * declared over allt to allow for external data files that extend beyond end_year $onempty -table prescribednonrsc(allt,pcat,r,*) "--MW-- raw prescribed capacity data for non-RSC tech created by writecapdat.py" +table prescribednonrsc(allt,pcat,r,*) "--MW-- raw prescribed capacity data for non-RSC tech" $offlisting $ondelim $include inputs_case%ds%prescribed_nonRSC.csv @@ -1655,7 +1655,7 @@ $onlisting $offempty $onempty -table prescribednonrsc_energy(allt,pcat,r,*) "--MWh-- raw prescribed energy capacity data for non-RSC tech created by writecapdat.py" +table prescribednonrsc_energy(allt,pcat,r,*) "--MWh-- raw prescribed energy capacity data for non-RSC tech" $offlisting $ondelim $include inputs_case%ds%prescribed_nonRSC_energy.csv @@ -1664,9 +1664,9 @@ $onlisting ; $offempty -*Created using input_processing\writecapdat.py +*Created using reeds/input_processing/writecapdat.py $onempty -table prescribedrsc(allt,pcat,r,*) "--MW-- raw prescribed capacity data for RSC tech created by .\input_processing\writecapdat.py" +table prescribedrsc(allt,pcat,r,*) "--MW-- raw prescribed capacity data for RSC tech" $offlisting $ondelim $include inputs_case%ds%prescribed_rsc.csv @@ -1700,11 +1700,11 @@ $offempty prescribedrsc(allt,"wind-ofs",r,"value") = prescribed_wind_ofs(r,allt,"capacity") ; -*created by /input_processing/writecapdat.py +*created by reeds/input_processing/writecapdat.py *following does not include wind *Retirements for techs binned by heatrates are handled in hintage_data.csv $onempty -table prescribedretirements(allt,r,i,*) "--MW-- raw prescribed power capacity retirement data for non-RSC, non-heatrate binned tech created by /input_processing/writecapdat.py" +table prescribedretirements(allt,r,i,*) "--MW-- raw prescribed power capacity retirement data for non-RSC, non-heatrate binned tech" $offlisting $ondelim $include inputs_case%ds%retirements.csv @@ -1713,10 +1713,10 @@ $onlisting ; $offempty -*created by /input_processing/writecapdat.py +*created by reeds/input_processing/writecapdat.py *Retirements for techs binned by heatrates are handled in hintage_data.csv $onempty -table prescribedretirements_energy(allt,r,i,*) "--MWh-- raw prescribed energy capacity retirement data for battery tech created by /input_processing/writecapdat.py" +table prescribedretirements_energy(allt,r,i,*) "--MWh-- raw prescribed energy capacity retirement data for battery tech" $offlisting $ondelim $include inputs_case%ds%retirements_energy.csv @@ -1750,8 +1750,8 @@ $include inputs_case%ds%hintage_char.csv $onlisting / ; -*created by /input_processing/writehintage.py -table hintage_data(i,v,r,allt,hintage_char) "table of existing unit characteristics written by writehintage.py" +*created by reeds/input_processing/writehintage.py +table hintage_data(i,v,r,allt,hintage_char) "table of existing unit characteristics" $offlisting $ondelim $include inputs_case%ds%hintage_data.csv @@ -1791,7 +1791,7 @@ if(Sw_Upgrades = 1, ) ; ) ; -*created by /input_processing/writecapdat.py +*created by reeds/input_processing/writecapdat.py parameter binned_capacity(i,v,r,allt) "existing capacity (that is not rsc, but including distpv) binned by heat rates" ; binned_capacity(i,v,r,allt) = hintage_data(i,v,r,allt,"cap") ; @@ -2612,7 +2612,6 @@ m_capacity_exog(i,v,r,t)$[forced_retire(i,r,t)$(not Sw_Clean_Air_Act)$coal_noccs * If, in the last year in which coal must either retire or upgrade, coal is upgraded, we can continue to * use that m_capacity_exog beyond caa_coal_retire_year. But unabated coal plants must not have capacity after caa_coal_retire_year. -* This is expanded in d_solveoneyear.gms. m_capacity_exog(i,v,r,t)$[forced_retire(i,r,t)$coal_noccs(i) $(t.val > caa_coal_retire_year) $(sum{ii$(not forced_retire(ii,r,t)), upgrade_from(ii,i) }) ] = 0 ; @@ -2754,7 +2753,7 @@ co2_captured_incentive(i,v,r,t)$[initv(v)$upgrade(i) $(yeart(t)>=Sw_UpgradeYear) $(yeart(t) <= co2_capture_incentive_last_year_)] = * note we populate the incentive for all years and then trim the incentive for later years -* when the upgrade occurs - this is after the solve statement in d_solveoneyear +* when the upgrade occurs - this is after the solve statement * we also cast this forward based on whether or not a plant was built in that year * but remove the last year for consideration of that below via co2_capture_incentive_last_year_ sum{(ii,vv,tt)$[upgrade_to(i,ii)$newv(vv) @@ -4257,7 +4256,7 @@ $onlisting / ; parameter ilr(i) "--unitless-- inverter loading ratio - used to convert DC capacity to AC capacity for PV" ; -* assign a default value of 1.0 for every technology (used in e_report.gms) +* assign a default value of 1.0 for every technology ilr(i)$[valcap_i(i)] = 1 ; ilr(i)$[upv(i)] = ilr_utility ; ilr(i)$distpv(i) = ilr_dist ; @@ -4606,7 +4605,7 @@ heat_rate(i,v,r,t)$[heat_rate_adj(i,'post2010')$newv(v)] = heat_rate_adj(i,'post parameter fuel_price(i,r,t) "$/MMBtu - fuel prices by technology" ; -* Written by input_processing\fuelcostprep.py +* Written by reeds/input_processing/fuelcostprep.py * declared over allt to allow for external data files that extend beyond end_year table fprice(allt,r,f) "--2004$/MMBtu-- fuel prices by fuel type" $offlisting @@ -4674,7 +4673,7 @@ $endif.climatewater * -- Capacity Factor Adjustments over Time -- *============================================= -*created by /input_processing/writecapdat.py +*created by reeds/input_processing/writecapdat.py parameter cap_hyd_ccseason_adj(i,ccseason,r) "--fraction-- ccseason max capacity adjustment for dispatchable hydro" / $offlisting @@ -5310,7 +5309,7 @@ capture_rate(e,i,v,r,t)$[capture_rate_fuel(i,e)$valcap(i,v,r,t)] capture_rate(e,i,v,r,t)$[upgrade(i)$capture_rate_fuel(i,e)] = round(heat_rate(i,v,r,t) * capture_rate_fuel(i,e),10) ; -* Declare regional emissions rate (used in c_supplymodel, defined in d_solveoneyear) +* Regional emissions rate parameter co2_emit_rate_r(r,t) "--metric tons per MWh-- CO2 emissions rate by ReEDS region, for use in state carbon caps" co2_emit_rate_regional(%GSw_StateCO2ImportLevel%,t) "--metric tons per MWh-- CO2 regional emissions rate, for use in state carbon caps" @@ -5639,7 +5638,7 @@ nat_beta(t)$(not tfirst(t)) = nat_beta_nonenergy ; $endif.gassector -* Written by input_processing\fuelcostprep.py +* Written by reeds/input_processing/fuelcostprep.py * declared over allt to allow for external data files that extend beyond end_year table cd_alpha(allt,cendiv) "--$/MMBtu-- alpha value for natural gas supply curves" $offlisting @@ -6112,7 +6111,7 @@ scalar bio_transport_cost ; * biomass transport cost enter in $ per ton, convert to $ per MMBtu bio_transport_cost = Sw_BioTransportCost / bio_energy_content ; -* get price of cheapest supply curve bin that has resources (needed for Augur) +* get price of cheapest supply curve bin that has resources (needed for resource adequacy calculations) * price includes any transport costs for biomass parameter rep_bio_price_unused(r) "--2004$/MWh-- marginal price of lowest cost available supply curve bin for biofuel" ; rep_bio_price_unused(r)$[sum{usda_region, 1$r_usda(r,usda_region) }] = @@ -6128,7 +6127,7 @@ cost_curt(t)$[yeart(t)>=model_builds_start_yr] = Sw_CurtMarket ; *====================== parameter emit_cap(eall,t) "--metric tons per year-- annual CO2 emissions cap", - yearweight(t) "--unitless-- weights applied to each solve year for the banking and borrowing cap - updated in d_solveprep.gms", + yearweight(t) "--unitless-- weights applied to each solve year for the banking and borrowing cap - updated in e_solveprep.gms", emit_tax(e,r,t) "--$ per metric ton-- tax applied to emissions" ; emit_cap(e,t) = 0 ; @@ -6504,7 +6503,7 @@ allow_cap_up(i,v,r,rscbin,t)$[valcap(i,v,r,t)$cap_cap_up(i,v,r,rscbin,t)$(t.val> allow_ener_up(i,v,r,rscbin,t)$[valcap(i,v,r,t)$cap_ener_up(i,v,r,rscbin,t)$(t.val>=Sw_UpgradeYear)] = yes ; -* Track the initial amount of m_rsc_dat capacity to compare in e_report +* Track the initial amount of m_rsc_dat capacity to compare in report.gms * We adjust upwards by small amounts given potential for infeasibilities * in very tiny amounts and thus track the extent of the adjustments parameter m_rsc_dat_init(r,i,rscbin) "--MW-- Initial amount of resource supply curve capacity to compare with final amounts after adjustments" ; @@ -6589,7 +6588,7 @@ cost_co2_spurline_cap(r,cs,t) = %GSw_CO2_CostAdj% * cost_co2_spurline_cap(r,cs, * Parameter tracking for sequential solve parameter - m_capacity_exog0(i,v,r,t) "--MW-- original value of m_capacity_exog used in d_solveoneyear to make sure upgraded capacity isnt forced into retirement" + m_capacity_exog0(i,v,r,t) "--MW-- original value of m_capacity_exog used to make sure upgraded capacity isnt forced into retirement" z_rep(t) "--$-- objective function value by year" z_rep_inv(t) "--$-- investment component of objective function by year" z_rep_op(t) "--$-- operation component of objective function by year" @@ -6599,10 +6598,10 @@ z_rep_op(t) = 0 ; *================================================================================================ -*== h- and szn-dependent sets and parameters (declared here, populated in d1_temporal_params) === +*== h- and szn-dependent sets and parameters (declared here, populated in 2_temporal_params) === *================================================================================================ -* allh and allszn need to be populated here so they can be used in c_supplymodel and c_supplyobjective +* allh and allszn need to be populated here so they can be used in c_model and d_objective Set allh "all potentially modeled hours" / $offlisting diff --git a/c_supplymodel.gms b/reeds/core/setup/c_model.gms similarity index 99% rename from c_supplymodel.gms rename to reeds/core/setup/c_model.gms index d0a5d102..c6a8b834 100644 --- a/c_supplymodel.gms +++ b/reeds/core/setup/c_model.gms @@ -2381,7 +2381,7 @@ eq_offshore_no_backflow(r,rr,trtype,h,t) * --------------------------------------------------------------------------- * Because EMIT is only evaluated for emit_modeled, the full emissions need to be -* calculated in the e_report rather than relying on the EMIT variable +* calculated in report.gms rather than relying on the EMIT variable eq_emit_accounting(etype,e,r,t)$[emit_modeled(e,r,t)$tmodel(t)].. EMIT(etype,e,r,t) diff --git a/c_mga.gms b/reeds/core/setup/d_mga.gms similarity index 100% rename from c_mga.gms rename to reeds/core/setup/d_mga.gms diff --git a/c_supplyobjective.gms b/reeds/core/setup/d_objective.gms similarity index 100% rename from c_supplyobjective.gms rename to reeds/core/setup/d_objective.gms diff --git a/d_solveprep.gms b/reeds/core/setup/e_solveprep.gms similarity index 99% rename from d_solveprep.gms rename to reeds/core/setup/e_solveprep.gms index dbaa6df9..1d30682b 100644 --- a/d_solveprep.gms +++ b/reeds/core/setup/e_solveprep.gms @@ -135,7 +135,7 @@ $endif.seq $ifthen.intwin ((%timetype%=="int") or (%timetype%=="win")) set - loadset "set used for loading in merged gdx files" / ReEDS_Augur_%startyear%*ReEDS_Augur_%endyear% / + loadset "set used for loading in merged gdx files" / ccdata_%startyear%*ccdata_%endyear% / ; parameter diff --git a/tc_phaseout.py b/reeds/core/solve/1_tc_phaseout.py similarity index 96% rename from tc_phaseout.py rename to reeds/core/solve/1_tc_phaseout.py index eaac63ac..66454f47 100644 --- a/tc_phaseout.py +++ b/reeds/core/solve/1_tc_phaseout.py @@ -14,10 +14,13 @@ ########### #%% IMPORTS import argparse +import gdxpds import pandas as pd import numpy as np -import gdxpds import os +import sys +from pathlib import Path +sys.path.append(str(Path(__file__).parent.parent.parent.parent)) import reeds ########## @@ -208,7 +211,7 @@ def calc_tc_phaseout_mult(year, case, use_historical=use_historical): #%% PROCEDURE if __name__ == '__main__': - parser = argparse.ArgumentParser(description="""Running tc_phaseout.py""") + parser = argparse.ArgumentParser(description="Running 1_tc_phaseout.py") parser.add_argument("year", help="ReEDS solve year", type=int) parser.add_argument("case", help="filepath for ReEDS case") args = parser.parse_args() @@ -221,6 +224,6 @@ def calc_tc_phaseout_mult(year, case, use_historical=use_historical): logpath=os.path.join(case,'gamslog.txt'), ) - print(f'starting tc_phaseout.py for {year}') + print(f'starting 1_tc_phaseout.py for {year}') calc_tc_phaseout_mult(year, case, use_historical=use_historical) - print(f'finished tc_phaseout.py for {year}') + print(f'finished 1_tc_phaseout.py for {year}') diff --git a/d1_financials.gms b/reeds/core/solve/2_financials.gms similarity index 100% rename from d1_financials.gms rename to reeds/core/solve/2_financials.gms diff --git a/d1_temporal_params.gms b/reeds/core/solve/2_temporal_params.gms similarity index 99% rename from d1_temporal_params.gms rename to reeds/core/solve/2_temporal_params.gms index 444c7fc5..68ca1901 100644 --- a/d1_temporal_params.gms +++ b/reeds/core/solve/2_temporal_params.gms @@ -223,7 +223,7 @@ $offdelim $onlisting / ; -* Written by input_processing/hourly_writetimeseries.py +* Written by reeds/input_processing/hourly_writetimeseries.py parameter frac_h_quarter_weights(allh,quarter) "--unitless-- fraction of timeslice associated with each quarter" / $offlisting @@ -456,7 +456,7 @@ cf_in(i,r,h)$[i_water_cooling(i)$Sw_WaterMain] = cf_rsc(i,v,r,allh,t) = 0 ; cf_rsc(i,v,r,h,t)$[cf_in(i,r,h)$cf_tech(i)$valcap(i,v,r,t)] = cf_in(i,r,h) ; -* Written by input_processing/hourly_writetimeseries.py +* Written by reeds/input_processing/hourly_writetimeseries.py $onempty parameter cf_hyd(i,allszn,r,allt) "--fraction-- hydro capacity factors by season and year" / @@ -492,7 +492,7 @@ cf_hyd(i,szn,r,t)$[hydro_d(i)$(yeart(t)>=Sw_ClimateStartYear)] = $endif.climatehydro -*created by /input_processing/writecapdat.py +*created by reeds/input_processing/writecapdat.py parameter cap_hyd_szn_adj(i,allszn,r) "--fraction-- seasonal max capacity adjustment for dispatchable hydro" / $offlisting diff --git a/d_solveallyears.gms b/reeds/core/solve/3_solve_allyears.gms similarity index 96% rename from d_solveallyears.gms rename to reeds/core/solve/3_solve_allyears.gms index 204d1bc5..e06d360b 100644 --- a/d_solveallyears.gms +++ b/reeds/core/solve/3_solve_allyears.gms @@ -37,7 +37,7 @@ $endif.loadref *indicate we're loading data tload(t)$tmodel(t) = yes ; -$gdxin ReEDS_Augur%ds%augur_data%ds%ReEDS_Augur_merged_%niter%.gdx +$gdxin handoff%ds%reeds_data%ds%ccdata_merged_%niter%.gdx $loaddcr cc_old_load2 = cc_old $loaddcr cc_mar_load2 = cc_mar $loaddcr cc_evmc_load2 = cc_evmc @@ -120,7 +120,7 @@ cc_int(i,v,r,szn,t)$[cc_int(i,v,r,szn,t) < 0.001] = 0 ; cc_iter(i,v,r,szn,t,"%niter%")$cc_int(i,v,r,szn,t) = cc_int(i,v,r,szn,t) ; -execute_unload 'ReEDS_Augur%ds%augur_data%ds%curtout_%case%_%niter%.gdx' cc_int ; +execute_unload 'handoff%ds%reeds_data%ds%curtout_%case%_%niter%.gdx' cc_int ; *following line will load in the level values if the switch is enabled *note that this is still within the conditional that we are now past the first iteration @@ -162,4 +162,4 @@ gen_iter(i,v,r,t,"%niter%")$valcap(i,v,r,t) = sum{h, GEN.l(i,v,r,h,t) * hours(h) gen_iter(i,v,r,t,"%niter%")$[vre(i)$valcap(i,v,r,t)] = sum{h, m_cf(i,v,r,h,t) * CAP.l(i,v,r,t) * hours(h) } ; cap_firm_iter(i,v,r,szn,t,"%niter%")$cc_int(i,v,r,szn,t) = cc_int(i,v,r,szn,t) * CAP.l(i,v,r,t) ; cap_firm_iter(i,v,r,szn,t,"%niter%")$storage(i) = sum{sdbin, CAP_SDBIN.l(i,v,r,szn,sdbin,t) * cc_storage(i,sdbin) } ; -cap_energy_firm_iter(i,v,r,szn,t,"%niter%")$storage(i) = sum{sdbin, CAP_SDBIN_ENERGY.l(i,v,r,szn,sdbin,t) } ; \ No newline at end of file +cap_energy_firm_iter(i,v,r,szn,t,"%niter%")$storage(i) = sum{sdbin, CAP_SDBIN_ENERGY.l(i,v,r,szn,sdbin,t) } ; diff --git a/d_solveoneyear.gms b/reeds/core/solve/3_solve_oneyear.gms similarity index 94% rename from d_solveoneyear.gms rename to reeds/core/solve/3_solve_oneyear.gms index b3484a80..dde80524 100644 --- a/d_solveoneyear.gms +++ b/reeds/core/solve/3_solve_oneyear.gms @@ -1,10 +1,10 @@ * Includes these scripts, in order: -* - d1_temporal_params.gms -* - d1_financials.gms +* - 2_temporal_params.gms +* - 2_financials.gms * - * solves the model * -* - d2_post_solve_adjustments.gms -* - d2_varfix.gms -* - d3_data_dump.gms +* - 4_post_solve_adjustments.gms +* - 5_varfix.gms +* - 6_data_dump.gms $setglobal ds \ @@ -30,7 +30,7 @@ $log ' Year: %cur_year%' *** Define the h- and szn-dependent parameters $onMultiR -$include d1_temporal_params.gms +$include reeds%ds%core%ds%solve%ds%2_temporal_params.gms $offMulti @@ -81,7 +81,7 @@ if(Sw_Upgrades = 1, m_capacity_exog(i,v,r,t)$[valcap(i,v,r,t)$sameas(t,"%cur_year%") $(sum{(ii,tt)$[(tt.val <= t.val)$(t.val - tt.val <= Sw_UpgradeLifespan) $valcap(ii,v,r,tt)$upgrade_from(ii,i)], UPGRADES.l(ii,v,r,tt) } ) ] = -* [maximum of] initial capacity recorded in d_solveprep +* [maximum of] initial capacity recorded in e_solveprep max( m_capacity_exog0(i,v,r,t), * -or- capacity of upgrades that have occurred from this i v r t combination sum{(ii,tt)$[(tt.val <= t.val)$(t.val - tt.val <= Sw_UpgradeLifespan) @@ -136,14 +136,13 @@ if(Sw_GrowthPenalties > 0, $endif.post_startyear * Load capacity credit results -$ifthene.tcheck %cur_year%>%GSw_SkipAugurYear% +$ifthene.tcheck %cur_year%>%GSw_SkipRAyear% *indicate we're loading data tload("%cur_year%") = yes ; -*file written by ReEDS_Augur.py * loaddcr = domain check (dc) + overwrite values storage previously (r) -$gdxin ReEDS_Augur%ds%augur_data%ds%ReEDS_Augur_%prev_year%.gdx +$gdxin handoff%ds%reeds_data%ds%ccdata_%prev_year%.gdx $loaddcr cc_old_load = cc_old $loaddcr cc_mar_load = cc_mar $loaddcr cc_evmc_load = cc_evmc @@ -188,7 +187,7 @@ $endif.tcheck *** Calculate financial multipliers * These are calculated here because the ITC phaseout can influence these parameters, * and the timing of the phaseout is not known beforehand. -$include d1_financials.gms +$include reeds%ds%core%ds%solve%ds%2_financials.gms $ifthene %cur_year%==%startyear% @@ -219,7 +218,7 @@ if(Sw_GrowthPenalties > 0, * Write the inputs for debugging and error checks: * Always write data for the first solve year (currently always 2010). -* Overwrites the versions written by d_solveprep.gms and d1_temporal_params.gms. +* Overwrites the versions written by e_solveprep.gms and 2_temporal_params.gms. $ifthene.write %cur_year%=%startyear% execute_unload 'inputs_case%ds%inputs.gdx' ; $endif.write @@ -275,14 +274,14 @@ $endif.mga *** Adjust some parameters based on the solution for this solve year -$include d2_post_solve_adjustments.gms +$include reeds%ds%core%ds%solve%ds%4_post_solve_adjustments.gms *** Fix decision variables to their optimized levels for this solve year tfix("%cur_year%") = yes ; -$include d2_varfix.gms +$include reeds%ds%core%ds%solve%ds%5_varfix.gms *** Dump data used in calculations between solve years -$include d3_data_dump.gms +$include reeds%ds%core%ds%solve%ds%6_data_dump.gms *** Abort if the solver returns an error if (ReEDSmodel.modelStat > 1, diff --git a/d_solvewindow.gms b/reeds/core/solve/3_solve_window.gms similarity index 96% rename from d_solvewindow.gms rename to reeds/core/solve/3_solve_window.gms index 3fa679bf..325ada4f 100644 --- a/d_solvewindow.gms +++ b/reeds/core/solve/3_solve_window.gms @@ -44,7 +44,7 @@ $ifthene.notfirstiter %niter%>0 *indicate we're loading data tload(t)$tmodel(t) = yes ; -$gdxin ReEDS_Augur%ds%augur_data%ds%ReEDS_Augur_merged_%niter%.gdx +$gdxin handoff%ds%reeds_data%ds%ccdata_merged_%niter%.gdx $loaddcr loadset = merged_set_1 $loaddcr cc_old_load2 = cc_old $loaddcr cc_mar_load2 = cc_mar @@ -126,7 +126,7 @@ cc_int(i,v,r,szn,t)$[cc_int(i,v,r,szn,t) < 0.001] = 0 ; cc_iter(i,v,r,szn,t,"%niter%")$cc_int(i,v,r,szn,t) = cc_int(i,v,r,szn,t) ; -execute_unload 'ReEDS_Augur%ds%augur_data%ds%curtout_%case%_%niter%.gdx' cc_int ; +execute_unload 'handoff%ds%reeds_data%ds%curtout_%case%_%niter%.gdx' cc_int ; *following line will load in the level values if the switch is enabled *note that this is still within the conditional that we are now past the first iteration @@ -149,6 +149,6 @@ $ifthene.lastiter %niter%=%maxiter% $eval nextwindow %window% + 1 tfix(t)$(tmodel(t)$(yeart(t) int(sw.GSw_SkipAugurYear)): - Augur.main(t=t, tnext=tnext[t], casedir=casepath, iteration=iteration) + #%%### Run resource adequacy calculations + if (not onlygams) and (tnext[t] > int(sw.GSw_SkipRAyear)): + reeds.resource_adequacy.ra_calcs.main( + t=t, + tnext=tnext[t], + casedir=casepath, + iteration=iteration, + ) #%% Driver function @@ -105,15 +108,15 @@ def main(casepath, t, overwrite=False): and os.path.isfile( os.path.join( sw.casedir, 'inputs_case', f'stress{t}i{iteration+1}', 'cf_vre.csv')) - ## Check if Augur finished + ## Check if resource adequacy calculations finished and os.path.isfile( os.path.join( - sw.casedir, 'ReEDS_Augur', 'augur_data', f'ReEDS_Augur_{t}.gdx')) + sw.casedir, 'handoff', 'reeds_data', f'ccdata_{t}.gdx')) ): print(f'Already ran {t}i{iteration} so continuing to next iteration') continue - #%% Run ReEDS and Augur + #%% Run ReEDS and RA calculations run_reeds(casepath, t, iteration=iteration) #%% Stop here if there's no stress period data for the next iteration @@ -151,10 +154,6 @@ def main(casepath, t, overwrite=False): help='year to run') parser.add_argument('--iteration', '-i', type=int, default=0, help='iteration counter for this run') - parser.add_argument('--onlygams', '-g', action='store_true', - help='Only run GAMS (skip Augur)') - parser.add_argument('--onlyaugur', '-a', action='store_true', - help='Only run Augur (skip GAMS)') parser.add_argument('--overwrite', '-o', action='store_true', help='Overwrite iterations that have already finished') @@ -162,8 +161,6 @@ def main(casepath, t, overwrite=False): casepath = args.casepath t = args.t iteration = args.iteration - onlygams = args.onlygams - onlyaugur = args.onlyaugur overwrite = args.overwrite #%% Switch to run folder diff --git a/d_solvepcm.gms b/reeds/core/solve_pcm/solve_pcm.gms similarity index 82% rename from d_solvepcm.gms rename to reeds/core/solve_pcm/solve_pcm.gms index 711b58c7..7742a76e 100644 --- a/d_solvepcm.gms +++ b/reeds/core/solve_pcm/solve_pcm.gms @@ -9,11 +9,11 @@ Sw_PCM = 1 ; Sw_MinCF = 0 ; *** Unfix the operational variables -$include d2_unfix_op.gms +$include reeds%ds%core%ds%solve_pcm%ds%unfix_op.gms *** Define the h- and szn-dependent parameters $onMultiR -$include d1_temporal_params.gms +$include reeds%ds%core%ds%solve%ds%2_temporal_params.gms $offMulti *** Solve it diff --git a/d2_unfix_op.gms b/reeds/core/solve_pcm/unfix_op.gms similarity index 100% rename from d2_unfix_op.gms rename to reeds/core/solve_pcm/unfix_op.gms diff --git a/dump_alldata.gms b/reeds/core/terminus/dump_alldata.gms similarity index 100% rename from dump_alldata.gms rename to reeds/core/terminus/dump_alldata.gms diff --git a/reeds/get_last_iter.py b/reeds/core/terminus/get_last_iter.py similarity index 100% rename from reeds/get_last_iter.py rename to reeds/core/terminus/get_last_iter.py diff --git a/e_powfrac_calc.gms b/reeds/core/terminus/powfrac_calc.gms similarity index 100% rename from e_powfrac_calc.gms rename to reeds/core/terminus/powfrac_calc.gms diff --git a/e_report.gms b/reeds/core/terminus/report.gms similarity index 99% rename from e_report.gms rename to reeds/core/terminus/report.gms index ebd14d10..bb729367 100644 --- a/e_report.gms +++ b/reeds/core/terminus/report.gms @@ -113,10 +113,10 @@ h2_demand_type / "electricity", "cross-sector"/ ; -* Parameter definitions in the following file are read from e_report_params.csv +* Parameter definitions in the following file are read from report_params.csv * and parsed in copy_files.py. -* All output parameters should be defined in e_report_params.csv. -$include e_report_params.gms +* All output parameters should be defined in report_params.csv. +$include autocode%ds%report_params.gms * Restrict operational outputs to representative timeslices and seasons h(h)$[not h_rep(h)] = no ; @@ -2077,15 +2077,15 @@ h2_usage(r,h,t)$tmodel_new(t) = *======================================== $ifthene.powerfrac %GSw_calc_powfrac% == 1 -$include e_powfrac_calc.gms +$include reeds%ds%core%ds%terminus%ds%powfrac_calc.gms $endif.powerfrac *======================================== * Dump results *======================================== -* The parameter list in the following file is read from e_report_params.csv +* The parameter list in the following file is read from report_params.csv * and parsed in copy_files.py execute_unload "outputs%ds%rep_%fname%.gdx" -$include e_report_paramlist.txt +$include autocode%ds%report_paramlist.txt ; diff --git a/e_report_dump.py b/reeds/core/terminus/report_dump.py similarity index 95% rename from e_report_dump.py rename to reeds/core/terminus/report_dump.py index faa3a20a..7e3ef747 100644 --- a/e_report_dump.py +++ b/reeds/core/terminus/report_dump.py @@ -7,11 +7,10 @@ import os import traceback import sys - -# Third-party packages -import pandas as pd import gdxpds - +import pandas as pd +from pathlib import Path +sys.path.append(str(Path(__file__).parent.parent.parent.parent)) import reeds @@ -201,13 +200,13 @@ def postprocess_outputs(case, outputs_path=None, verbose=0): _outputs_path = os.path.join(case, 'outputs') if outputs_path is None else outputs_path ## System cost - reeds.output_calc.calc_systemcost(case).to_csv( + reeds.results.calc_systemcost(case).to_csv( os.path.join(_outputs_path, 'post_systemcost_annualized.csv'), index=False, ) ## Reinforcement and spur-line - reeds.output_calc.calc_reinforcement_spur_capacity_miles(case).to_csv( + reeds.results.calc_reinforcement_spur_capacity_miles(case).to_csv( os.path.join(_outputs_path, 'post_tech_transmission.csv'), index=False, ) @@ -249,7 +248,7 @@ def postprocess_outputs(case, outputs_path=None, verbose=0): reeds_path = os.path.abspath(os.path.dirname(__file__)) log = reeds.log.makelog(scriptname=__file__, logpath=os.path.join(case, "gamslog.txt")) - print("Starting e_report_dump.py") + print("Starting report_dump.py") # %%### Parse inputs and get switches outputs_path = os.path.join(case, "outputs") @@ -259,7 +258,7 @@ def postprocess_outputs(case, outputs_path=None, verbose=0): ### Get new file names if applicable dfparams = pd.read_csv( - os.path.join(case, "e_report_params.csv"), + os.path.join(case, 'reeds', 'core', 'terminus', 'report_params.csv'), comment="#", index_col="param", ) @@ -304,8 +303,8 @@ def postprocess_outputs(case, outputs_path=None, verbose=0): #%% All done - print("Completed e_report_dump.py") + print("Completed report_dump.py") try: - toc(tic=tic, year=0, path=case, process="e_report_dump.py") + toc(tic=tic, year=0, path=case, process="report_dump.py") except NameError: print("reeds/log.py not found, so not logging output") diff --git a/e_report_params.csv b/reeds/core/terminus/report_params.csv similarity index 99% rename from e_report_params.csv rename to reeds/core/terminus/report_params.csv index 70b37020..92d20d84 100644 --- a/e_report_params.csv +++ b/reeds/core/terminus/report_params.csv @@ -235,7 +235,7 @@ tax_expenditure_ptc(t),2004$,PTC tax expenditure,,, "watret_ivrt(i,v,r,t)",Mgal,retired water capacity by region and vintage,,, "watret_out(i,r,t)",Mgal,retired water capacity by region,,, "watret_ann_out(i,v,r,t)",Mgal/yr,annual retired water capacity by region,,, -# Parameters defined before e_report.gms - some are used in r2x,,,,, +# Parameters defined earlier in model - some are used in r2x,,,,, bcr,,,,,1 biosupply,,,,,1 cap_hyd_szn_adj,,,,,1 diff --git a/aws_setup.sh b/reeds/hpc/aws_setup.sh similarity index 98% rename from aws_setup.sh rename to reeds/hpc/aws_setup.sh index 285eec2a..74944cb9 100644 --- a/aws_setup.sh +++ b/reeds/hpc/aws_setup.sh @@ -95,7 +95,7 @@ git clone git@github.com:ReEDS-Model/ReEDS.git # Run ReEDS! # (using nohup to keep the process from dying when you end your ssh session) -#nohup python runbatch_aws.py -c weekendcentroid -r 4 -b centwknd > myout.txt & +#nohup python runreeds.py -c weekendcentroid -r 4 -b centwknd > myout.txt & #======================================== @@ -130,4 +130,3 @@ git clone git@github.com:ReEDS-Model/ReEDS.git #move to your git directory #!!! could be different for different users #cd ~/r2/r2_aws - diff --git a/srun_template.sh b/reeds/hpc/srun_template.sh similarity index 100% rename from srun_template.sh rename to reeds/hpc/srun_template.sh diff --git a/input_processing/WriteHintage.py b/reeds/input_processing/WriteHintage.py similarity index 99% rename from input_processing/WriteHintage.py rename to reeds/input_processing/WriteHintage.py index 79310347..68e01041 100644 --- a/input_processing/WriteHintage.py +++ b/reeds/input_processing/WriteHintage.py @@ -49,7 +49,8 @@ import datetime from sklearn.cluster import KMeans from sklearn.exceptions import ConvergenceWarning -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +from pathlib import Path +sys.path.append(str(Path(__file__).parent.parent.parent)) import reeds # Time the operation of this script tic = datetime.datetime.now() diff --git a/input_processing/__init__.py b/reeds/input_processing/__init__.py similarity index 100% rename from input_processing/__init__.py rename to reeds/input_processing/__init__.py diff --git a/input_processing/aggregate_regions.py b/reeds/input_processing/aggregate_regions.py similarity index 99% rename from input_processing/aggregate_regions.py rename to reeds/input_processing/aggregate_regions.py index fd353381..bdd9513a 100644 --- a/input_processing/aggregate_regions.py +++ b/reeds/input_processing/aggregate_regions.py @@ -21,15 +21,16 @@ import argparse import numpy as np import os -import pandas as pd import gdxpds +import pandas as pd import shutil import sys import datetime from glob import glob from warnings import warn import h5py -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +from pathlib import Path +sys.path.append(str(Path(__file__).parent.parent.parent)) import reeds ## Time the operation of this script tic = datetime.datetime.now() @@ -1071,7 +1072,7 @@ def agg_disagg(filepath, r2aggreg_glob, r_ba_glob, runfiles_row): #%% Get the settings file runfiles = ( pd.read_csv( - os.path.join(reeds_path, 'runfiles.csv'), + os.path.join(reeds_path, 'reeds', 'input_processing', 'runfiles.csv'), dtype={'fix_cols':str}, index_col='filename', comment='#', ).fillna({'fix_cols':''}) diff --git a/input_processing/calc_financial_inputs.py b/reeds/input_processing/calc_financial_inputs.py similarity index 99% rename from input_processing/calc_financial_inputs.py rename to reeds/input_processing/calc_financial_inputs.py index 96d74e4c..f2611a52 100644 --- a/input_processing/calc_financial_inputs.py +++ b/reeds/input_processing/calc_financial_inputs.py @@ -9,7 +9,8 @@ import sys import datetime # Time the operation of this script -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +from pathlib import Path +sys.path.append(str(Path(__file__).parent.parent.parent)) import reeds diff --git a/input_processing/check_inputs.py b/reeds/input_processing/check_inputs.py similarity index 98% rename from input_processing/check_inputs.py rename to reeds/input_processing/check_inputs.py index 6e4450f7..d94fff11 100644 --- a/input_processing/check_inputs.py +++ b/reeds/input_processing/check_inputs.py @@ -5,7 +5,8 @@ import argparse import datetime import yaml -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +from pathlib import Path +sys.path.append(str(Path(__file__).parent.parent.parent)) import reeds tic = datetime.datetime.now() diff --git a/input_processing/climateprep.py b/reeds/input_processing/climateprep.py similarity index 99% rename from input_processing/climateprep.py rename to reeds/input_processing/climateprep.py index b031dcf0..e7c33970 100644 --- a/input_processing/climateprep.py +++ b/reeds/input_processing/climateprep.py @@ -73,8 +73,8 @@ import os import sys import pandas as pd - -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +from pathlib import Path +sys.path.append(str(Path(__file__).parent.parent.parent)) import reeds # Time the operation of this script tic = datetime.datetime.now() diff --git a/input_processing/copy_files.py b/reeds/input_processing/copy_files.py similarity index 98% rename from input_processing/copy_files.py rename to reeds/input_processing/copy_files.py index 4059eef7..55a60880 100644 --- a/input_processing/copy_files.py +++ b/reeds/input_processing/copy_files.py @@ -13,7 +13,7 @@ import h5py from pathlib import Path # Local Imports -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +sys.path.append(str(Path(__file__).parent.parent.parent)) import reeds @@ -48,7 +48,7 @@ def read_runfiles(reeds_path, inputs_case, sw, agglevel_variables): """ runfiles = ( pd.read_csv( - os.path.join(reeds_path, 'runfiles.csv'), + os.path.join(reeds_path, 'reeds', 'input_processing', 'runfiles.csv'), dtype={'fix_cols':str, 'depends_on_switch':str, 'depends_on_switch_value': str}, @@ -154,7 +154,7 @@ def get_source_deflator_map(reeds_path): """ # Inflation-adjusted inputs sources_dollaryear = pd.read_csv( - os.path.join(reeds_path,'sources.csv'), + os.path.join(reeds_path,'docs','sources.csv'), usecols=["RelativeFilePath", "DollarYear"] ) deflator = pd.read_csv( @@ -769,14 +769,14 @@ def scalar_csv_to_txt(path_to_scalar_csv): return dfscalar -def param_csv_to_txt(path_to_param_csv, writelist=True): +def param_csv_to_txt(infilepath, outdirpath, writelist=True): """ Write a parameter csv to GAMS-readable text Format of csv should be: parameter(indices),units,comment """ # Load the csv dfparams = pd.read_csv( - path_to_param_csv, + infilepath, index_col='param', comment='#', ) # Create the GAMS-readable param definition string (comments must be ≤255 characters) @@ -786,15 +786,17 @@ def param_csv_to_txt(path_to_param_csv, writelist=True): for i, row in dfparams.loc[dfparams.input != 1].iterrows() ]) # Write it to a file, replacing .csv with .gms in the filename - param_gms_path = path_to_param_csv.replace('.csv','.gms') + param_gms_path = Path(outdirpath, Path(infilepath).stem + '.gms') with open(param_gms_path, 'w') as w: w.write(paramtext) # Write the list of parameters if desired if writelist: # Create the GAMS-readable list of parameters (without indices) paramlist = '\n'.join(dfparams.index.map(lambda x: x.split('(')[0]).tolist()) - param_list_path = ( - path_to_param_csv.replace('params','paramlist').replace('.csv','.txt')) + param_list_path = Path( + outdirpath, + Path(infilepath).stem.replace('params','paramlist') + '.txt' + ) with open(param_list_path, 'w') as w: w.write(paramlist) @@ -929,7 +931,7 @@ def write_GAMS_sets(runfiles, reeds_path, inputs_case): for i, row in sets.iterrows() ]) + '\n$onlisting\n' # Write to file - with open(os.path.join(casedir,'b_sets.gms'), 'w') as f: + with open(os.path.join(casedir, 'autocode', 'b_sets.gms'), 'w') as f: f.write(settext) @@ -987,10 +989,6 @@ def write_non_region_file(filename, filepath, src_file, dir_dst, sw, regions_and else: shutil.copy(src_file, os.path.join(dir_dst,filename)) - if filename == 'e_report_params.csv': - # Rewrite e_report_params as GAMS-readable definition - param_csv_to_txt(os.path.join(dir_dst,'e_report_params.csv')) - if filename == 'scalars.csv': # Rewrite scalars.csv as GAMS-readable definition scalars = reeds.io.get_scalars(full=True) @@ -1302,7 +1300,12 @@ def write_miscellaneous_files( Many of these files are not in the non_region_files and region_files set (runfiles.csv - from function read_runfiles). """ - # ---- Miscellaneous files not in non_region_files or region_files ---- + ### Solver file + case = Path(inputs_case).parent + optfile = reeds.io.get_optfile(case) + shutil.copy(Path(reeds_path, 'reeds', 'solver', optfile), case) + + ### Parsed switches pd.DataFrame( {'*pvb_type': [f'pvb{i}' for i in sw['GSw_PVB_Types'].split('_')], 'ilr': [np.around(float(c) / 100, 2) for c in sw['GSw_PVB_ILR'].split('_') @@ -1554,6 +1557,12 @@ def write_miscellaneous_files( ).rename_axis('tech').dropna().astype(int) unitsize.to_csv(fpath_out) + # Rewrite report_params as GAMS-readable definitions + param_csv_to_txt( + infilepath=Path(reeds_path, 'reeds', 'core', 'terminus', 'report_params.csv'), + outdirpath=Path(inputs_case, '..', 'autocode'), + ) + def generate_maps_gpkg(inputs_case): """ @@ -1609,7 +1618,7 @@ def main(reeds_path, inputs_case): write_GAMS_sets(runfiles, reeds_path, inputs_case) # Rewrite the switches tables as GAMS-readable definition - # (gswitches.csv is first written at runbatch.py) + # (gswitches.csv is first written at runreeds.py) scalar_csv_to_txt(os.path.join(inputs_case,'gswitches.csv')) source_deflator_map = get_source_deflator_map(reeds_path) diff --git a/input_processing/forecast.py b/reeds/input_processing/forecast.py similarity index 99% rename from input_processing/forecast.py rename to reeds/input_processing/forecast.py index f20a81d3..f1442c46 100644 --- a/input_processing/forecast.py +++ b/reeds/input_processing/forecast.py @@ -16,16 +16,17 @@ ### =========================================================================== import argparse +import gdxpds import pandas as pd import numpy as np -import gdxpds import os import sys import shutil from glob import glob from warnings import warn import datetime -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +from pathlib import Path +sys.path.append(str(Path(__file__).parent.parent.parent)) import reeds # Time the operation of this script tic = datetime.datetime.now() @@ -522,4 +523,4 @@ def forecast( ## infiles.to_csv( ## os.path.join(reeds_path, 'inputs', 'userinput', 'futurefiles.csv'), ## index=False -## ) \ No newline at end of file +## ) diff --git a/input_processing/fuelcostprep.py b/reeds/input_processing/fuelcostprep.py similarity index 98% rename from input_processing/fuelcostprep.py rename to reeds/input_processing/fuelcostprep.py index 2fff1d4f..5ea65223 100644 --- a/input_processing/fuelcostprep.py +++ b/reeds/input_processing/fuelcostprep.py @@ -17,7 +17,8 @@ import sys import argparse import datetime -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +from pathlib import Path +sys.path.append(str(Path(__file__).parent.parent.parent)) import reeds # Time the operation of this script tic = datetime.datetime.now() diff --git a/input_processing/h2_storage.py b/reeds/input_processing/h2_storage.py similarity index 97% rename from input_processing/h2_storage.py rename to reeds/input_processing/h2_storage.py index 6f93c6e6..0a8ba75c 100644 --- a/input_processing/h2_storage.py +++ b/reeds/input_processing/h2_storage.py @@ -9,7 +9,8 @@ import os import sys import datetime -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +from pathlib import Path +sys.path.append(str(Path(__file__).parent.parent.parent)) import reeds @@ -138,4 +139,4 @@ def main(reeds_path, inputs_case): reeds.log.toc(tic=tic, year=0, process='input_processing/h2_storage.py', path=os.path.join(inputs_case,'..')) - print('Finished h2_storage.py') \ No newline at end of file + print('Finished h2_storage.py') diff --git a/input_processing/hourly_load.py b/reeds/input_processing/hourly_load.py similarity index 99% rename from input_processing/hourly_load.py rename to reeds/input_processing/hourly_load.py index 516a50ca..ce4b8e0a 100644 --- a/input_processing/hourly_load.py +++ b/reeds/input_processing/hourly_load.py @@ -33,7 +33,7 @@ import pandas as pd from pathlib import Path import sys -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +sys.path.append(str(Path(__file__).parent.parent.parent)) import reeds diff --git a/input_processing/hourly_plots.py b/reeds/input_processing/hourly_plots.py similarity index 99% rename from input_processing/hourly_plots.py rename to reeds/input_processing/hourly_plots.py index a11987aa..2e7d8bb6 100644 --- a/input_processing/hourly_plots.py +++ b/reeds/input_processing/hourly_plots.py @@ -11,10 +11,10 @@ import matplotlib.pyplot as plt from matplotlib import patheffects as pe import cmocean - -import hourly_repperiods -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +from pathlib import Path +sys.path.append(str(Path(__file__).parent.parent.parent)) import reeds +from reeds.input_processing import hourly_repperiods from reeds import plots plots.plotparams() diff --git a/input_processing/hourly_repperiods.py b/reeds/input_processing/hourly_repperiods.py similarity index 99% rename from input_processing/hourly_repperiods.py rename to reeds/input_processing/hourly_repperiods.py index 6303bc30..5570f603 100644 --- a/input_processing/hourly_repperiods.py +++ b/reeds/input_processing/hourly_repperiods.py @@ -32,10 +32,11 @@ import sklearn.cluster import sklearn.neighbors import traceback -import hourly_writetimeseries -import hourly_plots -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +from pathlib import Path +sys.path.append(str(Path(__file__).parent.parent.parent)) import reeds +from reeds.input_processing import hourly_writetimeseries +from reeds.input_processing import hourly_plots ## Time the operation of this script tic = datetime.datetime.now() diff --git a/input_processing/hourly_writetimeseries.py b/reeds/input_processing/hourly_writetimeseries.py similarity index 99% rename from input_processing/hourly_writetimeseries.py rename to reeds/input_processing/hourly_writetimeseries.py index 5fa32608..5954516a 100644 --- a/input_processing/hourly_writetimeseries.py +++ b/reeds/input_processing/hourly_writetimeseries.py @@ -8,7 +8,8 @@ import datetime import pandas as pd import numpy as np -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +from pathlib import Path +sys.path.append(str(Path(__file__).parent.parent.parent)) import reeds ##% Time the operation of this script tic = datetime.datetime.now() @@ -789,7 +790,7 @@ def main(sw, reeds_path, inputs_case, periodtype='rep', make_plots=1, logging=Tr np.ravel([[c]*GSw_HourlyChunkLength for c in outchunks_allyrs]) )) - # %%### h_dt_szn for Augur + # %%### h_dt_szn for resource adequacy calculations if not len(hmap_myr) % 8760: ## Important: When modeling a single weather year, rep periods in the ## h_dt_szn table are just the single-year periods concatenated n times. @@ -1330,7 +1331,7 @@ def main(sw, reeds_path, inputs_case, periodtype='rep', make_plots=1, logging=Tr False, False, ], - ## 8760 hour linkage set for Augur (h,szn,year,hour) + ## 8760 hour linkage set for resource adequacy (h,szn,year,hour) "h_dt_szn": [ h_dt_szn[["h", "season", "ccseason", "year", "hour"]].assign( h=h_dt_szn.h.map(chunkmap) diff --git a/input_processing/hydcf.py b/reeds/input_processing/hydcf.py similarity index 99% rename from input_processing/hydcf.py rename to reeds/input_processing/hydcf.py index 196c9e20..c62e3b20 100644 --- a/input_processing/hydcf.py +++ b/reeds/input_processing/hydcf.py @@ -19,7 +19,8 @@ import os import sys import datetime -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +from pathlib import Path +sys.path.append(str(Path(__file__).parent.parent.parent)) import reeds def get_monthly_plant_generation(inputs_case: str) -> ( @@ -466,4 +467,4 @@ def main(reeds_path, inputs_case): reeds.log.toc(tic=tic, year=0, process='input_processing/hydcf.py', path=os.path.join(inputs_case,'..')) - print('Finished hydcf.py') \ No newline at end of file + print('Finished hydcf.py') diff --git a/input_processing/mcs_sampler.py b/reeds/input_processing/mcs_sampler.py similarity index 99% rename from input_processing/mcs_sampler.py rename to reeds/input_processing/mcs_sampler.py index 8dd3e514..ebea7d5d 100644 --- a/input_processing/mcs_sampler.py +++ b/reeds/input_processing/mcs_sampler.py @@ -18,9 +18,10 @@ from collections import defaultdict # Local Imports -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +from pathlib import Path +sys.path.append(str(Path(__file__).parent.parent.parent)) import reeds -from input_processing import copy_files +from reeds.input_processing import copy_files #%% =========================================================================== @@ -425,7 +426,7 @@ def get_dist_instructions(reeds_path: str, inputs_case: str, run_ReEDS: bool = T mcs_dist_groups = sw['MCS_dist_groups'].split('.') if not run_ReEDS: - # Since you did not run using runbatch.py - check inputs here + # Since you did not run using runreeds.py - check inputs here general_mcs_dist_validation(reeds_path, mcs_dist_path, sw) # Ignore all cases not in mcs_dist_groups diff --git a/input_processing/outage_rates.py b/reeds/input_processing/outage_rates.py similarity index 99% rename from input_processing/outage_rates.py rename to reeds/input_processing/outage_rates.py index 9bf26f41..80d7d00a 100644 --- a/input_processing/outage_rates.py +++ b/reeds/input_processing/outage_rates.py @@ -6,12 +6,13 @@ import datetime import argparse import h5py -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +from pathlib import Path +sys.path.append(str(Path(__file__).parent.parent.parent)) import reeds +reeds_path = reeds.io.reeds_path ## Time the operation of this script tic = datetime.datetime.now() -reeds_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) #%%### Fixed inputs tz_in = 'UTC' diff --git a/input_processing/plantcostprep.py b/reeds/input_processing/plantcostprep.py similarity index 99% rename from input_processing/plantcostprep.py rename to reeds/input_processing/plantcostprep.py index 45cc67ce..2956af20 100644 --- a/input_processing/plantcostprep.py +++ b/reeds/input_processing/plantcostprep.py @@ -8,7 +8,8 @@ import sys import argparse import datetime -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +from pathlib import Path +sys.path.append(str(Path(__file__).parent.parent.parent)) import reeds # Time the operation of this script tic = datetime.datetime.now() diff --git a/input_processing/recf.py b/reeds/input_processing/recf.py similarity index 99% rename from input_processing/recf.py rename to reeds/input_processing/recf.py index 340ad565..a340bb9f 100644 --- a/input_processing/recf.py +++ b/reeds/input_processing/recf.py @@ -5,7 +5,7 @@ Resources: - Creates a resource-to-(i,r,ccreg) lookup table for use in hourly_writesupplycurves.py - and Augur + and resource adequacy calculations - Add the distributed PV resources RECF: - Add the distributed PV recf profiles @@ -23,7 +23,8 @@ import os import pandas as pd import sys -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +from pathlib import Path +sys.path.append(str(Path(__file__).parent.parent.parent)) import reeds diff --git a/runfiles.csv b/reeds/input_processing/runfiles.csv similarity index 95% rename from runfiles.csv rename to reeds/input_processing/runfiles.csv index 1a60f823..366b1025 100644 --- a/runfiles.csv +++ b/reeds/input_processing/runfiles.csv @@ -231,7 +231,7 @@ interconnection_queues.csv,inputs/capacity_exogenous/interconnection_queues.csv, itc_energy_comm_bonus.csv,,1,mean,ignore,r,*i,,0,0,,1,,,, itc_frac_monetized.csv,,1,ignore,ignore,,,,,0,,,,,, itc_fractions.csv,,1,ignore,ignore,,"i,country,t",,0,0,,,,,, -ivt.csv,,1,ignore,ignore,,,,,0,,,,,,created in runbatch.py +ivt.csv,,1,ignore,ignore,,,,,0,,,,,,created in runreeds.py ivt_step.csv,,1,ignore,ignore,,,,,0,,,,,, lcclike.csv,inputs/sets/lcclike.csv,1,ignore,ignore,,,,,0,,,,lcclike,, load_2010.csv,,1,sum,ignore,r,wide,,1,0,,1,,,,disaggfunc set to ignore because load will already be in correct spatial resolution @@ -374,7 +374,6 @@ rggicon.csv,inputs/emission_constraints/rggicon.csv,int(sw.GSw_RGGI) != 0,ignore rps_fraction.csv,inputs/state_policies/rps_fraction.csv,int(sw.GSw_StateRPS) != 0,ignore,ignore,st,st,,1,0,,,,,, rsc_combined.csv,,1,sc_cat,ignore,r,"*i,rscbin",*i,0,0,value,1,,,,done for upv/csp/wind; disaggfunc set to ignore because supply curve data is already at county level rsc_wsc.csv,,1,ignore,ignore,,,,,0,,,,,, -runfiles.csv,runfiles.csv,1,ignore,ignore,ignore,,,0,0,,,,,,so meta sc_cat.csv,inputs/sets/sc_cat.csv,1,ignore,ignore,,,,,0,,,,sc_cat,, scalars.csv,inputs/scalars.csv,1,ignore,ignore,,,,,0,,1,,,, set_actualszn.csv,,1,ignore,ignore,,,,,0,,,,,, @@ -488,38 +487,7 @@ recf.h5,,1,recf,ignore,wide,datetime,,1,keepindex,,1,,,, csp.h5,,1,csp,ignore,wide,datetime,,1,keepindex,,1,,,, gswitches.txt,,1,ignore,ignore,,,,,0,,,,,, scalars.txt,,1,ignore,ignore,,,,,0,,,,,, -Augur.py,Augur.py,1,ignore,ignore,,,,,,,,,,, Project.toml,Project.toml,1,ignore,ignore,,,,,,,,,,, -b_inputs.gms,b_inputs.gms,1,ignore,ignore,,,,,,,,,,, -c_mga.gms,c_mga.gms,1,ignore,ignore,,,,,,,,,,, -c_supplymodel.gms,c_supplymodel.gms,1,ignore,ignore,,,,,,,,,,, -c_supplyobjective.gms,c_supplyobjective.gms,1,ignore,ignore,,,,,,,,,,, -cbc.opt,cbc.opt,1,ignore,ignore,,,,,,,,,,, -cplex.op2,cplex.op2,1,ignore,ignore,,,,,,,,,,, -cplex.opt,cplex.opt,1,ignore,ignore,,,,,,,,,,, -createmodel.gms,createmodel.gms,1,ignore,ignore,,,,,,,,,,, -d_solveallyears.gms,d_solveallyears.gms,1,ignore,ignore,,,,,,,,,,, -d_solveoneyear.gms,d_solveoneyear.gms,1,ignore,ignore,,,,,,,,,,, -d_solve_iterate.py,d_solve_iterate.py,1,ignore,ignore,,,,,,,,,,, -d_solvepcm.gms,d_solvepcm.gms,1,ignore,ignore,,,,,,,,,,, -d_solveprep.gms,d_solveprep.gms,1,ignore,ignore,,,,,,,,,,, -d_solvewindow.gms,d_solvewindow.gms,1,ignore,ignore,,,,,,,,,,, -d1_temporal_params.gms,d1_temporal_params.gms,1,ignore,ignore,,,,,,,,,,, -d1_financials.gms,d1_financials.gms,1,ignore,ignore,,,,,,,,,,, -d2_post_solve_adjustments.gms,d2_post_solve_adjustments.gms,1,ignore,ignore,,,,,,,,,,, -d2_unfix_op.gms,d2_unfix_op.gms,1,ignore,ignore,,,,,,,,,,, -d2_varfix.gms,d2_varfix.gms,1,ignore,ignore,,,,,,,,,,, -d3_data_dump.gms,d3_data_dump.gms,1,ignore,ignore,,,,,,,,,,, -dump_alldata.gms,dump_alldata.gms,1,ignore,ignore,,,,,,,,,,, -e_powfrac_calc.gms,e_powfrac_calc.gms,1,ignore,ignore,,,,,,,,,,, -e_report.gms,e_report.gms,1,ignore,ignore,,,,,,,,,,, -e_report_dump.py,e_report_dump.py,1,ignore,ignore,,,,,,,,,,, -e_report_params.csv,e_report_params.csv,1,ignore,ignore,,,,,,,,,,, gamslice.txt,gamslice.txt,0,ignore,ignore,,,,,,,,,,, -gurobi.opt,gurobi.opt,1,ignore,ignore,,,,,,,,,,, -interim_report.py,interim_report.py,1,ignore,ignore,,,,,,,,,,, max_hintage_number.txt,,1,ignore,ignore,,,,,0,,,,,, -raw_value_streams.py,raw_value_streams.py,1,ignore,ignore,,,,,,,,,,, -runbatch.py,runbatch.py,1,ignore,ignore,,,,,,,,,,, -tc_phaseout.py,tc_phaseout.py,1,ignore,ignore,,,,,,,,,,, -valuestreams.py,valuestreams.py,1,ignore,ignore,,,,,,,,,,, +runreeds.py,runreeds.py,1,ignore,ignore,,,,,,,,,,, diff --git a/input_processing/transmission.py b/reeds/input_processing/transmission.py similarity index 99% rename from input_processing/transmission.py rename to reeds/input_processing/transmission.py index 913ea293..8b733df6 100644 --- a/input_processing/transmission.py +++ b/reeds/input_processing/transmission.py @@ -10,7 +10,7 @@ import sys import datetime from pathlib import Path -sys.path.append(str(Path(__file__).parent.parent)) +sys.path.append(str(Path(__file__).parent.parent.parent)) import reeds tic = datetime.datetime.now() diff --git a/input_processing/writecapdat.py b/reeds/input_processing/writecapdat.py similarity index 99% rename from input_processing/writecapdat.py rename to reeds/input_processing/writecapdat.py index b0cb134b..42a391fa 100644 --- a/input_processing/writecapdat.py +++ b/reeds/input_processing/writecapdat.py @@ -30,7 +30,8 @@ import os import sys import pandas as pd -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +from pathlib import Path +sys.path.append(str(Path(__file__).parent.parent.parent)) import reeds diff --git a/input_processing/writedrshift.py b/reeds/input_processing/writedrshift.py similarity index 96% rename from input_processing/writedrshift.py rename to reeds/input_processing/writedrshift.py index 106528f7..2d6d99dd 100644 --- a/input_processing/writedrshift.py +++ b/reeds/input_processing/writedrshift.py @@ -18,7 +18,8 @@ import argparse import pandas as pd import datetime -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +from pathlib import Path +sys.path.append(str(Path(__file__).parent.parent.parent)) import reeds # Time the operation of this script tic = datetime.datetime.now() diff --git a/input_processing/writesupplycurves.py b/reeds/input_processing/writesupplycurves.py similarity index 99% rename from input_processing/writesupplycurves.py rename to reeds/input_processing/writesupplycurves.py index 52b80a44..342b5e94 100644 --- a/input_processing/writesupplycurves.py +++ b/reeds/input_processing/writesupplycurves.py @@ -18,7 +18,8 @@ import h5py import datetime import pandas as pd -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +from pathlib import Path +sys.path.append(str(Path(__file__).parent.parent.parent)) import reeds reeds_path = reeds.io.reeds_path diff --git a/reeds/inputs.py b/reeds/inputs.py index 45dfc002..416bebbd 100644 --- a/reeds/inputs.py +++ b/reeds/inputs.py @@ -13,7 +13,7 @@ from warnings import warn sys.path.append(str(Path(__file__).parent.parent)) import reeds -from input_processing import mcs_sampler +from reeds.input_processing import mcs_sampler ### Functions def parse_regions(case_or_string, case=None): @@ -338,6 +338,62 @@ def parse_cases( return dfcases_out +def solvestring_sequential( + batch_case, caseSwitches, + cur_year, next_year, prev_year, restartfile, + toLogGamsString=' logOption=4 logFile=gamslog.txt appendLog=1 ', + hpc=0, iteration=0, stress_year=None, + temporal_inputs='rep', + ): + """ + Typical inputs: + * restartfile: batch_case if first solve year else {batch_case}_{prev_year} + * caseSwitches: loaded from {batch_case}/inputs_case/switches.csv + """ + savefile = f"{batch_case}_{cur_year}i{iteration}" + _stress_year = f"{cur_year}i0" if stress_year is None else stress_year + out = ( + f"gams {Path('reeds','core','solve','3_solve_oneyear.gms')}" + + (" license=gamslice.txt" if hpc else '') + + " o=" + os.path.join("lstfiles", f"{savefile}.lst") + + " r=" + os.path.join("g00files", restartfile) + + " gdxcompress=1" + + " xs=" + os.path.join("g00files", savefile) + + toLogGamsString + + f" --case={batch_case}" + + f" --cur_year={cur_year}" + + f" --next_year={next_year}" + + f" --prev_year={prev_year}" + + f" --stress_year={_stress_year}" + + f" --temporal_inputs={temporal_inputs}" + + ''.join([f" --{s}={caseSwitches[s]}" for s in [ + 'GSw_Canada', + 'GSw_ClimateHydro', + 'GSw_ClimateWater', + 'GSw_gopt', + 'GSw_HourlyChunkLengthRep', + 'GSw_HourlyChunkLengthStress', + 'GSw_HourlyType', + 'GSw_HourlyWrapLevel', + 'GSw_MGA_CostDelta', + 'GSw_MGA_Direction', + 'GSw_PVB_Dur', + 'GSw_SkipRAyear', + 'GSw_StateCO2ImportLevel', + 'GSw_StartMarkets', + 'GSw_ValStr', + 'solver', + 'debug', + 'startyear', + 'diagnose', + 'diagnose_year' + ]]) + + '\n' + ) + + return out + + def get_bin( df_in, bin_num, diff --git a/reeds/io.py b/reeds/io.py index 434ee172..d4bcd313 100644 --- a/reeds/io.py +++ b/reeds/io.py @@ -8,7 +8,6 @@ import numpy as np import pandas as pd import geopandas as gpd -from glob import glob from pathlib import Path from pandas.api.types import is_float_dtype @@ -490,12 +489,13 @@ def read_output( df[col] = df[col].str.decode('utf-8') except KeyError: ## Empty dataframes aren't written to h5 file, so make one ourselves - e_report_params = pd.read_csv( - os.path.join(case, 'e_report_params.csv'), - comment='#', - ) - _index = e_report_params.loc[ - e_report_params.param.map(lambda x: x.split('(')[0]) == key, 'param' + fpath = Path(case, 'reeds', 'core', 'terminus', 'report_params.csv') + ## Fall back to older params list if necessary for backwards compatibility + if not fpath.is_file(): + fpath = Path(case, 'e_report_params.csv') + report_params = pd.read_csv(fpath, comment='#') + _index = report_params.loc[ + report_params.param.map(lambda x: x.split('(')[0]) == key, 'param' ].squeeze() if not len(_index): raise KeyError(f"{filename} is not in {h5path}") @@ -619,10 +619,9 @@ def standardize_case(case=None): return case -def get_switches(case=None, **kwargs): +def get_switches_base(case=None, **kwargs): """ - Get pd.Series of switch values from switches.csv, augur_switches.csv, - and CPLEX opt file. + Get pd.Series of switch values from switches.csv. Accepts either {case} or {case}/inputs_case as input. If {case} is None, the default switch values listed in cases.csv are retrieved. @@ -646,11 +645,49 @@ def get_switches(case=None, **kwargs): index_col=0, header=None, ).squeeze(1) - ### Augur-specific switches + return sw + + +def get_optfile(case=None, **kwargs): + """ + Get the name of the optfile used by GAMS, formatted as described by + https://gams.com/49/docs/UG_GamsCall.html#GAMSAOoptfile + """ + sw = get_switches_base(case, **kwargs) + GSw_gopt = int(sw.GSw_gopt) + if GSw_gopt == 1: + suffix = 'opt' + elif len(str(GSw_gopt)) == 1: + suffix = f'op{GSw_gopt}' + elif len(str(GSw_gopt)) == 2: + suffix = f'o{GSw_gopt}' + else: + suffix = str(GSw_gopt) + optfile = f'{sw.solver}.{suffix}'.lower() + return optfile + + +def get_switches(case=None, **kwargs): + """ + Get pd.Series of switch values from switches.csv, ra_switches.csv, + and solver settings file. + Accepts either {case} or {case}/inputs_case as input. + + If {case} is None, the default switch values listed in cases.csv are retrieved. + + If additional keyword arguments are provided, they replace the values specified + in {case}. This behavior can be used to read all the switches for a case (or all + the default settings) but change a single switch to a different value (when + making plots for different input settings, for example). If a key is provided + that is not a valid switch name, it is ignored. + """ + case = standardize_case(case) + sw = get_switches_base(case) + ### Resource-adequacy-specific switches try: fpath_asw = os.path.join( (case if case is not None else reeds_path), - 'ReEDS_Augur', 'augur_switches.csv', + 'reeds', 'resource_adequacy', 'ra_switches.csv', ) asw = pd.read_csv(fpath_asw, index_col='key') for i, row in asw.iterrows(): @@ -670,7 +707,7 @@ def get_switches(case=None, **kwargs): row.value = float(row.value) sw = pd.concat([sw, asw.value]) except FileNotFoundError: - print(f"{fpath_asw} not found so leaving out Augur switches") + print(f"{fpath_asw} not found so leaving out resource adequacy switches") ### Add derivative switches sw['resource_adequacy_years_list'] = [int(y) for y in sw['resource_adequacy_years'].split('_')] sw['num_resource_adequacy_years'] = len(sw['resource_adequacy_years_list']) @@ -679,22 +716,27 @@ def get_switches(case=None, **kwargs): sw['future_hydcf_rep_years_list'] = [ int(y) for y in sw.get('GSw_FutureHydCF_RepYears', _fallback).split('_') ] - ### Get number of threads to use in PRAS - opt_file = 'cplex.opt' if int(sw.GSw_gopt) == 1 else f'cplex.op{sw.GSw_gopt}' - try: - threads = get_param_value(os.path.join(case, opt_file), "threads", dtype=int) - except (FileNotFoundError, TypeError): - threads = get_param_value(os.path.join(reeds_path, opt_file), "threads", dtype=int) + ## Get number of threads to use in PRAS + ## (read from case folder; fall back to repo if case folder doesn't exist yet) + opt_file = get_optfile(case) + fpath_repo = Path(reeds_path, 'reeds', 'solver', opt_file) + if case is None: + fpath_opt = fpath_repo + else: + fpath_opt = Path(case, opt_file) + if not fpath_opt.is_file(): + fpath_opt = fpath_repo + threads = get_param_value(fpath_opt, "threads", dtype=int) sw['threads'] = threads - ### Determine whether run is on HPC + ## Determine whether run is on HPC sw['hpc'] = True if int(os.environ.get('REEDS_USE_SLURM', 0)) else False - ### Add the run location + ## Add the run location sw['casedir'] = case sw['reeds_path'] = reeds_path if case is None else os.path.dirname(os.path.dirname(case)) - ### Get the number of hours per period to use in plots + ## Get the number of hours per period to use in plots sw['hoursperperiod'] = {'day': 24, 'wek': 120, 'year': 24}[sw['GSw_HourlyType']] sw['periodsperyear'] = {'day': 365, 'wek': 73, 'year': 365}[sw['GSw_HourlyType']] - + ### Overwrite values with keyword arguments if provided for key, value in kwargs.items(): if key in sw.keys(): sw[key] = value @@ -1228,16 +1270,19 @@ def get_last_iteration(case, year=2050, datum=None, samples=None): """Get the last iteration of PRAS for a given case/year""" if datum not in [None,'flow','energy']: raise ValueError(f"datum must be in [None,'flow','energy'] but is {datum}") - pattern = f"PRAS_{year}i*" \ - + (f'-{samples}' if samples is not None else '') \ - + (f'-{datum}' if datum is not None else '') \ + pattern = ( + f"PRAS_{year}i*" + + (f'-{samples}' if samples is not None else '') + + (f'-{datum}' if datum is not None else '') + '.h5' - matches = list(Path(case, 'ReEDS_Augur', 'PRAS').glob(pattern)) + ) + matches = list(Path(case, 'handoff', 'PRAS').glob(pattern)) if not matches: - print(f"ERROR: The run {case} has not solved last modeled year {year}.", file=sys.stderr) - sys.exit(1) + raise ValueError(f"{case} has not solved year {year}") infile = max( matches, + ## File names are formatted as 'PRAS_{year}i{iteration}.h5' or + ## 'PRAS_{year}i{iteration}-{other_identifiers}.h5'; keep the largest iteration key=lambda f: int(f.stem[f.stem.rfind('i')+1:].split('-')[0]) ) iteration = int( @@ -1257,11 +1302,11 @@ def get_pras_system(case, year=None, iteration='last', verbose=0): get_last_iteration(case, t)[1] if iteration in [None, 'last'] else iteration ) - infile = os.path.join(case, 'ReEDS_Augur', 'PRAS', f"PRAS_{t}i{_iteration}.pras") + infile = os.path.join(case, 'handoff', 'PRAS', f"PRAS_{t}i{_iteration}.pras") if not os.path.exists(infile): raise FileNotFoundError( f'{infile} does not exist; run postprocessing/run_reeds2pras.py or rerun ' - 'the ReEDS case with keep_augur_files=1' + 'the ReEDS case with keep_resource_adequacy_files=1' ) pras = {} with h5py.File(infile,'r') as f: diff --git a/reeds/log.py b/reeds/log.py index 9816d719..ea9a9130 100755 --- a/reeds/log.py +++ b/reeds/log.py @@ -103,7 +103,7 @@ def get_solve_times(path=''): process = line[len('--- Job ') : line.index(' Stop ')] x = f'--- Job {process} Stop ' y = ' elapsed ' - label = process if process != 'd_solveoneyear.gms' else stress_year + label = process if process != '3_solve_oneyear.gms' else stress_year lengths['total'][label] = pd.Timedelta(line[line.index(y) + len(y) :]) times['stop'][label] = pd.Timestamp(line[len(x) : line.index(y)]) times['start'][label] = times['stop'][label] - lengths['total'][label] @@ -164,7 +164,7 @@ def write_last_solve_time(path=''): scriptname = lasttime.name year = 0 else: - scriptname = 'd_solveoneyear.gms' + scriptname = '3_solve_oneyear.gms' year = int(lasttime.name.split('i')[0]) towrite = { 'gams': scriptname, @@ -173,7 +173,7 @@ def write_last_solve_time(path=''): 'remainder': 'solver/remainder', } with open(os.path.join(path, 'meta.csv'), 'a') as METAFILE: - if (scriptname == 'd_solveoneyear.gms') and all([i in lasttime for i in towrite]): + if (scriptname == '3_solve_oneyear.gms') and all([i in lasttime for i in towrite]): for i, process in enumerate(towrite): METAFILE.writelines( '{},{},{},{},{}\n'.format( diff --git a/reeds/prasplots.py b/reeds/prasplots.py index 9722ff6e..ba09429d 100644 --- a/reeds/prasplots.py +++ b/reeds/prasplots.py @@ -252,7 +252,7 @@ def plot_pras_eue_timeseries_full( else: _iteration = iteration infile = os.path.join( - case, 'ReEDS_Augur', 'PRAS', + case, 'handoff', 'PRAS', f"PRAS_{year}i{_iteration}" + (f'-{samples}' if samples is not None else '') + '.h5' ) dfpras = reeds.io.read_pras_results(infile) @@ -363,7 +363,7 @@ def plot_pras_samples( ### Get unit availability filebase = os.path.join( - case, 'ReEDS_Augur', 'PRAS', + case, 'handoff', 'PRAS', f"PRAS_{t}i{_iteration}" f"{f'-{samples}' if isinstance(samples, int) else ''}" ) diff --git a/reeds/ra.py b/reeds/ra.py deleted file mode 100644 index 22d8ee35..00000000 --- a/reeds/ra.py +++ /dev/null @@ -1,125 +0,0 @@ -### Imports -import os -import sys -import numpy as np -import pandas as pd -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -import reeds - - -### Functions -def get_pras_eue(case, t, iteration=0): - """ - """ - ### Get PRAS outputs - dfpras = reeds.io.read_pras_results( - os.path.join(case, 'ReEDS_Augur', 'PRAS', f"PRAS_{t}i{iteration}.h5") - ) - ### Create the time index - sw = reeds.io.get_switches(case) - dfpras.index = reeds.timeseries.get_timeindex(sw['resource_adequacy_years']) - - ### Keep the EUE columns by zone - eue_tail = '_EUE' - dfeue = dfpras[[ - c for c in dfpras - if (c.endswith(eue_tail) and not c.startswith('USA')) - ]].copy() - ## Drop the tailing _EUE - dfeue = dfeue.rename( - columns=dict(zip(dfeue.columns, [c[:-len(eue_tail)] for c in dfeue]))) - - return dfeue - - -def get_eue_periods( - case, t, iteration=0, - hierarchy_level='transgrp', - stress_metric='EUE', - period_agg_method='sum', - ): - """_summary_ - - Args: - sw (pd.series): ReEDS switches for this run. - t (int): Model solve year. - iteration (int, optional): Iteration number of this solve year. Defaults to 0. - hierarchy_level (str, optional): column of hierarchy.csv specifying the spatial - level over which to calculate stress_metric. Defaults to 'country'. - stress_metric (str, optional): 'EUE' or 'NEUE'. Defaults to 'EUE'. - period_agg_method (str, optional): 'sum' or 'max', indicating how to aggregate - over the hours in each period. Defaults to 'sum'. - - Raises: - NotImplementedError: if invalid value for stress_metric or GSw_PRM_StressModel - - Returns: - pd.DataFrame: Table of periods sorted in descending order by stress metric. - """ - sw = reeds.io.get_switches(case) - ### Get the region aggregator - rmap = reeds.io.get_rmap(case=case, hierarchy_level=hierarchy_level) - - ### Get EUE from PRAS - dfeue = get_pras_eue(case=case, t=t, iteration=iteration) - ## Aggregate to hierarchy_level - dfeue = ( - dfeue - .rename_axis('r', axis=1).rename_axis('h', axis=0) - .rename(columns=rmap).groupby(axis=1, level=0).sum() - ) - - ###### Calculate the stress metric by period - if stress_metric.upper() == 'EUE': - ### Aggregate according to period_agg_method - dfmetric_period = ( - dfeue - .groupby([dfeue.index.year, dfeue.index.month, dfeue.index.day]) - .agg(period_agg_method) - .rename_axis(['y','m','d']) - ) - elif stress_metric.upper() == 'NEUE': - ### Get load at hierarchy_level - dfload = reeds.io.read_h5py_file( - os.path.join( - case,'ReEDS_Augur','augur_data',f'pras_load_{t}.h5') - ).rename(columns=rmap).groupby(level=0, axis=1).sum() - dfload.index = dfeue.index - - ### Recalculate NEUE [ppm] and aggregate appropriately - if period_agg_method == 'sum': - dfmetric_period = ( - dfeue - .groupby([dfeue.index.year, dfeue.index.month, dfeue.index.day]) - .agg(period_agg_method) - .rename_axis(['y','m','d']) - ) / ( - dfload - .groupby([dfload.index.year, dfload.index.month, dfload.index.day]) - .agg(period_agg_method) - .rename_axis(['y','m','d']) - ) * 1e6 - elif period_agg_method == 'max': - dfmetric_period = ( - (dfeue / dfload) - .groupby([dfeue.index.year, dfeue.index.month, dfeue.index.day]) - .agg(period_agg_method) - .rename_axis(['y','m','d']) - ) * 1e6 - - ### Sort and drop zeros and duplicates - dfmetric_top = ( - dfmetric_period.stack('r') - .sort_values(ascending=False) - .replace(0,np.nan).dropna() - .reset_index().drop_duplicates(['y','m','d'], keep='first') - .set_index(['y','m','d','r']).squeeze(1).rename(stress_metric) - .reset_index('r') - ) - ## Convert to timestamp, then to ReEDS period - dfmetric_top['actual_period'] = [ - reeds.timeseries.timestamp2h(pd.Timestamp(*d), sw['GSw_HourlyType']).split('h')[0] - for d in dfmetric_top.index.values - ] - - return dfmetric_top diff --git a/reeds/reedsplots.py b/reeds/reedsplots.py index c853ac64..dae1aa92 100644 --- a/reeds/reedsplots.py +++ b/reeds/reedsplots.py @@ -4992,8 +4992,7 @@ def plot_seed_stressperiods( ): """ """ - sys.path.append(os.path.join(reeds.io.reeds_path, 'input_processing')) - import hourly_repperiods + from reeds.input_processing import hourly_repperiods sw = reeds.io.get_switches(case) hierarchy = reeds.io.get_hierarchy(case) @@ -6207,21 +6206,20 @@ def map_stressors( dflevel = dfmap[level].copy() ### Get the data - augur_files = { - 'vre_gen': os.path.join(case, 'ReEDS_Augur', 'augur_data', f'pras_vre_gen_{t}.h5'), - 'load': os.path.join(case, 'ReEDS_Augur', 'augur_data', f'pras_load_{t}.h5'), + ra_files = { + 'vre_gen': os.path.join(case, 'handoff', 'reeds_data', f'pras_vre_gen_{t}.h5'), + 'load': os.path.join(case, 'handoff', 'reeds_data', f'pras_load_{t}.h5'), } - if any([not os.path.exists(fpath) for fpath in augur_files.values()]): - import ReEDS_Augur.prep_data as prep_data - prep_data.main(t, case, iteration) + if any([not os.path.exists(fpath) for fpath in ra_files.values()]): + reeds.resource_adequacy.prep_data.main(t, case, iteration) - vre_gen = reeds.io.read_file(augur_files['vre_gen'], parse_timestamps=True) + vre_gen = reeds.io.read_file(ra_files['vre_gen'], parse_timestamps=True) vre_gen.columns = pd.MultiIndex.from_tuples( vre_gen.columns.map(lambda x: tuple(x.split('|'))), names=['i','r'], ) - load = reeds.io.read_file(augur_files['load'], parse_timestamps=True) + load = reeds.io.read_file(ra_files['load'], parse_timestamps=True) recf = reeds.io.read_file( os.path.join(case, 'inputs_case', 'recf.h5'), diff --git a/reeds/resource_adequacy/__init__.py b/reeds/resource_adequacy/__init__.py new file mode 100644 index 00000000..63c5372a --- /dev/null +++ b/reeds/resource_adequacy/__init__.py @@ -0,0 +1,4 @@ +from . import capacity_credit as capacity_credit +from . import prep_data as prep_data +from . import ra_calcs as ra_calcs +from . import stress_periods as stress_periods diff --git a/ReEDS_Augur/capacity_credit.py b/reeds/resource_adequacy/capacity_credit.py similarity index 98% rename from ReEDS_Augur/capacity_credit.py rename to reeds/resource_adequacy/capacity_credit.py index d778d6da..0440e102 100644 --- a/ReEDS_Augur/capacity_credit.py +++ b/reeds/resource_adequacy/capacity_credit.py @@ -1,8 +1,8 @@ #%% IMPORTS import os +import gdxpds import numpy as np import pandas as pd -import gdxpds import reeds @@ -34,8 +34,8 @@ def set_marg_vre_step_size(t, sw, gdx, hierarchy): Inputs * marg_vre_steps [int]: Number of previous solve years to consider when evaluating the marginal VRE step size (default: 2). Must be at least 1; - a value of 2 can help reduce oscillations. Augur will automatically drop - from consideration solves that are more than 5 years from the previous solve. + a value of 2 can help reduce oscillations. Solves that are more than + 5 years from the previous solve are automatically dropped. ''' # load yearset for getting various previous steps yearset = gdx['tmodel_new'].allt.astype(int).tolist() @@ -45,8 +45,7 @@ def set_marg_vre_step_size(t, sw, gdx, hierarchy): step_sizes = [] for step in range(int(sw['marg_vre_steps'])): - # try-except to handle cases where there aren't multiple - # steps to go back to (e.g. running Augur after 1st solve) + # try-except to handle cases where there aren't multiple steps to go back to try: target_last_step = yearset[yearset.index(t)-step] @@ -56,7 +55,7 @@ def set_marg_vre_step_size(t, sw, gdx, hierarchy): step_sizes.append(get_relative_step_sizes(t, yearset, target_last_step)) prev_year_list.append(target_last_step) except Exception: - print('First Augur year so no previous steps') + print('First resource adequacy year so no previous steps') relative_step_sizes = pd.DataFrame(list(zip(prev_year_list, step_sizes)), columns=['t', 'step']) @@ -121,10 +120,10 @@ def reeds_cc(t, tnext, casedir): hierarchy = reeds.io.get_hierarchy(casedir).reset_index() resources = pd.read_csv(os.path.join(inputs_case, 'resources.csv')) - augur_data = os.path.join(casedir,'ReEDS_Augur','augur_data') - cap = pd.read_csv(os.path.join(augur_data, f'max_cap_{t}.csv')) + reeds_data = os.path.join(casedir, 'handoff', 'reeds_data') + cap = pd.read_csv(os.path.join(reeds_data, f'max_cap_{t}.csv')) - gdx = gdxpds.to_dataframes(os.path.join(augur_data,f'reeds_data_{t}.gdx')) + gdx = gdxpds.to_dataframes(os.path.join(reeds_data, f'reeds_data_{t}.gdx')) techs = gdx['i_subsets'].pivot(columns='i_subtech',index='i',values='Value') techs.columns = techs.columns.str.lower() r = gdx['rfeas'] @@ -170,9 +169,9 @@ def reeds_cc(t, tnext, casedir): ### Prepare the seasonal profiles ## vre_gen needs to have tech_class_r columns ## last version has (ccseason,year,h,hour) index - vre_gen = pd.read_hdf(os.path.join(augur_data,f'vre_gen_exist_{t}.h5')) + vre_gen = pd.read_hdf(os.path.join(reeds_data,f'vre_gen_exist_{t}.h5')) ## vre_cf_marg has same columns and index as vre_gen - vre_cf_marg = pd.read_hdf(os.path.join(augur_data,f'vre_cf_marg_{t}.h5')) + vre_cf_marg = pd.read_hdf(os.path.join(reeds_data,f'vre_cf_marg_{t}.h5')) if int(sw['GSw_PRM_CapCreditMulti']) == 0: # Restrict capacity credit evaluation to use 2012 only (rather than multi-year) @@ -191,7 +190,7 @@ def reeds_cc(t, tnext, casedir): load_profiles = ( # HOURLY_PROFILES['load'].profiles - pd.read_hdf(os.path.join(augur_data,f'load_{t}.h5')) + pd.read_hdf(os.path.join(reeds_data,f'load_{t}.h5')) ### Map BA regions to ccreg's and sum over them .rename(columns=hierarchy.set_index('r').ccreg) .groupby(axis=1, level=0).sum() diff --git a/ReEDS_Augur/diagnostic_plots.py b/reeds/resource_adequacy/diagnostic_plots.py similarity index 95% rename from ReEDS_Augur/diagnostic_plots.py rename to reeds/resource_adequacy/diagnostic_plots.py index 03870e6c..36145664 100644 --- a/ReEDS_Augur/diagnostic_plots.py +++ b/reeds/resource_adequacy/diagnostic_plots.py @@ -1,6 +1,7 @@ #%%### Imports import os import sys +import gdxpds import pandas as pd import numpy as np import matplotlib as mpl @@ -8,10 +9,10 @@ from matplotlib import patheffects as pe from glob import glob import traceback -import gdxpds import cmocean ### Local imports -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +from pathlib import Path +sys.path.append(str(Path(__file__).parent.parent.parent)) import reeds from reeds import plots @@ -30,10 +31,10 @@ def delete_temporary_files(sw): Delete temporary csv, pkl, and h5 files """ dropfiles = ( - glob(os.path.join(sw['casedir'],'ReEDS_Augur','augur_data',f"*_{sw['t']}.pkl")) - + glob(os.path.join(sw['casedir'],'ReEDS_Augur','augur_data',f"*_{sw['t']}.h5")) - + glob(os.path.join(sw['casedir'],'ReEDS_Augur','augur_data',f"*_{sw['t']}.csv")) - + glob(os.path.join(sw['casedir'],'ReEDS_Augur','PRAS',f"PRAS_{sw['t']}*.pras")) + glob(os.path.join(sw['casedir'],'handoff','reeds_data',f"*_{sw['t']}.pkl")) + + glob(os.path.join(sw['casedir'],'handoff','reeds_data',f"*_{sw['t']}.h5")) + + glob(os.path.join(sw['casedir'],'handoff','reeds_data',f"*_{sw['t']}.csv")) + + glob(os.path.join(sw['casedir'],'handoff','PRAS',f"PRAS_{sw['t']}*.pras")) ) for keyword in sw['keepfiles']: @@ -45,7 +46,7 @@ def delete_temporary_files(sw): #%% Input-loading function def get_inputs(sw): ### Make savepath - sw['savepath'] = os.path.join(sw['casedir'], 'outputs', 'Augur_plots') + sw['savepath'] = os.path.join(sw['casedir'], 'outputs', 'figures', 'resource_adequacy') os.makedirs(sw['savepath'], exist_ok=True) ##### Load shared parameters @@ -60,7 +61,7 @@ def get_inputs(sw): h_dt_szn['d'] = h_dt_szn.datetime.dt.strftime('sy%Yd%j') gdxreeds = gdxpds.to_dataframes( - os.path.join(sw['casedir'],'ReEDS_Augur','augur_data',f'reeds_data_{sw["t"]}.gdx')) + os.path.join(sw['casedir'],'handoff','reeds_data',f'reeds_data_{sw["t"]}.gdx')) techs = gdxreeds['i_subsets'].pivot(columns='i_subtech',index='i',values='Value') h2dac = techs['CONSUME'].dropna().index @@ -88,7 +89,7 @@ def get_inputs(sw): ### Load and aggregate the VRE generation profiles by tech group try: vre_gen = reeds.io.read_file( - os.path.join(sw['casedir'],'ReEDS_Augur','augur_data',f'pras_vre_gen_{sw.t}.h5'), + os.path.join(sw['casedir'],'handoff','reeds_data',f'pras_vre_gen_{sw.t}.h5'), parse_timestamps=True, ) except FileNotFoundError: @@ -117,7 +118,7 @@ def get_inputs(sw): try: load_r = pd.read_hdf( os.path.join( - sw['casedir'],'ReEDS_Augur','augur_data',f'load_{sw.t}.h5') + sw['casedir'],'handoff','reeds_data',f'load_{sw.t}.h5') ) load_r.index = fulltimeindex except FileNotFoundError: @@ -126,7 +127,7 @@ def get_inputs(sw): ### Load PRAS load try: pras_load = reeds.io.read_file( - os.path.join(sw['casedir'],'ReEDS_Augur','augur_data',f'pras_load_{sw.t}.h5'), + os.path.join(sw['casedir'],'handoff','reeds_data',f'pras_load_{sw.t}.h5'), parse_timestamps=True, ) except FileNotFoundError: @@ -135,7 +136,7 @@ def get_inputs(sw): try: pras_h2dac_load = reeds.io.read_file( os.path.join( - sw['casedir'],'ReEDS_Augur','augur_data', + sw['casedir'],'handoff','reeds_data', f"pras_h2dac_load_{sw['t']}.h5"), parse_timestamps=True, ) @@ -147,7 +148,7 @@ def get_inputs(sw): try: max_cap = pd.read_csv( os.path.join( - sw['casedir'],'ReEDS_Augur','augur_data',f"max_cap_{sw['t']}.csv")) + sw['casedir'],'handoff','reeds_data',f"max_cap_{sw['t']}.csv")) max_cap.i = reeds.reedsplots.simplify_techs(max_cap.i, display_level = 'diagnostics') except FileNotFoundError: max_cap = pd.DataFrame(columns=['i','v','r','MW']) @@ -155,7 +156,7 @@ def get_inputs(sw): ### Load LOLE/EUE/NEUE from PRAS try: pras = reeds.io.read_pras_results( - os.path.join(sw['casedir'], 'ReEDS_Augur', 'PRAS', + os.path.join(sw['casedir'], 'handoff', 'PRAS', f"PRAS_{sw.t}i{sw.iteration}.h5") ) pras.index = fulltimeindex @@ -498,14 +499,14 @@ def plot_pras_ICAP(sw, dfs): plt.close() -def plot_augur_pras_capacity(sw, dfs): +def plot_reeds_pras_capacity(sw, dfs): """ - Plot the nameplate capacity from Augur and PRAS to check consistency + Plot the nameplate capacity from ReEDS and PRAS to check consistency """ if not len(dfs['pras_system']): print('PRAS system was not loaded') return - savename = f"PRAS-Augur-capacity-{sw['t']}.png" + savename = f"PRAS-ReEDS-capacity-{sw['t']}.png" ### Get the colors tech_style = dfs['tech_style']['color'].squeeze() ### Collect the PRAS system capacities @@ -522,12 +523,12 @@ def plot_augur_pras_capacity(sw, dfs): .groupby(axis=1, level=[1,0]).sum().max().rename('MW') ) - ### Collect the Augur capacities - cap['augur'] = dfs['max_cap'].groupby(['i','r'], as_index=False).MW.sum() + ### Collect the ReEDS capacities + cap['reeds'] = dfs['max_cap'].groupby(['i','r'], as_index=False).MW.sum() ## Convert from s to p regions - cap['augur'].r = cap['augur'].r + cap['reeds'].r = cap['reeds'].r ## Aggregate by type - cap['augur'] = (cap['augur'] + cap['reeds'] = (cap['reeds'] .replace({'i':{'Hydropower Existing':'Hydropower', 'Hydropower New':'Hydropower'}}) .groupby(['r','i']).MW.sum() / 1e3 ) @@ -555,7 +556,7 @@ def plot_augur_pras_capacity(sw, dfs): ) alltechs = set() for r in zones: - df = pd.concat({'A':cap['augur'].get(r,pd.Series()), 'P':cap['pras'].get(r,pd.Series())}, axis=1).T + df = pd.concat({'A':cap['reeds'].get(r,pd.Series()), 'P':cap['pras'].get(r,pd.Series())}, axis=1).T order = [c for c in tech_style.index if c in df] missing = [c for c in df if c not in order] if len(missing): @@ -905,17 +906,17 @@ def plot_pras_load_units(sw, dfs): plt.close() -def plot_pras_augur_load(sw, dfs): - """PRAS load against Augur load""" +def plot_pras_reeds_load(sw, dfs): + """PRAS load against ReEDS load""" dfpras = dfs['pras_system']['load'].sum(axis=1).rename('PRAS') - dfaugur = dfs['load_r'].set_axis(dfpras.index).sum(axis=1).rename('Augur') + dfreeds = dfs['load_r'].set_axis(dfpras.index).sum(axis=1).rename('ReEDS') years = dfpras.index.year.unique() - linecolors = {'Augur':'C0', 'PRAS':'C3'} + linecolors = {'ReEDS':'C0', 'PRAS':'C3'} for year in years: - savename = f"demand_USA-Augur-PRAS-w{year}-{sw['t']}.png" + savename = f"demand_USA-ReEDS-PRAS-w{year}-{sw['t']}.png" plt.close() f,ax = plots.plotyearbymonth( - dfaugur.loc[str(year)], style='line', colors=linecolors['Augur']) + dfreeds.loc[str(year)], style='line', colors=linecolors['ReEDS']) plots.plotyearbymonth( dfpras.loc[str(year)], style='line', colors=linecolors['PRAS'], f=f, ax=ax) ## Legend @@ -1032,8 +1033,7 @@ def plot_cc_mar(sw, dfs): if not int(sw['GSw_PRM_CapCredit']): raise KeyError('No capacity credit values to plot') cc_results = gdxpds.to_dataframes(os.path.join( - sw['casedir'],'ReEDS_Augur','augur_data', - 'ReEDS_Augur_{}.gdx'.format(sw['t']) + sw['casedir'], 'handoff', 'reeds_data', f"ccdata_{sw['t']}.gdx" )) dfplot = cc_results[param].drop('t',axis=1).copy() @@ -1176,7 +1176,7 @@ def plot_netloadhours_histogram(sw, dfs): def plot_stressors(sw, dfs): """ - Map demand/CF/FOR (organized differently to allow use outside of Augur) + Map demand/CF/FOR (organized differently to allow for independent use) """ for iteration in range(sw['iteration']): plot_generator = reeds.reedsplots.map_stressors( @@ -1230,9 +1230,9 @@ def main(sw, debug=False): print('plot_pras_unitnumber() failed:', traceback.format_exc()) try: - plot_augur_pras_capacity(sw, dfs) + plot_reeds_pras_capacity(sw, dfs) except Exception: - print('plot_augur_pras_capacity() failed:', traceback.format_exc()) + print('plot_reeds_pras_capacity() failed:', traceback.format_exc()) try: plot_pras_load(sw, dfs) @@ -1278,9 +1278,9 @@ def main(sw, debug=False): if debug: try: - plot_pras_augur_load(sw, dfs) + plot_pras_reeds_load(sw, dfs) except Exception: - print('plot_pras_augur_load() failed:', traceback.format_exc()) + print('plot_pras_reeds_load() failed:', traceback.format_exc()) try: plot_pras_ICAP(sw, dfs) @@ -1346,7 +1346,7 @@ def main(sw, debug=False): sw['iteration'] = iteration ### Make the plots - print('plotting intermediate Augur results...') + print('plotting intermediate resource adequacy results...') try: main(sw, debug) except Exception as _err: @@ -1354,5 +1354,5 @@ def main(sw, debug=False): print(traceback.format_exc()) ### Remove intermediate csv files to save drive space - if (not int(sw['keep_augur_files'])) and (not int(sw['debug'])): + if (not int(sw['keep_resource_adequacy_files'])) and (not int(sw['debug'])): delete_temporary_files(sw) diff --git a/ReEDS_Augur/prep_data.py b/reeds/resource_adequacy/prep_data.py similarity index 97% rename from ReEDS_Augur/prep_data.py rename to reeds/resource_adequacy/prep_data.py index c1bab198..f7ad96c6 100644 --- a/ReEDS_Augur/prep_data.py +++ b/reeds/resource_adequacy/prep_data.py @@ -5,7 +5,7 @@ * run_pras.jl -> ReEDS2PRAS.jl -> PRAS.jl (probabilistic resource adequacy) The files used by PRAS are: -* In {case}/ReEDS_Augur/augur_data: +* In {case}/handoff/reeds_data: * cap_converter_{year}.csv * energy_cap_{year}.csv * max_cap_{year}.csv @@ -25,9 +25,9 @@ #%% General imports import os import re +import gdxpds import pandas as pd import numpy as np -import gdxpds ### Local imports import reeds @@ -105,11 +105,11 @@ def errorcheck_reeds2pras(casedir, csvout, h5out): def main(t, casedir, iteration=0): #%%### DEBUGGING: Inputs # t = 2020 - # reeds_path = os.path.expanduser('~/github2/ReEDS') - # casedir = os.path.join(reeds_path,'runs','v20230214_PRMaugurM0_Pacific_d7fIrh4_CC_y2012') + # reeds_path = os.path.expanduser('~/github/ReEDS') + # casedir = os.path.join(reeds_path,'runs','v20260417_reorgM0_Pacific') #%%### Get inputs from ReEDS - gdx_file = os.path.join(casedir,'ReEDS_Augur','augur_data',f'reeds_data_{t}.gdx') + gdx_file = os.path.join(casedir,'handoff','reeds_data',f'reeds_data_{t}.gdx') gdxreeds = gdxpds.to_dataframes(gdx_file) ### Use indices as multiindex for key in gdxreeds: @@ -567,7 +567,7 @@ def intify(v): #%% .csv files for key in csvout: csvout[key].round(int(sw['decimals'])).to_csv( - os.path.join(casedir,'ReEDS_Augur','augur_data',f'{key}_{t}.csv'), + os.path.join(casedir,'handoff','reeds_data',f'{key}_{t}.csv'), ) #%% .h5 files @@ -576,11 +576,11 @@ def intify(v): reeds.io.write_profile_to_h5( df=h5out[key].astype(np.float32), filename=f'{key}_{t}.h5', - outfolder=os.path.join(casedir,'ReEDS_Augur','augur_data'), + outfolder=os.path.join(casedir,'handoff','reeds_data'), ) else: h5out[key].astype(np.float32).to_hdf( - os.path.join(casedir,'ReEDS_Augur','augur_data',f'{key}_{t}.h5'), + os.path.join(casedir,'handoff','reeds_data',f'{key}_{t}.h5'), key='data', complevel=4, mode='w', ) diff --git a/Augur.py b/reeds/resource_adequacy/ra_calcs.py similarity index 86% rename from Augur.py rename to reeds/resource_adequacy/ra_calcs.py index 11c856eb..fc296808 100644 --- a/Augur.py +++ b/reeds/resource_adequacy/ra_calcs.py @@ -4,13 +4,11 @@ import sys import subprocess import datetime -import pandas as pd import gdxpds - +import pandas as pd +from pathlib import Path +sys.path.append(str(Path(__file__).parent.parent.parent)) import reeds -import ReEDS_Augur.prep_data as prep_data -import ReEDS_Augur.capacity_credit as capacity_credit -import ReEDS_Augur.stress_periods as stress_periods #%% Functions @@ -59,7 +57,7 @@ def run_pras( '--threads=1' if (sys.platform == 'darwin') or int(sw.get('pras_singlethread', 0)) else f"--threads={sw['threads'] if sw['threads'] > 0 else 'auto'}" ), - f"{os.path.join(scriptpath, 'ReEDS_Augur','run_pras.jl')}", + f"{os.path.join(scriptpath, 'reeds', 'resource_adequacy', 'run_pras.jl')}", f"--reeds_path={sw['reeds_path']}", f"--reedscase={casedir}", f"--solve_year={t}", @@ -101,20 +99,20 @@ def run_pras( def main(t, tnext, casedir, iteration=0): # #%% To debug, uncomment these lines and update the run path - # t = 2026 - # tnext = 2029 + # t = 2020 + # tnext = 2023 # reeds_path = reeds.io.reeds_path # casedir = os.path.join( - # reeds_path,'runs','v20250521_prasM0_Pacific') + # reeds_path,'runs','v20260417_reorgM1_Pacific') # iteration = 0 # assert tnext >= t # os.chdir(casedir) # ## Copy reeds2pras from repo to run folder # import shutil - # shutil.rmtree(os.path.join(casedir, 'reeds2pras')) + # shutil.rmtree(os.path.join(casedir, 'reeds', 'resource_adequacy', 'reeds2pras')) # shutil.copytree( - # os.path.join(reeds_path, 'reeds2pras'), - # os.path.join(casedir, 'reeds2pras'), + # os.path.join(reeds_path, 'reeds', 'resource_adequacy', 'reeds2pras'), + # os.path.join(casedir, 'reeds', 'resource_adequacy', 'reeds2pras'), # ignore=shutil.ignore_patterns('test'), # ) @@ -125,15 +123,15 @@ def main(t, tnext, casedir, iteration=0): #%% Prep data for resource adequacy print('Preparing data for resource adequacy calculations') tic = datetime.datetime.now() - augur_csv, augur_h5 = prep_data.main(t, casedir, iteration) - reeds.log.toc(tic=tic, year=t, process='ReEDS_Augur/prep_data.py') + reeds_csv, reeds_h5 = reeds.resource_adequacy.prep_data.main(t, casedir, iteration) + reeds.log.toc(tic=tic, year=t, process='ra/prep_data.py') #%% Calculate capacity credit if necessary; otherwise bypass print('calculating capacity credit...') tic = datetime.datetime.now() if int(sw['GSw_PRM_CapCredit']): - cc_results = capacity_credit.reeds_cc(t, tnext, casedir) + cc_results = reeds.resource_adequacy.capacity_credit.reeds_cc(t, tnext, casedir) else: cc_results = { 'cc_mar': pd.DataFrame(columns=['i','r','ccreg','szn','t','Value']), @@ -142,7 +140,7 @@ def main(t, tnext, casedir, iteration=0): 'sdbin_size': pd.DataFrame(columns=['ccreg','szn','bin','t','Value']), } - reeds.log.toc(tic=tic, year=t, process='ReEDS_Augur/capacity_credit.py') + reeds.log.toc(tic=tic, year=t, process='ra/capacity_credit.py') #%% Run PRAS if necessary solveyears = pd.read_csv( @@ -173,8 +171,8 @@ def main(t, tnext, casedir, iteration=0): ('user' not in sw['GSw_PRM_StressModel'].lower()) or ((int(sw.GSw_PRM_StressIterateMax)) and int(sw['GSw_PRM_CapCredit'])) ): - stress_periods.main(sw=sw, t=t, iteration=iteration) - reeds.log.toc(tic=tic, year=t, process='ReEDS_Augur/stress_periods.py') + reeds.resource_adequacy.stress_periods.main(sw=sw, t=t, iteration=iteration) + reeds.log.toc(tic=tic, year=t, process='ra/stress_periods.py') #%% Write gdx file explicitly to ensure that all entries ### (even empty dataframes) are written as parameters, not sets @@ -188,14 +186,13 @@ def main(t, tnext, casedir, iteration=0): ) gdx[-1].dataframe = cc_results[key] gdx.write( - os.path.join('ReEDS_Augur', 'augur_data', f'ReEDS_Augur_{t}.gdx') + os.path.join('handoff', 'reeds_data', f'ccdata_{t}.gdx') ) # #%% Uncomment to run diagnostic_plots # ### (typically run from call_{}.sh script for parallelization) # try: - # import ReEDS_Augur.diagnostic_plots as diagnostic_plots - # diagnostic_plots.main(sw) + # reeds.resource_adequacy.diagnostic_plots.main(sw) # except Exception as err: # print('diagnostic_plots.py failed with the following exception:') # print(err) @@ -204,7 +201,7 @@ def main(t, tnext, casedir, iteration=0): #%% Procedure if __name__ == '__main__': - parser = argparse.ArgumentParser(description="""Running ReEDS Augur""") + parser = argparse.ArgumentParser(description="Resource adequacy calculations") parser.add_argument("tnext", help="Next ReEDS solve year", type=int) parser.add_argument("t", help="Previous ReEDS solve year", type=int) diff --git a/ReEDS_Augur/augur_switches.csv b/reeds/resource_adequacy/ra_switches.csv similarity index 89% rename from ReEDS_Augur/augur_switches.csv rename to reeds/resource_adequacy/ra_switches.csv index bff65dbe..092ca9cd 100644 --- a/ReEDS_Augur/augur_switches.csv +++ b/reeds/resource_adequacy/ra_switches.csv @@ -12,5 +12,5 @@ cc_stor_stepsize,100,int,step size (in MW) used when determining the peaking cap decimals,3,int,number of decimals to round results to for ReEDS flex_consume_techs,"dac,electrolyzer",list,list of consume techs that are flexible keepfiles,"dropped_load,cf",list,list of temporary files to keep -marg_vre_steps,2,int,Number of previous solve years to consider when evaluating the marginal VRE step size (default: 2). Must be at least 1; a value of 2 can help reduce oscillations. Augur will automatically drop from consideration solves that are more than 5 years from the previous solve. +marg_vre_steps,2,int,Number of previous solve years to consider when evaluating the marginal VRE step size (default: 2). Must be at least 1; a value of 2 can help reduce oscillations. Solves that are more than 5 years from the previous solve are automatically dropped from consideration. storcap_cutoff,1,float,[MW and MWh] Minimum storage capacity to send to ReEDS2PRAS (applies to both power and energy capacity) diff --git a/reeds2pras/R2P_Test_Summary.png b/reeds/resource_adequacy/reeds2pras/R2P_Test_Summary.png similarity index 100% rename from reeds2pras/R2P_Test_Summary.png rename to reeds/resource_adequacy/reeds2pras/R2P_Test_Summary.png diff --git a/reeds2pras/README.md b/reeds/resource_adequacy/reeds2pras/README.md similarity index 81% rename from reeds2pras/README.md rename to reeds/resource_adequacy/reeds2pras/README.md index ca8db808..4662cb6f 100644 --- a/reeds2pras/README.md +++ b/reeds/resource_adequacy/reeds2pras/README.md @@ -31,15 +31,14 @@ If you have a completed ReEDS run and a REPL with ReEDS2PRAS (`using ReEDS2PRAS` ```julia using ReEDS2PRAS - -reedscase = "/projects/ntps/llavin/ReEDS/runs/ntpsrerun_Xlim_DemHi_90by2035EarlyPhaseout__core" # path to completed ReEDS run -solve_year = 2035 #need ReEDS Augur data for the input solve year -weather_year = 2012 # must be 2007-2013 or 2016-2023 +# path to completed ReEDS run +reedscase = "/path/to/ReEDS/runs/v20260511_main0_USA_defaults" +solve_year = 2035 +weather_year = 2012 timesteps = 8760 -user_descriptors = "your_user_descriptors_json_location_here" # Optional - if not passed uses default values # returns a parameterized PRAS system -pras_system = ReEDS2PRAS.reeds_to_pras(reedscase, solve_year, timesteps, weather_year, user_descriptors = user_descriptors) +pras_system = ReEDS2PRAS.reeds_to_pras(reedscase, solve_year, timesteps, weather_year) ``` This will save out a pras system to the variable `pras_system` from the ReEDS2PRAS run. The user can also save a PRAS system to a specific location using `PRAS.savemodel(pras_system, joinpath("MYPATH"*".pras")`. The saved PRAS system may then be read in by other tools like PRAS Analytics (`https://github.nrel.gov/PRAS/PRAS-Analytics`) for further analysis, post-processing, and plotting. @@ -52,14 +51,13 @@ ReEDS2PRAS can be run for multiple weather years of a completed ReEDS run by pas using ReEDS2PRAS # path to completed ReEDS run -reedscase = "/projects/ntps/llavin/ReEDS/runs/ntpsrerun_Xlim_DemHi_90by2035EarlyPhaseout__core" -solve_year = 2035 #need ReEDS Augur data for the input solve year -weather_year = 2007 # must be 2007-2013 or 2016-2023 +reedscase = "/path/to/ReEDS/runs/v20260511_main0_USA_defaults" +solve_year = 2035 +weather_year = 2007 timesteps = 61320 -user_descriptors = "your_user_descriptors_json_location_here" # Optional - if not passed uses default values # returns a parameterized PRAS system -pras_system = ReEDS2PRAS.reeds_to_pras(reedscase, solve_year, timesteps, weather_year, user_descriptors = user_descriptors) +pras_system = ReEDS2PRAS.reeds_to_pras(reedscase, solve_year, timesteps, weather_year) ``` Importantly, the timesteps count from the first hour of the first `weather_year`, so the user must input `2007` as the `weather_year` to run all 61320 hourly timesteps. diff --git a/reeds2pras/src/ReEDS2PRAS.jl b/reeds/resource_adequacy/reeds2pras/src/ReEDS2PRAS.jl similarity index 100% rename from reeds2pras/src/ReEDS2PRAS.jl rename to reeds/resource_adequacy/reeds2pras/src/ReEDS2PRAS.jl diff --git a/reeds2pras/src/main/create_pras_system.jl b/reeds/resource_adequacy/reeds2pras/src/main/create_pras_system.jl similarity index 100% rename from reeds2pras/src/main/create_pras_system.jl rename to reeds/resource_adequacy/reeds2pras/src/main/create_pras_system.jl diff --git a/reeds2pras/src/main/parse_reeds_data.jl b/reeds/resource_adequacy/reeds2pras/src/main/parse_reeds_data.jl similarity index 100% rename from reeds2pras/src/main/parse_reeds_data.jl rename to reeds/resource_adequacy/reeds2pras/src/main/parse_reeds_data.jl diff --git a/reeds2pras/src/models/Battery.jl b/reeds/resource_adequacy/reeds2pras/src/models/Battery.jl similarity index 100% rename from reeds2pras/src/models/Battery.jl rename to reeds/resource_adequacy/reeds2pras/src/models/Battery.jl diff --git a/reeds2pras/src/models/Gen_Storage.jl b/reeds/resource_adequacy/reeds2pras/src/models/Gen_Storage.jl similarity index 100% rename from reeds2pras/src/models/Gen_Storage.jl rename to reeds/resource_adequacy/reeds2pras/src/models/Gen_Storage.jl diff --git a/reeds2pras/src/models/Generator.jl b/reeds/resource_adequacy/reeds2pras/src/models/Generator.jl similarity index 100% rename from reeds2pras/src/models/Generator.jl rename to reeds/resource_adequacy/reeds2pras/src/models/Generator.jl diff --git a/reeds2pras/src/models/Interface.jl b/reeds/resource_adequacy/reeds2pras/src/models/Interface.jl similarity index 100% rename from reeds2pras/src/models/Interface.jl rename to reeds/resource_adequacy/reeds2pras/src/models/Interface.jl diff --git a/reeds2pras/src/models/Line.jl b/reeds/resource_adequacy/reeds2pras/src/models/Line.jl similarity index 100% rename from reeds2pras/src/models/Line.jl rename to reeds/resource_adequacy/reeds2pras/src/models/Line.jl diff --git a/reeds2pras/src/models/Region.jl b/reeds/resource_adequacy/reeds2pras/src/models/Region.jl similarity index 100% rename from reeds2pras/src/models/Region.jl rename to reeds/resource_adequacy/reeds2pras/src/models/Region.jl diff --git a/reeds2pras/src/models/Storage.jl b/reeds/resource_adequacy/reeds2pras/src/models/Storage.jl similarity index 100% rename from reeds2pras/src/models/Storage.jl rename to reeds/resource_adequacy/reeds2pras/src/models/Storage.jl diff --git a/reeds2pras/src/models/Thermal_Gen.jl b/reeds/resource_adequacy/reeds2pras/src/models/Thermal_Gen.jl similarity index 100% rename from reeds2pras/src/models/Thermal_Gen.jl rename to reeds/resource_adequacy/reeds2pras/src/models/Thermal_Gen.jl diff --git a/reeds2pras/src/models/Variable_Gen.jl b/reeds/resource_adequacy/reeds2pras/src/models/Variable_Gen.jl similarity index 100% rename from reeds2pras/src/models/Variable_Gen.jl rename to reeds/resource_adequacy/reeds2pras/src/models/Variable_Gen.jl diff --git a/reeds2pras/src/models/utils.jl b/reeds/resource_adequacy/reeds2pras/src/models/utils.jl similarity index 100% rename from reeds2pras/src/models/utils.jl rename to reeds/resource_adequacy/reeds2pras/src/models/utils.jl diff --git a/reeds2pras/src/reeds_to_pras.jl b/reeds/resource_adequacy/reeds2pras/src/reeds_to_pras.jl similarity index 100% rename from reeds2pras/src/reeds_to_pras.jl rename to reeds/resource_adequacy/reeds2pras/src/reeds_to_pras.jl diff --git a/reeds2pras/src/utils/reeds_data_parsing.jl b/reeds/resource_adequacy/reeds2pras/src/utils/reeds_data_parsing.jl similarity index 99% rename from reeds2pras/src/utils/reeds_data_parsing.jl rename to reeds/resource_adequacy/reeds2pras/src/utils/reeds_data_parsing.jl index ec1acc2d..2e434a5e 100644 --- a/reeds2pras/src/utils/reeds_data_parsing.jl +++ b/reeds/resource_adequacy/reeds2pras/src/utils/reeds_data_parsing.jl @@ -173,7 +173,7 @@ function split_generator_types(ReEDS_data::ReEDSdatapaths) ## Read {case}/inputs_case/tech-subset-table.csv tech_subset_table = get_technology_types(ReEDS_data) @debug "tech_subset_table is $(tech_subset_table)" - ## Read {case}/ReEDS_Augur/augur_data/max_cap_{year}.csv + ## Read {case}/handoff/reeds_data/max_cap_{year}.csv capacity_data = get_ICAP_data(ReEDS_data) ## Read {case}/inputs_case/resources.csv resources = get_valid_resources(ReEDS_data) @@ -738,8 +738,8 @@ function process_storages( efficiency_in = Dict( polarity => DataFrames.DataFrame(CSV.File(joinpath( ReEDS_data.ReEDSfilepath, - "ReEDS_Augur", - "augur_data", + "handoff", + "reeds_data", "$(polarity)_eff_$(ReEDS_data.year).csv" ))) for polarity in ["charge", "discharge"] diff --git a/reeds2pras/src/utils/reeds_input_parsing.jl b/reeds/resource_adequacy/reeds2pras/src/utils/reeds_input_parsing.jl similarity index 94% rename from reeds2pras/src/utils/reeds_input_parsing.jl rename to reeds/resource_adequacy/reeds2pras/src/utils/reeds_input_parsing.jl index 4a83a1f2..446792f8 100644 --- a/reeds2pras/src/utils/reeds_input_parsing.jl +++ b/reeds/resource_adequacy/reeds2pras/src/utils/reeds_input_parsing.jl @@ -75,15 +75,15 @@ end Returns ------- HDF5.h5read(filepath, "data") - A readout of the Augur load h5 file associated with the given ReEDS + A readout of the load h5 file associated with the given ReEDS filepath and year. """ function get_load_file(data::ReEDSdatapaths) filepath = joinpath( data.ReEDSfilepath, - "ReEDS_Augur", - "augur_data", + "handoff", + "reeds_data", "pras_load_$(string(data.year)).h5", ) columns = HDF5.h5read(filepath, "columns") @@ -93,7 +93,7 @@ function get_load_file(data::ReEDSdatapaths) end """ - This function reads a hdf5 file from the ReEDS Augur directory, based on + This function reads a hdf5 file from the ReEDS directory, based on the year provided in the ReEDSdatapaths struct. Parameters @@ -109,8 +109,8 @@ end function get_vg_cf_data(data::ReEDSdatapaths) filepath = joinpath( data.ReEDSfilepath, - "ReEDS_Augur", - "augur_data", + "handoff", + "reeds_data", "pras_vre_gen_$(string(data.year)).h5", ) columns = HDF5.h5read(filepath, "columns") @@ -145,7 +145,7 @@ end """ function get_max_unitsize(data::ReEDSdatapaths) filepath = joinpath( - data.ReEDSfilepath, "ReEDS_Augur", "augur_data", + data.ReEDSfilepath, "handoff", "reeds_data", "max_unitsize_$(string(data.year)).csv" ) df = DataFrames.DataFrame(CSV.File(filepath)) @@ -154,7 +154,7 @@ end """ - Get the forced outage data from the augur files. + Get the forced outage data. Parameters ---------- @@ -227,8 +227,8 @@ function get_line_capacity_data(data::ReEDSdatapaths) #assumes this file has been formatted by ReEDS to be PRM line capacity data filepath = joinpath( data.ReEDSfilepath, - "ReEDS_Augur", - "augur_data", + "handoff", + "reeds_data", "tran_cap_$(string(data.year)).csv", ) return DataFrames.DataFrame(CSV.File(filepath)) @@ -251,8 +251,8 @@ end function get_converter_capacity_data(data::ReEDSdatapaths) filepath = joinpath( data.ReEDSfilepath, - "ReEDS_Augur", - "augur_data", + "handoff", + "reeds_data", "cap_converter_$(string(data.year)).csv", ) return DataFrames.DataFrame(CSV.File(filepath)) @@ -285,7 +285,7 @@ end """ Returns a DataFrame containing the installed capacity of generators for a - given year, read from {case}/ReEDS_Augur/augur_data/max_cap_{year}.csv. + given year, read from {case}/handoff/reeds_data/max_cap_{year}.csv. Parameters ---------- @@ -305,8 +305,8 @@ end function get_ICAP_data(data::ReEDSdatapaths) filepath = joinpath( data.ReEDSfilepath, - "ReEDS_Augur", - "augur_data", + "handoff", + "reeds_data", "max_cap_$(string(data.year)).csv", ) return DataFrames.DataFrame(CSV.File(filepath)) @@ -380,8 +380,8 @@ end function get_storage_energy_capacity_data(data::ReEDSdatapaths) filepath = joinpath( data.ReEDSfilepath, - "ReEDS_Augur", - "augur_data", + "handoff", + "reeds_data", "energy_cap_$(string(data.year)).csv", ) return DataFrames.DataFrame(CSV.File(filepath)) @@ -420,7 +420,7 @@ function get_hourly_scheduled_outage_data(data::ReEDSdatapaths) end """ - Get the hourly forced outage data from the augur files. + Get the hourly forced outage data. Parameters ---------- diff --git a/reeds2pras/src/utils/runchecks.jl b/reeds/resource_adequacy/reeds2pras/src/utils/runchecks.jl similarity index 67% rename from reeds2pras/src/utils/runchecks.jl rename to reeds/resource_adequacy/reeds2pras/src/utils/runchecks.jl index 9a5a88a6..31f4d72c 100644 --- a/reeds2pras/src/utils/runchecks.jl +++ b/reeds/resource_adequacy/reeds2pras/src/utils/runchecks.jl @@ -14,17 +14,17 @@ function check_file(loc::String) end function run_checks(data::ReEDSdatapaths) - augur_data_path = joinpath(data.ReEDSfilepath, "ReEDS_Augur", "augur_data") + reeds_data_path = joinpath(data.ReEDSfilepath, "handoff", "reeds_data") filepaths = [ - joinpath(augur_data_path, "cap_converter_$(string(data.year)).csv"), - joinpath(augur_data_path, "charge_eff_$(string(data.year)).csv"), - joinpath(augur_data_path, "discharge_eff_$(string(data.year)).csv"), - joinpath(augur_data_path, "energy_cap_$(string(data.year)).csv"), - joinpath(augur_data_path, "max_cap_$(string(data.year)).csv"), - joinpath(augur_data_path, "max_unitsize_$(string(data.year)).csv"), - joinpath(augur_data_path, "pras_load_$(string(data.year)).h5"), - joinpath(augur_data_path, "pras_vre_gen_$(string(data.year)).h5"), - joinpath(augur_data_path, "tran_cap_$(string(data.year)).csv"), + joinpath(reeds_data_path, "cap_converter_$(string(data.year)).csv"), + joinpath(reeds_data_path, "charge_eff_$(string(data.year)).csv"), + joinpath(reeds_data_path, "discharge_eff_$(string(data.year)).csv"), + joinpath(reeds_data_path, "energy_cap_$(string(data.year)).csv"), + joinpath(reeds_data_path, "max_cap_$(string(data.year)).csv"), + joinpath(reeds_data_path, "max_unitsize_$(string(data.year)).csv"), + joinpath(reeds_data_path, "pras_load_$(string(data.year)).h5"), + joinpath(reeds_data_path, "pras_vre_gen_$(string(data.year)).h5"), + joinpath(reeds_data_path, "tran_cap_$(string(data.year)).csv"), joinpath(data.ReEDSfilepath, "inputs_case", "hydcapadj.csv"), joinpath(data.ReEDSfilepath, "inputs_case", "hydcf.csv"), joinpath(data.ReEDSfilepath, "inputs_case", "mttr.csv"), diff --git a/reeds2pras/test/Project.toml b/reeds/resource_adequacy/reeds2pras/test/Project.toml similarity index 100% rename from reeds2pras/test/Project.toml rename to reeds/resource_adequacy/reeds2pras/test/Project.toml diff --git a/reeds2pras/test/runtests.jl b/reeds/resource_adequacy/reeds2pras/test/runtests.jl similarity index 100% rename from reeds2pras/test/runtests.jl rename to reeds/resource_adequacy/reeds2pras/test/runtests.jl diff --git a/reeds2pras/test/test-ReEDS2PRAS.jl b/reeds/resource_adequacy/reeds2pras/test/test-ReEDS2PRAS.jl similarity index 100% rename from reeds2pras/test/test-ReEDS2PRAS.jl rename to reeds/resource_adequacy/reeds2pras/test/test-ReEDS2PRAS.jl diff --git a/reeds2pras/test/test-benchmark.jl b/reeds/resource_adequacy/reeds2pras/test/test-benchmark.jl similarity index 100% rename from reeds2pras/test/test-benchmark.jl rename to reeds/resource_adequacy/reeds2pras/test/test-benchmark.jl diff --git a/reeds2pras/test/utils.jl b/reeds/resource_adequacy/reeds2pras/test/utils.jl similarity index 100% rename from reeds2pras/test/utils.jl rename to reeds/resource_adequacy/reeds2pras/test/utils.jl diff --git a/ReEDS_Augur/run_pras.jl b/reeds/resource_adequacy/run_pras.jl similarity index 99% rename from ReEDS_Augur/run_pras.jl rename to reeds/resource_adequacy/run_pras.jl index b4958ad0..10f59e0c 100644 --- a/ReEDS_Augur/run_pras.jl +++ b/reeds/resource_adequacy/run_pras.jl @@ -391,7 +391,7 @@ end function main(args::Dict) #%% Define some intermediate filenames pras_system_path = joinpath( - args["reedscase"],"ReEDS_Augur","PRAS", + args["reedscase"], "handoff", "PRAS", "PRAS_$(args["solve_year"])i$(args["iteration"]).pras" ) @@ -477,7 +477,9 @@ if abspath(PROGRAM_FILE) == @__FILE__ args = parse_commandline() #%% Include ReEDS2PRAS - include(joinpath(args["reedscase"], "reeds2pras", "src", "ReEDS2PRAS.jl")) + include(joinpath( + args["reedscase"], "reeds", "resource_adequacy", "reeds2pras", "src", "ReEDS2PRAS.jl" + )) #%% Run it main(args) diff --git a/ReEDS_Augur/stress_periods.py b/reeds/resource_adequacy/stress_periods.py similarity index 83% rename from ReEDS_Augur/stress_periods.py rename to reeds/resource_adequacy/stress_periods.py index e643db68..8d159deb 100644 --- a/ReEDS_Augur/stress_periods.py +++ b/reeds/resource_adequacy/stress_periods.py @@ -1,20 +1,14 @@ #%%### General imports import os -import site import traceback import pandas as pd import numpy as np from glob import glob import re import matplotlib.pyplot as plt -### Local imports - -## use this to import reeds when running locally for debugging -# import site -# this_dir_path = os.path.dirname(os.path.realpath(__file__)) -# site.addsitedir(os.path.join(this_dir_path, "..")) import reeds +from reeds.input_processing import hourly_writetimeseries # #%% Debugging # sw['reeds_path'] = os.path.expanduser('~/github/ReEDS/') @@ -24,6 +18,123 @@ #%%### Functions +def get_pras_eue(case, t, iteration=0): + """ + """ + ### Get PRAS outputs + dfpras = reeds.io.read_pras_results( + os.path.join(case, 'handoff', 'PRAS', f"PRAS_{t}i{iteration}.h5") + ) + ### Create the time index + sw = reeds.io.get_switches(case) + dfpras.index = reeds.timeseries.get_timeindex(sw['resource_adequacy_years']) + + ### Keep the EUE columns by zone + eue_tail = '_EUE' + dfeue = dfpras[[ + c for c in dfpras + if (c.endswith(eue_tail) and not c.startswith('USA')) + ]].copy() + ## Drop the tailing _EUE + dfeue = dfeue.rename( + columns=dict(zip(dfeue.columns, [c[:-len(eue_tail)] for c in dfeue]))) + + return dfeue + + +def get_eue_periods( + case, t, iteration=0, + hierarchy_level='transgrp', + stress_metric='EUE', + period_agg_method='sum', + ): + """_summary_ + + Args: + sw (pd.series): ReEDS switches for this run. + t (int): Model solve year. + iteration (int, optional): Iteration number of this solve year. Defaults to 0. + hierarchy_level (str, optional): column of hierarchy.csv specifying the spatial + level over which to calculate stress_metric. Defaults to 'country'. + stress_metric (str, optional): 'EUE' or 'NEUE'. Defaults to 'EUE'. + period_agg_method (str, optional): 'sum' or 'max', indicating how to aggregate + over the hours in each period. Defaults to 'sum'. + + Raises: + NotImplementedError: if invalid value for stress_metric or GSw_PRM_StressModel + + Returns: + pd.DataFrame: Table of periods sorted in descending order by stress metric. + """ + sw = reeds.io.get_switches(case) + ### Get the region aggregator + rmap = reeds.io.get_rmap(case=case, hierarchy_level=hierarchy_level) + + ### Get EUE from PRAS + dfeue = get_pras_eue(case=case, t=t, iteration=iteration) + ## Aggregate to hierarchy_level + dfeue = ( + dfeue + .rename_axis('r', axis=1).rename_axis('h', axis=0) + .rename(columns=rmap).groupby(axis=1, level=0).sum() + ) + + ###### Calculate the stress metric by period + if stress_metric.upper() == 'EUE': + ### Aggregate according to period_agg_method + dfmetric_period = ( + dfeue + .groupby([dfeue.index.year, dfeue.index.month, dfeue.index.day]) + .agg(period_agg_method) + .rename_axis(['y','m','d']) + ) + elif stress_metric.upper() == 'NEUE': + ### Get load at hierarchy_level + dfload = reeds.io.read_h5py_file( + os.path.join( + case,'handoff','reeds_data',f'pras_load_{t}.h5') + ).rename(columns=rmap).groupby(level=0, axis=1).sum() + dfload.index = dfeue.index + + ### Recalculate NEUE [ppm] and aggregate appropriately + if period_agg_method == 'sum': + dfmetric_period = ( + dfeue + .groupby([dfeue.index.year, dfeue.index.month, dfeue.index.day]) + .agg(period_agg_method) + .rename_axis(['y','m','d']) + ) / ( + dfload + .groupby([dfload.index.year, dfload.index.month, dfload.index.day]) + .agg(period_agg_method) + .rename_axis(['y','m','d']) + ) * 1e6 + elif period_agg_method == 'max': + dfmetric_period = ( + (dfeue / dfload) + .groupby([dfeue.index.year, dfeue.index.month, dfeue.index.day]) + .agg(period_agg_method) + .rename_axis(['y','m','d']) + ) * 1e6 + + ### Sort and drop zeros and duplicates + dfmetric_top = ( + dfmetric_period.stack('r') + .sort_values(ascending=False) + .replace(0,np.nan).dropna() + .reset_index().drop_duplicates(['y','m','d'], keep='first') + .set_index(['y','m','d','r']).squeeze(1).rename(stress_metric) + .reset_index('r') + ) + ## Convert to timestamp, then to ReEDS period + dfmetric_top['actual_period'] = [ + reeds.timeseries.timestamp2h(pd.Timestamp(*d), sw['GSw_HourlyType']).split('h')[0] + for d in dfmetric_top.index.values + ] + + return dfmetric_top + + def plot_eue_diagnostics(sw, t, iteration, high_eue_periods): try: dates = ( @@ -45,7 +156,7 @@ def plot_eue_diagnostics(sw, t, iteration, high_eue_periods): vmax=vmax[outage_type], ) plt.savefig( - os.path.join(sw.casedir, 'outputs', 'Augur_plots', savename) + os.path.join(sw.casedir, 'outputs', 'figures', 'resource_adequacy', savename) ) plt.close() except Exception as err: @@ -64,7 +175,7 @@ def get_and_write_neue(sw, write=True): """ infiles = [ i for i in sorted(glob( - os.path.join(sw['casedir'], 'ReEDS_Augur', 'PRAS', 'PRAS_*.h5'))) + os.path.join(sw['casedir'], 'handoff', 'PRAS', 'PRAS_*.h5'))) if re.match(r"PRAS_[0-9]+i[0-9]+.h5", os.path.basename(i)) ] eue = {} @@ -94,12 +205,12 @@ def get_annual_neue(case, t, iteration=0): """ """ ### Get EUE from PRAS - dfeue = reeds.ra.get_pras_eue(case=case, t=t, iteration=iteration) + dfeue = get_pras_eue(case=case, t=t, iteration=iteration) ### Get load (for calculating NEUE) dfload = reeds.io.read_h5py_file( os.path.join( - case,'ReEDS_Augur','augur_data',f'pras_load_{t}.h5') + case,'handoff','reeds_data',f'pras_load_{t}.h5') ) dfload.index = dfeue.index @@ -204,7 +315,7 @@ def get_shoulder_periods(sw, criterion, dfenergy_r, high_eue_periods): def get_eue_sorted_periods(sw, t, iteration): ### Get storage state of charge (SOC) to use in selection of "shoulder" stress periods dfenergy = reeds.io.read_pras_results( - os.path.join(sw['casedir'], 'ReEDS_Augur', 'PRAS', f"PRAS_{t}i{iteration}-energy.h5") + os.path.join(sw['casedir'], 'handoff', 'PRAS', f"PRAS_{t}i{iteration}-energy.h5") ) timeindex = reeds.timeseries.get_timeindex(sw['resource_adequacy_years']) dfenergy.index = timeindex @@ -236,7 +347,7 @@ def get_eue_sorted_periods(sw, t, iteration): ## Example: criterion = 'transgrp_10_EUE_sum' (hierarchy_level, ppm, stress_metric, period_agg_method) = criterion.split('_') - eue_periods = reeds.ra.get_eue_periods( + eue_periods = get_eue_periods( case=sw.casedir, t=t, iteration=iteration, hierarchy_level=hierarchy_level, stress_metric=stress_metric, @@ -364,7 +475,7 @@ def prm_increment_pras(sw, t, iteration, combined_periods_write, failed_regions) ## shortfall data # read the net shortfall (positive) and net surplus (negative) results # by sample from PRAS run (MWh) - filepath = os.path.join(sw['casedir'], 'ReEDS_Augur', 'PRAS', + filepath = os.path.join(sw['casedir'], 'handoff', 'PRAS', f'PRAS_{sw["t"]}i{iteration}-shortfall_samples.h5') net_short = reeds.io.read_pras_results(filepath) # get number of samples @@ -387,7 +498,7 @@ def prm_increment_pras(sw, t, iteration, combined_periods_write, failed_regions) ## get load data dfload = reeds.io.read_file( os.path.join( - sw['casedir'],'ReEDS_Augur','augur_data',f'pras_load_{t}.h5'), + sw['casedir'],'handoff','reeds_data',f'pras_load_{t}.h5'), parse_timestamps=True ) @@ -536,11 +647,6 @@ def update_prm(sw, t, iteration, failed, combined_periods_write): def main(sw, t, iteration=0, logging=True): """ """ - #%% More imports and settings - site.addsitedir(os.path.join(sw['casedir'],'input_processing')) - import hourly_writetimeseries - newstresspath = f'stress{t}i{iteration+1}' - #%% Write consolidated NEUE so far try: _neue_simple = get_and_write_neue(sw, write=True) @@ -573,6 +679,7 @@ def main(sw, t, iteration=0, logging=True): return #%% Write timeseries data for stress periods for the next iteration of ReEDS + newstresspath = f'stress{t}i{iteration+1}' hourly_writetimeseries.main( sw=sw, reeds_path=sw['reeds_path'], inputs_case=os.path.join(sw['casedir'], 'inputs_case'), diff --git a/reeds/output_calc.py b/reeds/results.py similarity index 99% rename from reeds/output_calc.py rename to reeds/results.py index 90f585dc..92ab66cc 100644 --- a/reeds/output_calc.py +++ b/reeds/results.py @@ -10,7 +10,6 @@ import os import sys import pandas as pd -import numpy as np from itertools import product sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) @@ -660,7 +659,7 @@ def calc_transmission_capacity(case,levels): # pull intra and interregional transmission data into dictionaries for processing dict_inter = {} - level_map = reeds.output_calc.get_level_map(case) + level_map = get_level_map(case) hierarchy = reeds.io.get_hierarchy(case) for level in levels: @@ -679,7 +678,7 @@ def calc_transmission_capacity(case,levels): dict_inter[spatial_resolution] = pd.concat({ r: ( trans_r.loc[ - (trans_r[f'inter_transreg'] == 1) + (trans_r['inter_transreg'] == 1) ].groupby(['trtype','t']).MW.sum() .rename('GW') / 1e3 diff --git a/cbc.opt b/reeds/solver/cbc.opt similarity index 100% rename from cbc.opt rename to reeds/solver/cbc.opt diff --git a/cplex.op2 b/reeds/solver/cplex.op2 similarity index 100% rename from cplex.op2 rename to reeds/solver/cplex.op2 diff --git a/cplex.opt b/reeds/solver/cplex.opt similarity index 100% rename from cplex.opt rename to reeds/solver/cplex.opt diff --git a/gurobi.opt b/reeds/solver/gurobi.opt similarity index 100% rename from gurobi.opt rename to reeds/solver/gurobi.opt diff --git a/runbatch.py b/runreeds.py similarity index 91% rename from runbatch.py rename to runreeds.py index bfc9ca75..6bcd6664 100644 --- a/runbatch.py +++ b/runreeds.py @@ -2,6 +2,7 @@ ### --- IMPORTS --- ### =========================================================================== +import reeds import os import git import queue @@ -17,7 +18,6 @@ from datetime import datetime import argparse from pathlib import Path -import reeds # Assert core programs are accessible CORE_PROGRAMS = ["gams"] @@ -532,62 +532,6 @@ def check_compatibility(sw): raise ModuleNotFoundError(err) -def solvestring_sequential( - batch_case, caseSwitches, - cur_year, next_year, prev_year, restartfile, - toLogGamsString=' logOption=4 logFile=gamslog.txt appendLog=1 ', - hpc=0, iteration=0, stress_year=None, - temporal_inputs='rep', - ): - """ - Typical inputs: - * restartfile: batch_case if first solve year else {batch_case}_{prev_year} - * caseSwitches: loaded from {batch_case}/inputs_case/switches.csv - """ - savefile = f"{batch_case}_{cur_year}i{iteration}" - _stress_year = f"{cur_year}i0" if stress_year is None else stress_year - out = ( - "gams d_solveoneyear.gms" - + (" license=gamslice.txt" if hpc else '') - + " o=" + os.path.join("lstfiles", f"{savefile}.lst") - + " r=" + os.path.join("g00files", restartfile) - + " gdxcompress=1" - + " xs=" + os.path.join("g00files", savefile) - + toLogGamsString - + f" --case={batch_case}" - + f" --cur_year={cur_year}" - + f" --next_year={next_year}" - + f" --prev_year={prev_year}" - + f" --stress_year={_stress_year}" - + f" --temporal_inputs={temporal_inputs}" - + ''.join([f" --{s}={caseSwitches[s]}" for s in [ - 'GSw_Canada', - 'GSw_ClimateHydro', - 'GSw_ClimateWater', - 'GSw_gopt', - 'GSw_HourlyChunkLengthRep', - 'GSw_HourlyChunkLengthStress', - 'GSw_HourlyType', - 'GSw_HourlyWrapLevel', - 'GSw_MGA_CostDelta', - 'GSw_MGA_Direction', - 'GSw_PVB_Dur', - 'GSw_SkipAugurYear', - 'GSw_StateCO2ImportLevel', - 'GSw_StartMarkets', - 'GSw_ValStr', - 'solver', - 'debug', - 'startyear', - 'diagnose', - 'diagnose_year' - ]]) - + '\n' - ) - - return out - - def setup_sequential_year( cur_year, prev_year, next_year, caseSwitches, hpc, @@ -601,12 +545,12 @@ def setup_sequential_year( if (cur_year >= min(solveyears)): ## solve one year OPATH.writelines( - solvestring_sequential( + reeds.inputs.solvestring_sequential( batch_case, caseSwitches, cur_year, next_year, prev_year, restartfile, toLogGamsString, hpc, )) - OPATH.writelines(writescripterrorcheck(f"d_solveoneyear.gms_{cur_year}")) + OPATH.writelines(writescripterrorcheck(f"3_solve_oneyear.gms_{cur_year}")) OPATH.writelines(f'python {logger} --year={cur_year}\n') if int(caseSwitches['GSw_ValStr']): @@ -615,17 +559,17 @@ def setup_sequential_year( ## check to see if the restart file exists OPATH.writelines(writeerrorcheck(os.path.join("g00files", savefile + ".g*"))) - ## Run Augur if it not the final solve year and if not skipping Augur + ## Run resource adequacy (RA) calculations if it not the final solve year and if not skipping RA if (( (cur_year < max(solveyears)) - and (next_year > int(caseSwitches['GSw_SkipAugurYear'])) + and (next_year > int(caseSwitches['GSw_SkipRAyear'])) ) or (cur_year == max(solveyears))): OPATH.writelines( - f"\npython Augur.py {next_year} {cur_year} {casedir}\n") - ## Check to make sure Augur ran successfully; quit otherwise + f"\npython {Path('reeds', 'resource_adequacy', 'ra_calcs.py')} {next_year} {cur_year} {casedir}\n") + ## Check to make sure RA ran successfully; quit otherwise OPATH.writelines( writeerrorcheck(os.path.join( - "ReEDS_Augur", "augur_data", f"ReEDS_Augur_{cur_year}.gdx"))) + "handoff", "reeds_data", f"ccdata_{cur_year}.gdx"))) ## delete the previous restart file unless we're keeping them if (cur_year > min(solveyears)) and (not int(caseSwitches['keep_g00_files'])): @@ -658,14 +602,14 @@ def setup_sequential( big_comment(f'Year: {cur_year}', OPATH) ### Write the tax credit phaseout call - OPATH.writelines(f"python tc_phaseout.py {cur_year} {casedir}\n\n") + OPATH.writelines(f"python {Path('reeds','core','solve','1_tc_phaseout.py')} {cur_year} {casedir}\n\n") - ### Write the GAMS LP and Augur calls + ### Write the GAMS LP and resource adequacy calls if int(caseSwitches['GSw_PRM_StressIterateMax']): OPATH.writelines( - f"python d_solve_iterate.py {casedir} {cur_year}\n" + f"python {Path('reeds','core','solve','solve.py')} {casedir} {cur_year}\n" ) - OPATH.writelines(writescripterrorcheck(f"d_solve_iterate.py_{cur_year}")) + OPATH.writelines(writescripterrorcheck(f"solve.py_{cur_year}")) else: setup_sequential_year( cur_year, prev_year, next_year, @@ -678,24 +622,22 @@ def setup_sequential( ### multipliers aren't created until the first solve year is run) if cur_year == min(solveyears): OPATH.writelines( - f"\npython {os.path.join(casedir, 'input_processing', 'check_inputs.py')} " + f"\npython {os.path.join(casedir, 'reeds', 'input_processing', 'check_inputs.py')} " f"{casedir}\n" ) OPATH.writelines(writescripterrorcheck('check_inputs.py')+'\n') - ### Run Augur plots in background + ### Run resource adequacy plots in background OPATH.writelines( - f"python {os.path.join('ReEDS_Augur','diagnostic_plots.py')} " + f"python {Path('reeds','resource_adequacy','diagnostic_plots.py')} " f"--reeds_path={reeds_path} --casedir={casedir} --t={cur_year} &\n") def setup_intertemporal( caseSwitches, startiter, niter, ccworkers, - solveyears, endyear, batch_case, toLogGamsString, yearset_augur, OPATH, + solveyears, endyear, batch_case, toLogGamsString, modeledyears, OPATH, ): - ### beginning year is passed to augurbatch - begyear = min(solveyears) - ### first save file from d_solveprep is just the case name + ### first save file from e_solveprep is just the case name savefile = batch_case ### if this is the first iteration if startiter == 0: @@ -724,7 +666,8 @@ def setup_intertemporal( OPATH.writelines(writeerrorcheck(os.path.join("g00files", restartfile + ".g*"))) OPATH.writelines( - "gams d_solveallyears.gms o="+os.path.join("lstfiles",batch_case + "_" + str(i) + ".lst") + f"gams {Path('reeds','core','3_solve_allyears.gms')} o=" + +os.path.join("lstfiles",batch_case + "_" + str(i) + ".lst") +" r="+os.path.join("g00files", restartfile) + " gdxcompress=1 xs="+os.path.join("g00files", savefile) + toLogGamsString + " --niter=" + str(i) + " --case=" + batch_case + ' \n') @@ -732,29 +675,19 @@ def setup_intertemporal( ## check to see if the save file exists OPATH.writelines(writeerrorcheck(os.path.join("g00files",savefile + ".g*"))) - ## start threads for cc/curt - ## no need to run cc curt scripts for final iteration + ## Run resource adequacy calculations (no need to run for final iteration) if i < niter-1: - ## batch out calls to augurbatch - OPATH.writelines( - "python augurbatch.py " + batch_case + " " + str(ccworkers) + " " - + yearset_augur + " " + savefile + " " + str(begyear) + " " - + str(endyear) + " " + caseSwitches['distpvscen'] + " " - + str(caseSwitches['calc_csp_cc']) + " " - + str(caseSwitches['timetype']) + " " - + str(caseSwitches['GSw_WaterMain']) + " " + str(i) + " " - + str(caseSwitches['marg_vre_mw']) + " " - + str(caseSwitches['marg_stor_mw']) + " " - + str(caseSwitches['marg_evmc_mw']) + " " - + '\n') + ## TODO: Run the RA calculations via ra_calcs.py for each solve year + ## (needs to be reimplemented) + ## merge all the resulting gdx files ## the output file will be for the next iteration nextiter = i+1 gdxmergedfile = os.path.join( - "ReEDS_Augur","augur_data","ReEDS_Augur_merged_" + str(nextiter)) + 'handoff', 'reeds_data', f'ccdata_merged_{nextiter}') OPATH.writelines( - "gdxmerge "+os.path.join("ReEDS_Augur","augur_data","ReEDS_Augur*") - + " output=" + gdxmergedfile + ' \n') + 'gdxmerge ' + os.path.join('handoff', 'reeds_data', 'ccdata*') + + f' output={gdxmergedfile} \n') ## check to make sure previous calls were successful OPATH.writelines(writeerrorcheck(gdxmergedfile+".gdx")) @@ -767,7 +700,7 @@ def setup_intertemporal( def setup_window( caseSwitches, startiter, niter, ccworkers, reeds_path, - batch_case, toLogGamsString, yearset_augur, OPATH, + batch_case, toLogGamsString, modeledyears, OPATH, ): ### load the windows win_in = list(csv.reader(open( @@ -780,12 +713,6 @@ def setup_window( ### for windows indicated in the csv file for win in win_in[1:]: - - ## beginning year is the first column (start) - begyear = win[1] - ## end year is the second column (end) - endyear = win[2] - ## for the number of iterations we have... for i in range(startiter,niter): big_comment(f'Window: {win}', OPATH) comment(f'Iteration: {i}', OPATH) @@ -796,32 +723,25 @@ def setup_window( OPATH.writelines(writeerrorcheck(os.path.join("g00files", restartfile + ".g*"))) ## solve via the window solve file OPATH.writelines( - "gams d_solvewindow.gms o=" + os.path.join("lstfiles", batch_case + "_" + str(i) + ".lst") + f"gams {Path('reeds','core','3_solvewindow.gms')} o=" + + os.path.join("lstfiles", batch_case + "_" + str(i) + ".lst") +" r=" + os.path.join("g00files", restartfile) + " gdxcompress=1 xs=g00files\\"+savefile + toLogGamsString + " --niter=" + str(i) + " --maxiter=" + str(niter-1) + " --case=" + batch_case + " --window=" + win[0] + ' \n') ## start threads for cc/curt OPATH.writelines(writeerrorcheck(os.path.join("g00files",savefile + ".g*"))) - OPATH.writelines( - "python augurbatch.py " + batch_case + " " + str(ccworkers) + " " - + yearset_augur + " " + savefile + " " + str(begyear) + " " - + str(endyear) + " " + caseSwitches['distpvscen'] + " " - + str(caseSwitches['calc_csp_cc']) + " " - + str(caseSwitches['timetype']) + " " - + str(caseSwitches['GSw_WaterMain']) + " " + str(i) + " " - + str(caseSwitches['marg_vre_mw']) + " " - + str(caseSwitches['marg_stor_mw']) + " " - + str(caseSwitches['marg_evmc_mw']) + " " - + '\n') + ## TODO: Run the RA calculations via ra_calcs.py for each solve year + ## in window (needs to be reimplemented) + ## merge all the resulting r2_in gdx files ## the output file will be for the next iteration nextiter = i+1 ## create names for then merge the curt and cc gdx files gdxmergedfile = os.path.join( - "ReEDS_Augur","augur_data","ReEDS_Augur_merged_" + str(nextiter)) + 'handoff', 'reeds_data', f'ccdata_merged_{nextiter}') OPATH.writelines( - "gdxmerge " + os.path.join("ReEDS_Augur","augur_data","ReEDS_Augur*") - + " output=" + gdxmergedfile + ' \n') + 'gdxmerge ' + os.path.join('handoff', 'reeds_data', 'ccdata*') + + f' output={gdxmergedfile} \n') ## check to make sure previous calls were successful OPATH.writelines(writeerrorcheck(gdxmergedfile+".gdx")) restartfile = savefile @@ -1168,6 +1088,7 @@ def write_batch_script( #%% Set up case-specific directory structure os.makedirs(inputs_case, exist_ok=True) + os.makedirs(os.path.join(casedir, 'autocode'), exist_ok=True) os.makedirs(os.path.join(casedir, 'g00files'), exist_ok=True) os.makedirs(os.path.join(casedir, 'lstfiles'), exist_ok=True) os.makedirs(os.path.join(casedir, 'outputs', 'figures'), exist_ok=True) @@ -1258,7 +1179,7 @@ def write_batch_script( shutil.copy2(os.path.join(reeds_path, cases_filename), casedir) ### Switches with values derived from other switches - ## Get hpc setting (used in Augur) + ## Determine whether we're running on the HPC caseSwitches['hpc'] = int(hpc) ## Get numclass from the max value in ivt caseSwitches['numclass'] = get_ivt_numclass( @@ -1285,20 +1206,19 @@ def write_batch_script( solveyears = [y for y in solveyears if (y <= endyear and y >= startyear)] - yearset_augur = os.path.join('inputs_case','modeledyears.csv') + modeledyears = os.path.join('inputs_case','modeledyears.csv') toLogGamsString = ' logOption=4 logFile=gamslog.txt appendLog=1 ' - ## Copy code folders - for dirname in ['reeds', 'ReEDS_Augur', 'input_processing', 'reeds2pras']: - shutil.copytree( - os.path.join(reeds_path, dirname), - os.path.join(casedir, dirname), - ignore=shutil.ignore_patterns('test'), - ) + ## Copy code folder + shutil.copytree( + os.path.join(reeds_path, 'reeds'), + os.path.join(casedir, 'reeds'), + ignore=shutil.ignore_patterns('test'), + ) - #make the augur_data folder - os.makedirs(os.path.join(casedir,'ReEDS_Augur','augur_data'), exist_ok=True) - os.makedirs(os.path.join(casedir,'ReEDS_Augur','PRAS'), exist_ok=True) + #make the reeds_data folder + os.makedirs(os.path.join(casedir,'handoff','reeds_data'), exist_ok=True) + os.makedirs(os.path.join(casedir,'handoff','PRAS'), exist_ok=True) ###### Replace files according to 'file_replacements' in cases. Ignore quotes in input text. # << is used to separate the file that is to be replaced from the file that is used @@ -1368,7 +1288,7 @@ def write_batch_script( OPATH.writelines(f"echo 'starting {s}.py'\n") OPATH.writelines(f"echo {'-'*12+'-'*len(s)}\n") OPATH.writelines( - f"python {os.path.join(casedir,'input_processing',s)}.py {reeds_path} {inputs_case}\n") + f"python {os.path.join(casedir,'reeds','input_processing',s)}.py {reeds_path} {inputs_case}\n") OPATH.writelines(writescripterrorcheck(s)+'\n') OPATH.writelines( @@ -1386,7 +1306,8 @@ def write_batch_script( big_comment('Compile model', OPATH) OPATH.writelines( - "\ngams createmodel.gms gdxcompress=1 xs="+os.path.join("g00files",batch_case) + f"\ngams {Path('reeds','core','setup','a_createmodel.gms')} " + + "gdxcompress=1 xs="+os.path.join("g00files",batch_case) + (' license=gamslice.txt' if hpc else '') + " o="+os.path.join("lstfiles","1_Inputs.lst") + options + " " + toLogGamsString + '\n') OPATH.writelines(f'python {logger}\n') @@ -1404,12 +1325,12 @@ def write_batch_script( elif caseSwitches['timetype'] == 'int': setup_intertemporal( caseSwitches, startiter, niter, ccworkers, - solveyears, endyear, batch_case, toLogGamsString, yearset_augur, OPATH, + solveyears, endyear, batch_case, toLogGamsString, modeledyears, OPATH, ) elif caseSwitches['timetype'] == 'win': setup_window( caseSwitches, startiter, niter, ccworkers, reeds_path, - batch_case, toLogGamsString, yearset_augur, OPATH, + batch_case, toLogGamsString, modeledyears, OPATH, ) ################################# @@ -1428,15 +1349,16 @@ def write_batch_script( ) ### Otherwise, run for the last iteration (selected numerically) else: + fpath = os.path.join(casedir, "reeds", "core", "terminus", "get_last_iter.py") OPATH.writelines( - f'r=$(python {os.path.join(casedir, "reeds", "get_last_iter.py")} {batch_case} {max(solveyears)})\n' + f'r=$(python {fpath} {batch_case} {max(solveyears)})\n' if LINUXORMAC else f'for /f "delims=" %%i in ' - f'(\'python {os.path.join(casedir, "reeds", "get_last_iter.py")} {batch_case} {max(solveyears)}\')' + f'(\'python {fpath} {batch_case} {max(solveyears)}\')' f' do set "r=%%i"\n' ) OPATH.writelines( - "gams e_report.gms" + f"gams {Path('reeds','core','terminus','report.gms')}" + f" o={os.path.join('lstfiles',f'report_{batch_case}.lst')}" + (' license=gamslice.txt' if hpc else '') + (' r=$r' if LINUXORMAC else ' r=!r!') @@ -1445,11 +1367,11 @@ def write_batch_script( + f"--fname={batch_case}" + f" --GSw_calc_powfrac={caseSwitches['GSw_calc_powfrac']} \n" ) - OPATH.writelines(writescripterrorcheck("e_report.gms")) + OPATH.writelines(writescripterrorcheck("report.gms")) if not LINUXORMAC: OPATH.writelines("endlocal\n") OPATH.writelines(f'python {logger}\n') - OPATH.writelines(f'python e_report_dump.py {casedir} -c\n\n') + OPATH.writelines(f"python {Path('reeds','core','terminus','report_dump.py')} {casedir} -c\n\n") if int(caseSwitches['diagnose']): OPATH.writelines( "python" @@ -1473,7 +1395,7 @@ def write_batch_script( ### Make script to unload all data to .gdx file command = ( - 'gams dump_alldata.gms' + f"gams {Path('reeds','core','terminus','dump_alldata.gms')}" + ' o='+os.path.join('lstfiles','dump_alldata_{}_{}.lst'.format(BatchName,case)) ) command_write = ( @@ -1564,7 +1486,9 @@ def write_batch_script( ### Run dispatch mode if desired if int(caseSwitches['pcm']): - OPATH.writelines(f'\npython run_pcm.py {casedir} -b\n\n') + OPATH.writelines( + f"\npython {Path('reeds','postprocessing','run_pcm.py')} {casedir} -b\n\n" + ) def submit_slurm_parallel_jobs( @@ -1589,7 +1513,7 @@ def submit_slurm_parallel_jobs( stop_case_index = min(start_case_index + cases_per_node, num_cases) casenames_print = casenames[start_case_index:stop_case_index] run_script_fpath = os.path.join(batch_folder, f"run_{'-'.join(casenames_print)}.sh") - shutil.copy("srun_template.sh", run_script_fpath) + shutil.copy(Path(reeds_path,'reeds','hpc','srun_template.sh'), run_script_fpath) job_name = f"{BatchName}_({','.join(casenames_print)})" writelines = [] @@ -1659,7 +1583,7 @@ def write_case_submission_script( """ # Create a copy of the SLURM template slurm_script_path = os.path.join(casedir, batch_case + ".sh") - shutil.copy("srun_template.sh", slurm_script_path) + shutil.copy(Path(reeds.io.reeds_path,'reeds','hpc','srun_template.sh'), slurm_script_path) # If using debug node, comment out time and replace with short time if debugnode: diff --git a/tests/objective_function_params.yaml b/tests/objective_function_params.yaml index 1764036c..8ffddb18 100644 --- a/tests/objective_function_params.yaml +++ b/tests/objective_function_params.yaml @@ -1,6 +1,6 @@ ### Notes # This file holds parameters used in the objective function. These parameters -# are checked using input_processing/check_inputs.py, which will identify missing +# are checked using reeds/input_processing/check_inputs.py, which will identify missing # values in the parameters if there are any. # Instructions: @@ -29,12 +29,12 @@ # Note that in many cases the specific configuration here does not match the GAMS # code. For example, the check for cost_cap excludes hydro and psh, even though -# those are not explicitly excluded in c_supplyobjective.gms. When values are +# those are not explicitly excluded in d_objective.gms. When values are # known to be zero they must be excluded otherwise they will be flagged as missing # (GAMS does not hold zero values, so values explicitly set to zero will appear # the same as missing values). -# The ordering of parameters here should follow the ordering in c_supplyobjective.gms. +# The ordering of parameters here should follow the ordering in d_objective.gms. # Most general-purpose parameters used in the objective function should be added to # this file. Parameters that are primarily user-defined or that define their own scope # without the use of separate index sets are not tested here. Examples of such skipped diff --git a/tests/test_outputs.py b/tests/test_outputs.py index cdd56086..c6a73abc 100644 --- a/tests/test_outputs.py +++ b/tests/test_outputs.py @@ -6,17 +6,17 @@ def test_output_files(casepath): outputs_folder = os.path.join(casepath, 'outputs') - e_report_params_path = os.path.join(casepath, 'e_report_params.csv') + report_params_path = os.path.join(casepath, 'reeds', 'core', 'terminus', 'report_params.csv') lastyear = reeds.io.get_years(casepath)[-1] - # each parameter in e_report_params.csv should be associated with an output csv file - e_report_params = pd.read_csv(e_report_params_path, comment='#') - e_report_params['fpath'] = e_report_params.param.map(lambda x: x.split('(')[0]) + # each parameter in report_params.csv should be associated with an output csv file + report_params = pd.read_csv(report_params_path, comment='#') + report_params['fpath'] = report_params.param.map(lambda x: x.split('(')[0]) # rename outputs as specified by output_rename column - rename = e_report_params.loc[ - ~e_report_params.output_rename.isnull() + rename = report_params.loc[ + ~report_params.output_rename.isnull() ].set_index('fpath').output_rename.to_dict() - e_report_params['fpath'] = e_report_params['fpath'].replace(rename) + '.csv' + report_params['fpath'] = report_params['fpath'].replace(rename) + '.csv' # Include additional files in outputs folder that should be generated for each run expected_plots = [ @@ -29,7 +29,7 @@ def test_output_files(casepath): ] expected_files = ( - e_report_params.fpath.tolist() + report_params.fpath.tolist() + [ # Standard bokeh outputs (postprocessing/bokehpivot) os.path.join('reeds-report', 'report.html'),