From e1e58aa1d0cc6c4d2d558db64f46351b68eb9fdf Mon Sep 17 00:00:00 2001 From: Patrick Brown <25125211+patrickbrown4@users.noreply.github.com> Date: Fri, 8 May 2026 14:32:36 -0600 Subject: [PATCH 1/7] update geospatial packages --- environment.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/environment.yml b/environment.yml index c0161fcb..2c2ec092 100644 --- a/environment.yml +++ b/environment.yml @@ -6,6 +6,7 @@ dependencies: - python=3.11 - bokeh=3.2 - click=8.0 + - geopandas=1.1 - git-lfs=2.13 - gitpython=3.1 - h5py=3.9 @@ -19,9 +20,12 @@ dependencies: - requests=2.31 - scikit-learn=1.2 - scipy=1.11 + - shapely=2.1 - tqdm=4.65 ## vvv The next packages are optional for default settings and may ## vvv be excluded if you're having trouble building the environment. + - fiona=1.10 # for interactive maps + - folium=0.20 # for interactive maps - ipykernel=6.25 # for interactive python in VS Code - ipywidgets=8.0 # for jupyter notebooks - mapclassify=2.5 # more mapping tools @@ -32,17 +36,11 @@ dependencies: - pip: - cmocean==3.0.3 - gdxpds==1.4.0 - - geopandas==0.14.0 - pulp==2.7.0 - - shapely==2.0.1 ## vvv The next packages are optional for default settings and may ## vvv be excluded if they cause trouble building the environment. - - fiona==1.9.5 # for interactive maps - - folium==0.14.0 # for interactive maps - gamspy_base==50.5.0 # for tests on the github runner - myst-parser==2.0.0 # for building documentation - - proj==0.2.0 # for plotting maps - - pyproj==3.6.1 # for plotting maps - python-pptx==0.6.22 # for postprocessing/compare_cases.py - sphinx_rtd_theme==2.0.0 # for documentation - sphinx==7.2.6 # for building documentation From 2ff493b575206027b5caa368bcb8ca9fedc3d9bb Mon Sep 17 00:00:00 2001 From: Patrick Brown <25125211+patrickbrown4@users.noreply.github.com> Date: Fri, 8 May 2026 14:49:39 -0600 Subject: [PATCH 2/7] update all python packages and use conda-forge for everything --- environment.yml | 62 ++++++++++++++++++++++++------------------------- 1 file changed, 30 insertions(+), 32 deletions(-) diff --git a/environment.yml b/environment.yml index 2c2ec092..0ce620b1 100644 --- a/environment.yml +++ b/environment.yml @@ -1,48 +1,46 @@ -name: reeds2 +name: reeds channels: - - defaults - conda-forge dependencies: - - python=3.11 - - bokeh=3.2 - - click=8.0 + - python=3.14 + - bokeh=3.9 + - click=8.3 + - cmocean=4.0 - geopandas=1.1 - - git-lfs=2.13 + - git-lfs=3.7 - gitpython=3.1 - - h5py=3.9 - - matplotlib=3.7 - - numpy=1.26 - - openpyxl=3.0 - - pandas=2.0 - - pip=23.2 - - pytables=3.8 - - pytest=7.4 - - requests=2.31 - - scikit-learn=1.2 - - scipy=1.11 + - h5py=3.16 + - matplotlib=3.10 + - numpy=2.4 + - openpyxl=3.1 + - pandas=3.0 + - pip=26.1 + - pulp=2.8 + - pytables=3.11 + - pytest=9.0 + - requests=2.33 + - scikit-learn=1.8 + - scipy=1.17 - shapely=2.1 - - tqdm=4.65 + - tqdm=4.67 ## vvv The next packages are optional for default settings and may ## vvv be excluded if you're having trouble building the environment. - fiona=1.10 # for interactive maps - folium=0.20 # for interactive maps - - ipykernel=6.25 # for interactive python in VS Code - - ipywidgets=8.0 # for jupyter notebooks - - mapclassify=2.5 # more mapping tools + - ipykernel=7.2 # for interactive python in VS Code + - ipywidgets=8.1 # for jupyter notebooks + - mapclassify=2.10 # more mapping tools - mscorefonts=0.0 # extra fonts for plotting (only relevant on linux/HPC) - - networkx=3.1 # for uncommonly-used network analysis postprocessing - - notebook=6.5 # for jupyter notebooks + - myst-parser=5.0 # for building documentation + - networkx=3.6 # for uncommonly-used network analysis postprocessing + - notebook=7.5 # for jupyter notebooks + - python-pptx=1.0 # for postprocessing/compare_cases.py + - sphinx=9.1 # for building documentation + - sphinx_rtd_theme=3.1 # for documentation + - sphinxcontrib-bibtex=2.6 # for documentation ## ^^^ - pip: - - cmocean==3.0.3 - gdxpds==1.4.0 - - pulp==2.7.0 ## vvv The next packages are optional for default settings and may ## vvv be excluded if they cause trouble building the environment. - - gamspy_base==50.5.0 # for tests on the github runner - - myst-parser==2.0.0 # for building documentation - - python-pptx==0.6.22 # for postprocessing/compare_cases.py - - sphinx_rtd_theme==2.0.0 # for documentation - - sphinx==7.2.6 # for building documentation - - sphinxcontrib-bibtex==2.6.2 # for documentation - ## ^^^ + - gamspy_base==53.5.0 # for tests on the github runner From bf4fd0433cc25dd340af41869b73bad474893c79 Mon Sep 17 00:00:00 2001 From: Patrick Brown <25125211+patrickbrown4@users.noreply.github.com> Date: Wed, 20 May 2026 15:47:43 -0600 Subject: [PATCH 3/7] update gdxpds, add gams.transfer, add highspy --- environment.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 0ce620b1..31b5dd0e 100644 --- a/environment.yml +++ b/environment.yml @@ -40,7 +40,9 @@ dependencies: - sphinxcontrib-bibtex=2.6 # for documentation ## ^^^ - pip: - - gdxpds==1.4.0 + - gamsapi[transfer]==53.5.0 + - gdxpds==1.5.0 ## vvv The next packages are optional for default settings and may ## vvv be excluded if they cause trouble building the environment. - gamspy_base==53.5.0 # for tests on the github runner + - highspy==1.14.0 # for future solver flexibility From 9c7dcbc4913013651e77c3f6cbf11aa0a6cac50f Mon Sep 17 00:00:00 2001 From: Patrick Brown <25125211+patrickbrown4@users.noreply.github.com> Date: Wed, 20 May 2026 16:09:11 -0600 Subject: [PATCH 4/7] fix reeds.io.get_switches() --- reeds/io.py | 19 ++++++++++--------- runreeds.py | 17 +++++++---------- 2 files changed, 17 insertions(+), 19 deletions(-) diff --git a/reeds/io.py b/reeds/io.py index d4bcd313..d662f5f7 100644 --- a/reeds/io.py +++ b/reeds/io.py @@ -689,23 +689,24 @@ def get_switches(case=None, **kwargs): (case if case is not None else reeds_path), 'reeds', 'resource_adequacy', 'ra_switches.csv', ) - asw = pd.read_csv(fpath_asw, index_col='key') - for i, row in asw.iterrows(): + dfra = pd.read_csv(fpath_asw, index_col='key', dtype='object') + ra_switches = {} + for key, row in dfra.iterrows(): if row['dtype'] == 'list': - row.value = row.value.split(',') + ra_switches[key] = row.value.split(',') try: - row.value = [int(i) for i in row.value] + ra_switches[key] = [int(i) for i in row.value] except ValueError: pass elif row['dtype'] == 'boolean': - row.value = False if row.value.lower() == 'false' else True + ra_switches[key] = False if row.value.lower() == 'false' else True elif row['dtype'] == 'str': - row.value = str(row.value) + ra_switches[key] = str(row.value) elif row['dtype'] == 'int': - row.value = int(row.value) + ra_switches[key] = int(row.value) elif row['dtype'] == 'float': - row.value = float(row.value) - sw = pd.concat([sw, asw.value]) + ra_switches[key] = float(row.value) + sw = pd.concat([sw, pd.Series(ra_switches)]) except FileNotFoundError: print(f"{fpath_asw} not found so leaving out resource adequacy switches") ### Add derivative switches diff --git a/runreeds.py b/runreeds.py index 6f15a290..9845e22d 100644 --- a/runreeds.py +++ b/runreeds.py @@ -826,22 +826,19 @@ def setupEnvironment( #%% Check whether the ReEDS conda environment is activated if (not skip_checks) and ( - ('reeds2' not in os.environ['CONDA_DEFAULT_ENV'].lower()) - or (not pd.__version__.startswith('2')) + ('reeds' not in os.environ['CONDA_DEFAULT_ENV'].lower()) + or (not pd.__version__.startswith('3')) ): - print( + err = ( f"Your environment is {os.environ['CONDA_DEFAULT_ENV']} and your pandas " - f"version is {pd.__version__}.\nThe default environment is 'reeds2', with\n" - "pandas version 2.x, so the python parts of ReEDS are unlikely to work.\n" + f"version is {pd.__version__}.\nThe supported environment is 'reeds', with\n" + "pandas version 3.x.\n" "To build the environment for the first time, run:\n" " `conda env create -f environment.yml`\n" "To activate the created environment, run:\n" - " `conda activate reeds2` (or `activate reeds2` on Windows)\n" - "Do you want to continue without activating the environment?" + " `conda activate reeds` (or `activate reeds` on Windows)" ) - confirm_env = str(input("Continue? y/[n]: ") or 'n') - if confirm_env not in ['y','Y','yes','Yes','YES']: - quit() + raise ValueError(err) #%% Load specified case file, infer other settings from cases.csv if cases_suffix in ['', 'default']: From 508e87ba12b84718558b7a4c2005c84555da5a97 Mon Sep 17 00:00:00 2001 From: Patrick Brown <25125211+patrickbrown4@users.noreply.github.com> Date: Wed, 20 May 2026 16:09:28 -0600 Subject: [PATCH 5/7] use raw strings for regex --- reeds/input_processing/copy_files.py | 4 ++-- reeds/resource_adequacy/prep_data.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/reeds/input_processing/copy_files.py b/reeds/input_processing/copy_files.py index 55a60880..bb6ec1ee 100644 --- a/reeds/input_processing/copy_files.py +++ b/reeds/input_processing/copy_files.py @@ -337,7 +337,7 @@ def get_regions_and_agglevel( # Sort hier_sub by r so that "ord(r)" commands in GAMS result in the properly # ordered outputs - hier_sub['numeric_value'] = hier_sub['r'].str.extract('(\d+)').astype(float) + hier_sub['numeric_value'] = hier_sub['r'].str.extract(r'(\d+)').astype(float) hier_sub = hier_sub.sort_values(by='numeric_value').drop('numeric_value', axis=1) # Output the itlgrp files for mixed and county resolution @@ -900,7 +900,7 @@ def write_scalars(scalars, inputs_case): scalars_write = pd.concat([scalars, toadd], axis=0) # Trim trailing decimal zeros - scalars_write.value = scalars_write.value.astype(str).replace('\.0+$', '', regex=True) + scalars_write.value = scalars_write.value.astype(str).replace(r'\.0+$', '', regex=True) scalars_write.to_csv(os.path.join(inputs_case, 'scalars.csv'), header=False) # Rewrite the scalar tables as GAMS-readable definition diff --git a/reeds/resource_adequacy/prep_data.py b/reeds/resource_adequacy/prep_data.py index f7ad96c6..d30a0d42 100644 --- a/reeds/resource_adequacy/prep_data.py +++ b/reeds/resource_adequacy/prep_data.py @@ -170,7 +170,7 @@ def main(t, casedir, iteration=0): ## e.g. "upv_5" -> "upv", "csp2_3" -> "csp" techs_vre_simplify = dict(zip( techs_vre, - [re.sub('\d?_\d+$', '', i) for i in techs_vre] + [re.sub(r'\d?_\d+$', '', i) for i in techs_vre] )) try: From 4b7d8c7e8f57ba566cc2085fd95035010da093bf Mon Sep 17 00:00:00 2001 From: Patrick Brown <25125211+patrickbrown4@users.noreply.github.com> Date: Wed, 20 May 2026 16:44:21 -0600 Subject: [PATCH 6/7] fix copy_files.py --- environment.yml | 2 +- reeds/input_processing/copy_files.py | 12 ++---------- reeds/io.py | 8 ++++---- 3 files changed, 7 insertions(+), 15 deletions(-) diff --git a/environment.yml b/environment.yml index 31b5dd0e..40d6eeea 100644 --- a/environment.yml +++ b/environment.yml @@ -41,7 +41,7 @@ dependencies: ## ^^^ - pip: - gamsapi[transfer]==53.5.0 - - gdxpds==1.5.0 + - gdxpds==1.6.0 ## vvv The next packages are optional for default settings and may ## vvv be excluded if they cause trouble building the environment. - gamspy_base==53.5.0 # for tests on the github runner diff --git a/reeds/input_processing/copy_files.py b/reeds/input_processing/copy_files.py index bb6ec1ee..3f94da6c 100644 --- a/reeds/input_processing/copy_files.py +++ b/reeds/input_processing/copy_files.py @@ -455,12 +455,7 @@ def read_banned_tech_file(full_path, filepath, inputs_case, r_county): .apply(list) .apply(sorted) ) - r_all_counties_map = ( - r_county.groupby('r') - ['county'] - .apply(list) - .apply(sorted) - ) + r_all_counties_map = r_county.county.tolist() county_ban_regions = list( r_ban_counties_map .loc[(r_ban_counties_map.isin(r_all_counties_map))] @@ -492,11 +487,8 @@ def read_special_h5file(full_path): df.columns = df.columns.str.replace('distpv|','') elif filename.startswith('transmission_cost_ac'): df = df.reset_index() - for col in ['r', 'rr', 'tscbin']: - df[col] = df[col].str.decode('utf-8') elif filename.startswith('transmission_distance'): df = df.stack().rename('miles').reset_index() - df['r'] = df['r'].str.decode('utf-8') return df @@ -1673,7 +1665,7 @@ def main(reeds_path, inputs_case): # #%% Settings for testing ### # reeds_path = reeds.io.reeds_path - # inputs_case = os.path.join(reeds_path,'runs','v20260305_itlM0_USA_defaults','inputs_case') + # inputs_case = os.path.join(reeds_path,'runs','v20260520_envM0_Pacific','inputs_case') # ---- Set up logger ---- diff --git a/reeds/io.py b/reeds/io.py index d662f5f7..5d12d2e8 100644 --- a/reeds/io.py +++ b/reeds/io.py @@ -774,7 +774,7 @@ def get_scalars(case=None, full=False): return scalars -def read_h5py_file(filename, decode_strings=False): +def read_h5py_file(filename): """Return dataframe object for a h5py file. This function returns a pandas dataframe of a h5py file. If the file has multiple dataset on it @@ -822,7 +822,7 @@ def read_h5py_file(filename, decode_strings=False): idx_cols.sort() for idx_col in idx_cols: df[idx_col] = pd.Series(f[idx_col]).values - if str(df[idx_col].dtype).startswith('|S') and decode_strings: + if str(df[idx_col].dtype).startswith('|S'): df[idx_col] = df[idx_col].str.decode('utf-8') df = df.set_index(idx_cols) @@ -843,7 +843,7 @@ def read_h5py_file(filename, decode_strings=False): return df -def read_file(filename, parse_timestamps=False, decode_strings=False): +def read_file(filename, parse_timestamps=False): """Return dataframe object of input file for multiple file formats. This function read multiple file formats for h5 file sand returns a dataframe from the file. @@ -873,7 +873,7 @@ def read_file(filename, parse_timestamps=False, decode_strings=False): # datasets that composes the h5 file. For a single dataset we use pandas (since it is the most # convenient) and h5py for the custom h5 file. try: - df = read_h5py_file(filename, decode_strings=decode_strings) + df = read_h5py_file(filename) except TypeError: df = pd.read_hdf(filename) From fa193fc627a78e148675489469f6c2ec5e0a53cc Mon Sep 17 00:00:00 2001 From: Patrick Brown <25125211+patrickbrown4@users.noreply.github.com> Date: Wed, 20 May 2026 16:53:50 -0600 Subject: [PATCH 7/7] bump gdxpds to 2.0.0 --- environment.yml | 2 +- postprocessing/raw_value_streams.py | 2 +- postprocessing/reValue/reValue.py | 2 +- .../retail_rate_module/calculate_historical_capex.py | 4 ---- 4 files changed, 3 insertions(+), 7 deletions(-) diff --git a/environment.yml b/environment.yml index 40d6eeea..2f600d03 100644 --- a/environment.yml +++ b/environment.yml @@ -41,7 +41,7 @@ dependencies: ## ^^^ - pip: - gamsapi[transfer]==53.5.0 - - gdxpds==1.6.0 + - gdxpds==2.0.0 ## vvv The next packages are optional for default settings and may ## vvv be excluded if they cause trouble building the environment. - gamspy_base==53.5.0 # for tests on the github runner diff --git a/postprocessing/raw_value_streams.py b/postprocessing/raw_value_streams.py index 7ed4b033..d1ade6ae 100644 --- a/postprocessing/raw_value_streams.py +++ b/postprocessing/raw_value_streams.py @@ -151,7 +151,7 @@ def get_df_jacobian(jacobian_file, var_list=None, con_list=None): coeff (float): Coefficient of the variable in the constraint or objective. ''' start = datetime.now() - df_A = gdxpds.to_dataframe(jacobian_file, 'A', old_interface=False) + df_A = gdxpds.to_dataframe(jacobian_file, 'A') for x in ['j','i']: #For i (equation) and j (variable) sets, I need to dump to csv to get the Text column, ugh x_file = jacobian_file.replace('.gdx',f'_{x}.csv') diff --git a/postprocessing/reValue/reValue.py b/postprocessing/reValue/reValue.py index 064376b2..7dfac90d 100644 --- a/postprocessing/reValue/reValue.py +++ b/postprocessing/reValue/reValue.py @@ -85,7 +85,7 @@ def get_prices(): yrs_less = [y for y in yrs if y < year] max_yr = max(yrs_less) df_ra = gdxpds.to_dataframe(f'{reeds_run_path}/handoff/reeds_data/ccdata_{max_yr}.gdx', - 'net_load_2012', old_interface=False) + 'net_load_2012') if int(df_ra['t'][0]) != year: raise ValueError(f'RA year ({int(df_ra["t"][0])}) does not match current scenario year ({year})') df_ra = df_ra.sort_values('Value', ascending=False) diff --git a/postprocessing/retail_rate_module/calculate_historical_capex.py b/postprocessing/retail_rate_module/calculate_historical_capex.py index 60699eb9..4868f24a 100644 --- a/postprocessing/retail_rate_module/calculate_historical_capex.py +++ b/postprocessing/retail_rate_module/calculate_historical_capex.py @@ -58,12 +58,10 @@ def get_earliest_cap_costs(inputs_case): cost_cap = gdxpds.to_dataframe( os.path.join(inputs_case, 'inputs.gdx'), 'cost_cap', - old_interface=False ) cost_cap_energy = gdxpds.to_dataframe( os.path.join(inputs_case, 'inputs.gdx'), 'cost_cap_energy', - old_interface=False ) cost_cap = ( pd.concat([cost_cap, cost_cap_energy]) @@ -83,7 +81,6 @@ def get_earliest_cap_costs(inputs_case): cost_cap_mult = gdxpds.to_dataframe( os.path.join(inputs_case, 'inputs.gdx'), 'cost_cap_fin_mult_out', - old_interface=False ) cost_cap_mult['t'] = cost_cap_mult['t'].astype(int) cost_cap_mult = cost_cap_mult.rename( @@ -112,7 +109,6 @@ def get_earliest_cap_costs(inputs_case): rsc_dat = gdxpds.to_dataframe( os.path.join(inputs_case, 'inputs.gdx'), 'rsc_dat', - old_interface=False ) cost_cap_rsc = ( rsc_dat.loc[~rsc_dat.i.isin(cost_cap_earliest['i'])]