diff --git a/environment.yml b/environment.yml index c0161fcb..2f600d03 100644 --- a/environment.yml +++ b/environment.yml @@ -1,50 +1,48 @@ -name: reeds2 +name: reeds channels: - - defaults - conda-forge dependencies: - - python=3.11 - - bokeh=3.2 - - click=8.0 - - git-lfs=2.13 + - python=3.14 + - bokeh=3.9 + - click=8.3 + - cmocean=4.0 + - geopandas=1.1 + - git-lfs=3.7 - gitpython=3.1 - - h5py=3.9 - - matplotlib=3.7 - - numpy=1.26 - - openpyxl=3.0 - - pandas=2.0 - - pip=23.2 - - pytables=3.8 - - pytest=7.4 - - requests=2.31 - - scikit-learn=1.2 - - scipy=1.11 - - tqdm=4.65 + - h5py=3.16 + - matplotlib=3.10 + - numpy=2.4 + - openpyxl=3.1 + - pandas=3.0 + - pip=26.1 + - pulp=2.8 + - pytables=3.11 + - pytest=9.0 + - requests=2.33 + - scikit-learn=1.8 + - scipy=1.17 + - shapely=2.1 + - tqdm=4.67 ## vvv The next packages are optional for default settings and may ## vvv be excluded if you're having trouble building the environment. - - ipykernel=6.25 # for interactive python in VS Code - - ipywidgets=8.0 # for jupyter notebooks - - mapclassify=2.5 # more mapping tools + - fiona=1.10 # for interactive maps + - folium=0.20 # for interactive maps + - ipykernel=7.2 # for interactive python in VS Code + - ipywidgets=8.1 # for jupyter notebooks + - mapclassify=2.10 # more mapping tools - mscorefonts=0.0 # extra fonts for plotting (only relevant on linux/HPC) - - networkx=3.1 # for uncommonly-used network analysis postprocessing - - notebook=6.5 # for jupyter notebooks + - myst-parser=5.0 # for building documentation + - networkx=3.6 # for uncommonly-used network analysis postprocessing + - notebook=7.5 # for jupyter notebooks + - python-pptx=1.0 # for postprocessing/compare_cases.py + - sphinx=9.1 # for building documentation + - sphinx_rtd_theme=3.1 # for documentation + - sphinxcontrib-bibtex=2.6 # for documentation ## ^^^ - pip: - - cmocean==3.0.3 - - gdxpds==1.4.0 - - geopandas==0.14.0 - - pulp==2.7.0 - - shapely==2.0.1 + - gamsapi[transfer]==53.5.0 + - gdxpds==2.0.0 ## vvv The next packages are optional for default settings and may ## vvv be excluded if they cause trouble building the environment. - - fiona==1.9.5 # for interactive maps - - folium==0.14.0 # for interactive maps - - gamspy_base==50.5.0 # for tests on the github runner - - myst-parser==2.0.0 # for building documentation - - proj==0.2.0 # for plotting maps - - pyproj==3.6.1 # for plotting maps - - python-pptx==0.6.22 # for postprocessing/compare_cases.py - - sphinx_rtd_theme==2.0.0 # for documentation - - sphinx==7.2.6 # for building documentation - - sphinxcontrib-bibtex==2.6.2 # for documentation - ## ^^^ + - gamspy_base==53.5.0 # for tests on the github runner + - highspy==1.14.0 # for future solver flexibility diff --git a/postprocessing/raw_value_streams.py b/postprocessing/raw_value_streams.py index 7ed4b033..d1ade6ae 100644 --- a/postprocessing/raw_value_streams.py +++ b/postprocessing/raw_value_streams.py @@ -151,7 +151,7 @@ def get_df_jacobian(jacobian_file, var_list=None, con_list=None): coeff (float): Coefficient of the variable in the constraint or objective. ''' start = datetime.now() - df_A = gdxpds.to_dataframe(jacobian_file, 'A', old_interface=False) + df_A = gdxpds.to_dataframe(jacobian_file, 'A') for x in ['j','i']: #For i (equation) and j (variable) sets, I need to dump to csv to get the Text column, ugh x_file = jacobian_file.replace('.gdx',f'_{x}.csv') diff --git a/postprocessing/reValue/reValue.py b/postprocessing/reValue/reValue.py index 064376b2..7dfac90d 100644 --- a/postprocessing/reValue/reValue.py +++ b/postprocessing/reValue/reValue.py @@ -85,7 +85,7 @@ def get_prices(): yrs_less = [y for y in yrs if y < year] max_yr = max(yrs_less) df_ra = gdxpds.to_dataframe(f'{reeds_run_path}/handoff/reeds_data/ccdata_{max_yr}.gdx', - 'net_load_2012', old_interface=False) + 'net_load_2012') if int(df_ra['t'][0]) != year: raise ValueError(f'RA year ({int(df_ra["t"][0])}) does not match current scenario year ({year})') df_ra = df_ra.sort_values('Value', ascending=False) diff --git a/postprocessing/retail_rate_module/calculate_historical_capex.py b/postprocessing/retail_rate_module/calculate_historical_capex.py index 60699eb9..4868f24a 100644 --- a/postprocessing/retail_rate_module/calculate_historical_capex.py +++ b/postprocessing/retail_rate_module/calculate_historical_capex.py @@ -58,12 +58,10 @@ def get_earliest_cap_costs(inputs_case): cost_cap = gdxpds.to_dataframe( os.path.join(inputs_case, 'inputs.gdx'), 'cost_cap', - old_interface=False ) cost_cap_energy = gdxpds.to_dataframe( os.path.join(inputs_case, 'inputs.gdx'), 'cost_cap_energy', - old_interface=False ) cost_cap = ( pd.concat([cost_cap, cost_cap_energy]) @@ -83,7 +81,6 @@ def get_earliest_cap_costs(inputs_case): cost_cap_mult = gdxpds.to_dataframe( os.path.join(inputs_case, 'inputs.gdx'), 'cost_cap_fin_mult_out', - old_interface=False ) cost_cap_mult['t'] = cost_cap_mult['t'].astype(int) cost_cap_mult = cost_cap_mult.rename( @@ -112,7 +109,6 @@ def get_earliest_cap_costs(inputs_case): rsc_dat = gdxpds.to_dataframe( os.path.join(inputs_case, 'inputs.gdx'), 'rsc_dat', - old_interface=False ) cost_cap_rsc = ( rsc_dat.loc[~rsc_dat.i.isin(cost_cap_earliest['i'])] diff --git a/reeds/input_processing/copy_files.py b/reeds/input_processing/copy_files.py index 55a60880..3f94da6c 100644 --- a/reeds/input_processing/copy_files.py +++ b/reeds/input_processing/copy_files.py @@ -337,7 +337,7 @@ def get_regions_and_agglevel( # Sort hier_sub by r so that "ord(r)" commands in GAMS result in the properly # ordered outputs - hier_sub['numeric_value'] = hier_sub['r'].str.extract('(\d+)').astype(float) + hier_sub['numeric_value'] = hier_sub['r'].str.extract(r'(\d+)').astype(float) hier_sub = hier_sub.sort_values(by='numeric_value').drop('numeric_value', axis=1) # Output the itlgrp files for mixed and county resolution @@ -455,12 +455,7 @@ def read_banned_tech_file(full_path, filepath, inputs_case, r_county): .apply(list) .apply(sorted) ) - r_all_counties_map = ( - r_county.groupby('r') - ['county'] - .apply(list) - .apply(sorted) - ) + r_all_counties_map = r_county.county.tolist() county_ban_regions = list( r_ban_counties_map .loc[(r_ban_counties_map.isin(r_all_counties_map))] @@ -492,11 +487,8 @@ def read_special_h5file(full_path): df.columns = df.columns.str.replace('distpv|','') elif filename.startswith('transmission_cost_ac'): df = df.reset_index() - for col in ['r', 'rr', 'tscbin']: - df[col] = df[col].str.decode('utf-8') elif filename.startswith('transmission_distance'): df = df.stack().rename('miles').reset_index() - df['r'] = df['r'].str.decode('utf-8') return df @@ -900,7 +892,7 @@ def write_scalars(scalars, inputs_case): scalars_write = pd.concat([scalars, toadd], axis=0) # Trim trailing decimal zeros - scalars_write.value = scalars_write.value.astype(str).replace('\.0+$', '', regex=True) + scalars_write.value = scalars_write.value.astype(str).replace(r'\.0+$', '', regex=True) scalars_write.to_csv(os.path.join(inputs_case, 'scalars.csv'), header=False) # Rewrite the scalar tables as GAMS-readable definition @@ -1673,7 +1665,7 @@ def main(reeds_path, inputs_case): # #%% Settings for testing ### # reeds_path = reeds.io.reeds_path - # inputs_case = os.path.join(reeds_path,'runs','v20260305_itlM0_USA_defaults','inputs_case') + # inputs_case = os.path.join(reeds_path,'runs','v20260520_envM0_Pacific','inputs_case') # ---- Set up logger ---- diff --git a/reeds/io.py b/reeds/io.py index d4bcd313..5d12d2e8 100644 --- a/reeds/io.py +++ b/reeds/io.py @@ -689,23 +689,24 @@ def get_switches(case=None, **kwargs): (case if case is not None else reeds_path), 'reeds', 'resource_adequacy', 'ra_switches.csv', ) - asw = pd.read_csv(fpath_asw, index_col='key') - for i, row in asw.iterrows(): + dfra = pd.read_csv(fpath_asw, index_col='key', dtype='object') + ra_switches = {} + for key, row in dfra.iterrows(): if row['dtype'] == 'list': - row.value = row.value.split(',') + ra_switches[key] = row.value.split(',') try: - row.value = [int(i) for i in row.value] + ra_switches[key] = [int(i) for i in row.value] except ValueError: pass elif row['dtype'] == 'boolean': - row.value = False if row.value.lower() == 'false' else True + ra_switches[key] = False if row.value.lower() == 'false' else True elif row['dtype'] == 'str': - row.value = str(row.value) + ra_switches[key] = str(row.value) elif row['dtype'] == 'int': - row.value = int(row.value) + ra_switches[key] = int(row.value) elif row['dtype'] == 'float': - row.value = float(row.value) - sw = pd.concat([sw, asw.value]) + ra_switches[key] = float(row.value) + sw = pd.concat([sw, pd.Series(ra_switches)]) except FileNotFoundError: print(f"{fpath_asw} not found so leaving out resource adequacy switches") ### Add derivative switches @@ -773,7 +774,7 @@ def get_scalars(case=None, full=False): return scalars -def read_h5py_file(filename, decode_strings=False): +def read_h5py_file(filename): """Return dataframe object for a h5py file. This function returns a pandas dataframe of a h5py file. If the file has multiple dataset on it @@ -821,7 +822,7 @@ def read_h5py_file(filename, decode_strings=False): idx_cols.sort() for idx_col in idx_cols: df[idx_col] = pd.Series(f[idx_col]).values - if str(df[idx_col].dtype).startswith('|S') and decode_strings: + if str(df[idx_col].dtype).startswith('|S'): df[idx_col] = df[idx_col].str.decode('utf-8') df = df.set_index(idx_cols) @@ -842,7 +843,7 @@ def read_h5py_file(filename, decode_strings=False): return df -def read_file(filename, parse_timestamps=False, decode_strings=False): +def read_file(filename, parse_timestamps=False): """Return dataframe object of input file for multiple file formats. This function read multiple file formats for h5 file sand returns a dataframe from the file. @@ -872,7 +873,7 @@ def read_file(filename, parse_timestamps=False, decode_strings=False): # datasets that composes the h5 file. For a single dataset we use pandas (since it is the most # convenient) and h5py for the custom h5 file. try: - df = read_h5py_file(filename, decode_strings=decode_strings) + df = read_h5py_file(filename) except TypeError: df = pd.read_hdf(filename) diff --git a/reeds/resource_adequacy/prep_data.py b/reeds/resource_adequacy/prep_data.py index f7ad96c6..d30a0d42 100644 --- a/reeds/resource_adequacy/prep_data.py +++ b/reeds/resource_adequacy/prep_data.py @@ -170,7 +170,7 @@ def main(t, casedir, iteration=0): ## e.g. "upv_5" -> "upv", "csp2_3" -> "csp" techs_vre_simplify = dict(zip( techs_vre, - [re.sub('\d?_\d+$', '', i) for i in techs_vre] + [re.sub(r'\d?_\d+$', '', i) for i in techs_vre] )) try: diff --git a/runreeds.py b/runreeds.py index 6f15a290..9845e22d 100644 --- a/runreeds.py +++ b/runreeds.py @@ -826,22 +826,19 @@ def setupEnvironment( #%% Check whether the ReEDS conda environment is activated if (not skip_checks) and ( - ('reeds2' not in os.environ['CONDA_DEFAULT_ENV'].lower()) - or (not pd.__version__.startswith('2')) + ('reeds' not in os.environ['CONDA_DEFAULT_ENV'].lower()) + or (not pd.__version__.startswith('3')) ): - print( + err = ( f"Your environment is {os.environ['CONDA_DEFAULT_ENV']} and your pandas " - f"version is {pd.__version__}.\nThe default environment is 'reeds2', with\n" - "pandas version 2.x, so the python parts of ReEDS are unlikely to work.\n" + f"version is {pd.__version__}.\nThe supported environment is 'reeds', with\n" + "pandas version 3.x.\n" "To build the environment for the first time, run:\n" " `conda env create -f environment.yml`\n" "To activate the created environment, run:\n" - " `conda activate reeds2` (or `activate reeds2` on Windows)\n" - "Do you want to continue without activating the environment?" + " `conda activate reeds` (or `activate reeds` on Windows)" ) - confirm_env = str(input("Continue? y/[n]: ") or 'n') - if confirm_env not in ['y','Y','yes','Yes','YES']: - quit() + raise ValueError(err) #%% Load specified case file, infer other settings from cases.csv if cases_suffix in ['', 'default']: