Disaggregate and re-aggregate inputs with regional scope of legacy (134) zones

reeds/input_processing/aggregate_regions.py

@@ -473,11 +473,6 @@ def agg_disagg(filepath, r2aggreg_glob, r_ba_glob, runfiles_row):
         if list_check == agglevel_check :
             if row['disaggfunc']=='ignore':
                 return
-
-    # If the file isn't in inputs_case, skip it
-    if filename not in inputfiles:
-        if verbose > 1:
-            logprint(filepath, 'skipped since not in inputs_case')
         return
 
     #%%##############
@@ -1079,33 +1074,6 @@ def agg_disagg(filepath, r2aggreg_glob, r_ba_glob, runfiles_row):
     .rename(columns={'wide (1 if any parameters are in wide format)':'wide',
                      'header (0 if file has column labels)':'header'})
     )
-#%% If any files are missing, stop and alert the user
-inputfiles = sorted([
-    f.split('inputs_case'+os.sep)[1]
-    for f in glob(os.path.join(inputs_case,'**'), recursive=True)
-    if 'metadata' not in f
-])
-## Drop the directories and backup h17 files
-inputfiles = [f for f in inputfiles if (('.' in f) and not f.endswith('_h17.csv'))]
-missingfiles = [f for f in inputfiles if (os.path.basename(f) not in runfiles.index.values)
-]
-if any(missingfiles):
-    if missing == 'raise':
-        raise Exception(
-            'Missing aggregation method for:\n{}\n'
-            '>>> Need to add entries for these files to runfiles.csv'
-            .format('\n'.join(missingfiles))
-        )
-    else:
-        from warnings import warn
-        warn(
-            'Missing aggregation directions for:\n{}\n'
-            '>>> For this run, these files are copied without modification'
-            .format('\n'.join(missingfiles))
-        )
-        for f in missingfiles:
-            shutil.copy(os.path.join(inputs_case, f), os.path.join(inputs_case, f))
-            print(f'copied {f}, which is missing from runfiles.csv')
 
 #%% Maps (special case)
 mapsfile = os.path.join(inputs_case, 'maps.gpkg')
@@ -1164,8 +1132,15 @@ def agg_disagg(filepath, r2aggreg_glob, r_ba_glob, runfiles_row):
     r2aggreg_glob = r2aggreg
 r_ba_glob = r_ba
 
-# loop over inputfiles from runfiles and call aggregation/disaggregation function
-for filepath in inputfiles:
+# loop over transmission_files from runfiles and call aggregation/disaggregation function
+transmission_files = [
+    'transmission_capacity_future.csv',
+    'transmission_capacity_future_baseline.csv',
+    'transmission_cost_ac.csv',
+    'transmission_cost_dc.csv',
+    'transmission_distance.csv',
+]
+for filepath in transmission_files:
     ### For debugging: Specify a file
     # filepath = ''
     ### Get the appropriate row from runfiles

reeds/input_processing/copy_files.py

@@ -1155,59 +1155,14 @@ def map_and_aggregate(
 
     return df
 
-def upscale_from_county_to_ba(
-    df,
-    region_file_entry,
-    agglevel_variables,
-    regions_and_agglevel,
-    aggfunc=None
-):
-    """
-    Changes the resolution of the provided region_col from county to BA
-    or mixed resolution and aggregates according to the provided aggfunc.
-    """
-    original_cols = df.columns
-    region_col = region_file_entry['region_col']
-
-    # Exception for cendiv
-    if region_col.strip('*') == 'r_cendiv':
-        val_cendiv = regions_and_agglevel['valid_regions']['cendiv']
-        df = df.loc[df['r'].isin(regions_and_agglevel['r_county']['county'])]
-        df = df.loc[:, df.columns.isin(["r"] + val_cendiv)].reset_index(drop=True)
-        region_col = 'r'
-
-    # Aggregate values to ba resolution if not running county-level resolution
-    # and if county level is not a desired resolution in mixed resolution runs
-    if 'county' not in agglevel_variables['agglevel']:
-        df = map_and_aggregate(df,regions_and_agglevel,region_file_entry,region_col,aggfunc)
-
-
-    # Mixed resolution procedure
-    elif agglevel_variables['lvl'] == 'mult':
-        df_ba = df[df[region_col].isin(agglevel_variables['BA_county_list'])]
-        df_ba = map_and_aggregate(df_ba,regions_and_agglevel,region_file_entry,region_col,aggfunc)
-
-        # Filter out county regions
-        df_county = df[df[region_col].isin(agglevel_variables['county_regions'])]
-        # Combine county and BA
-        df = pd.concat([df_ba, df_county])
-
-    else:
-         pass
-
-    df = df[original_cols]
-
-    return df
-
 
 def write_region_indexed_file(
     df,
     dir_dst,
     source_deflator_map,
     sw,
     region_file_entry,
-    regions_and_agglevel,
-    agglevel_variables
+    regions_and_agglevel
 ):
     """
     Write a single region-indexed file to the dir_dst directory
@@ -1216,6 +1171,35 @@ def write_region_indexed_file(
     # Get the filetype of the output file from the filename string
     filetype_out = os.path.splitext(filename)[1].strip('.')
 
+    transmission_files = [
+        'transmission_capacity_future.csv',
+        'transmission_capacity_future_baseline.csv',
+        'transmission_cost_ac.csv',
+        'transmission_cost_dc.csv',
+        'transmission_distance.csv',
+    ]
+    if filename not in transmission_files:
+        region_col = region_file_entry['region_col']
+        fix_cols = region_file_entry['fix_cols'].split(',')
+
+        if region_file_entry['disaggfunc'] != 'ignore':
+            df = reeds.spatial.downscale_from_legacy_zone_to_county(
+                df=df,
+                region_col=region_col,
+                fix_cols=fix_cols,
+                inputs_case=inputs_case,
+                disaggfunc=region_file_entry['disaggfunc']
+            )
+
+        if region_file_entry['aggfunc'] != 'ignore':
+            df = reeds.spatial.upscale_from_county_to_zone(
+                df=df,
+                region_col=region_col,
+                fix_cols=fix_cols,
+                inputs_case=inputs_case,
+                aggfunc=region_file_entry['aggfunc']
+            )
+
     #---- Write data to dir_dst (inputs_case) folder ----
     if filetype_out == 'h5':
         reeds.io.write_profile_to_h5(df, filename, dir_dst)
@@ -1226,27 +1210,6 @@ def write_region_indexed_file(
             case 'bio_supplycurve.csv':
                 # Adjust for inflation
                 df['price'] = df['price'].astype(float) * source_deflator_map[filepath]
-            case (
-                'can_exports.csv'
-                | 'can_imports.csv'
-                | 'demonstration_plants.csv'
-                | 'distpvcap.csv'
-                | 'h2_ba_share.csv'
-                | 'regional_cap_cost_diff.csv'
-                | 'cendivweights.csv'
-                | 'cap_existing_psh.csv'
-            ):
-                # The upscale_from_county_to_ba function correctly handles the different spatial resolution options
-                # This sections just needs to check if the run is at pure county resolution
-                # The above listed data need to be upscaled if the run includes anything coarser than county resolution
-                if agglevel_variables['lvl'] != 'county':
-                        df = upscale_from_county_to_ba(
-                            df=df,
-                            region_file_entry=region_file_entry,
-                            agglevel_variables=agglevel_variables,
-                            regions_and_agglevel=regions_and_agglevel,
-                            aggfunc=region_file_entry.aggfunc
-                        )
             case 'unitdata.csv':
                 fips_ba_map = regions_and_agglevel['ba_county'].dropna().set_index('county')['ba']
                 df['reeds_ba'] = df['FIPS'].map(fips_ba_map)
@@ -1301,8 +1264,7 @@ def write_region_indexed_files(
                 source_deflator_map,
                 sw,
                 region_file_entry,
-                regions_and_agglevel,
-                agglevel_variables
+                regions_and_agglevel
             )
 
 
@@ -1665,6 +1627,9 @@ def main(reeds_path, inputs_case):
 
     # Copy non-region files
     write_non_region_files(non_region_files, sw, inputs_case, regions_and_agglevel, source_deflator_map)
+
+    # Write files used for disaggregation
+    write_disagg_data_files(runfiles, inputs_case)
 
     # Copy region files
     write_region_indexed_files(
@@ -1676,9 +1641,6 @@ def main(reeds_path, inputs_case):
         source_deflator_map
     )
 
-    # Write files used for disaggregation
-    write_disagg_data_files(runfiles, inputs_case)
-
     # Create a maps.gpkg for this run
     # Skip if using region dis/aggregation, maps will be written in aggregation_regions.py.
     # Run if using mixed resolution aggreg-county combination 

reeds/input_processing/mcs_sampler.py

@@ -1830,8 +1830,7 @@ def write_samples(
             region_files_row = aux_files['region_files'].query('filename == @file_name').iloc[0]
             copy_files.write_region_indexed_file(sample_values, dir_dst, aux_files['source_deflator_map'],
                                                     aux_files['sw'], region_files_row,
-                                                    aux_files['regions_and_agglevel'],
-                                                    aux_files['agglevel_variables'])
+                                                    aux_files['regions_and_agglevel'])
         # ...if we have a csv file that isn't region-indexed (including switches.csv)
         elif file_termination == '.csv':
             if file_name == 'switches.csv':