From c44d5db0477b4babae22bc124baf4b56d08c4e66 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Tue, 27 Jul 2021 14:57:27 +0200 Subject: [PATCH 1/6] Add "compass cache" command This generates an updated cached_files.json (locally named ocean_cached_files.json or landice_cached_files.json) that lists cached output files, and copies the cached files into the appropriate location on the LCRC server. It can only be run on Anvil or Chrysalis. --- compass/__main__.py | 10 +++- compass/cache.py | 135 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 144 insertions(+), 1 deletion(-) create mode 100644 compass/cache.py diff --git a/compass/__main__.py b/compass/__main__.py index 9dd108e3bb..5d22cf83e8 100644 --- a/compass/__main__.py +++ b/compass/__main__.py @@ -2,9 +2,10 @@ import sys import argparse +import os import compass -from compass import list, setup, clean, suite, run +from compass import list, setup, clean, suite, run, cache def main(): @@ -40,11 +41,18 @@ def main(): args = parser.parse_args(sys.argv[1:2]) + # only allow the "compass cache" command if we're on Anvil or Chrysalis + allow_cache = ('COMPASS_MACHINE' in os.environ and + os.environ['COMPASS_MACHINE'] in ['anvil', 'chrysalis']) + commands = {'list': list.main, 'setup': setup.main, 'clean': clean.main, 'suite': suite.main, 'run': run.main} + if allow_cache: + commands['cache'] = cache.main + if args.command not in commands: print('Unrecognized command {}'.format(args.command)) parser.print_help() diff --git a/compass/cache.py b/compass/cache.py new file mode 100644 index 0000000000..ac2d4269ae --- /dev/null +++ b/compass/cache.py @@ -0,0 +1,135 @@ +import argparse +import json +import sys +from datetime import datetime +import os +from importlib import resources +import configparser +import shutil +import pickle + +from compass.config import add_config + + +def update_cache(step_paths, date_string=None, dry_run=False): + """ + Cache one or more compass output files for use in a cached variant of the + test case or step + + Parameters + ---------- + step_paths : list of str + The relative path of the original (uncached) steps from the base work + directory + + date_string : str, optional + The datestamp (YYMMDD) to use on the files. Default is today's date. + + dry_run : bool, optional + Whether this is a dry run (producing the json file but not copying + files to the LCRC server) + """ + if 'COMPASS_MACHINE' not in os.environ: + machine = None + invalid = True + else: + machine = os.environ['COMPASS_MACHINE'] + invalid = machine not in ['anvil', 'chrysalis'] + + if invalid: + raise ValueError('You must cache files from either Anvil or Chrysalis') + + config = configparser.ConfigParser( + interpolation=configparser.ExtendedInterpolation()) + add_config(config, 'compass.machines', '{}.cfg'.format(machine)) + + if date_string is None: + date_string = datetime.now().strftime("%y%m%d") + + # make a dictionary with MPAS cores as keys, and lists of steps as values + steps = dict() + for path in step_paths: + with open(f'{path}/step.pickle', 'rb') as handle: + _, step = pickle.load(handle) + + mpas_core = step.mpas_core.name + + if mpas_core in steps: + steps[mpas_core].append(step) + else: + steps[mpas_core] = [step] + + # now, iterate over cores and steps + for mpas_core in steps: + database_root = config.get('paths', f'{mpas_core}_database_root') + cache_root = f'{database_root}/compass_cache' + + package = f'compass.{mpas_core}' + try: + with open(f'{mpas_core}_cached_files.json') as data_file: + cached_files = json.load(data_file) + except FileNotFoundError: + # we don't have a local version of the file yet, let's see if + # there's a remote one for this MPAS core + try: + with resources.path(package, 'cached_files.json') as path: + with open(path) as data_file: + cached_files = json.load(data_file) + except FileNotFoundError: + # no cached files yet for this core + cached_files = dict() + + for step in steps[mpas_core]: + # load the step from its pickle file + + step_path = step.path + + for output in step.outputs: + output = os.path.basename(output) + out_filename = os.path.join(step_path, output) + # remove the MPAS core from the file path + target = out_filename[len(mpas_core)+1:] + path, ext = os.path.splitext(target) + target = f'{path}.{date_string}{ext}' + cached_files[out_filename] = target + + print(out_filename) + print(f' ==> {target}') + output_path = f'{cache_root}/{target}' + print(f' copy to: {output_path}') + print() + if not dry_run: + directory = os.path.dirname(output_path) + try: + os.makedirs(directory) + except FileExistsError: + pass + shutil.copyfile(out_filename, output_path) + + out_filename = f'{mpas_core}_cached_files.json' + with open(out_filename, 'w') as data_file: + json.dump(cached_files, data_file, indent=4) + + +def main(): + parser = argparse.ArgumentParser( + description='Cache the output files from one or more steps for use in ' + 'a cached variant of the step', + prog='compass cache') + parser.add_argument("-i", "--orig_steps", nargs='+', dest="orig_steps", + type=str, + help="The relative path of the original (uncached) " + "steps from the base work directory", + metavar="STEP") + parser.add_argument("-d", "--date_string", dest="date_string", type=str, + help="The datestamp (YYMMDD) to use on the files. " + "Default is today's date.", + metavar="DATE") + parser.add_argument("-r", "--dry_run", dest="dry_run", + help="Whether this is a dry run (producing the json " + "file but not copying files to the LCRC server).", + action="store_true") + + args = parser.parse_args(sys.argv[2:]) + update_cache(step_paths=args.orig_steps, date_string=args.date_string, + dry_run=args.dry_run) From bcaa196365853e0e2df547aae4232ae68f54b63c Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Tue, 27 Jul 2021 15:03:25 +0200 Subject: [PATCH 2/6] Add support for cached output files When constructing a step, if ``cached=True``, the outputs for this step will be downloaded to the appropriate local database and symlinked instead of being computed. Inputs (other than the cached outputs) and the run method will be skipped. Each MPAS core can optionally have a database (a python dictionary in a json file called cached_files.json) that keeps track of which files are available in the cache and what date stamp is in the filename. When setting up test cases, a user can supply test-case numbers with a "c" suffix to indicate that they should be cached. When setting up test cases individually with a path, a user can supply a list of steps in the test case that should use cached outputs. A test suite can supply a line with "cached" or "cached: " to indicate either that all steps in the test case or the listed steps should use cached outputs. --- compass/mpas_core.py | 25 +++++++++++++++ compass/setup.py | 72 ++++++++++++++++++++++++++++++++++++++------ compass/step.py | 42 +++++++++++++++++++++++--- compass/suite.py | 40 ++++++++++++++++++------ compass/testcase.py | 3 ++ 5 files changed, 159 insertions(+), 23 deletions(-) diff --git a/compass/mpas_core.py b/compass/mpas_core.py index 2f82cd2ae4..8911389ad4 100644 --- a/compass/mpas_core.py +++ b/compass/mpas_core.py @@ -1,3 +1,7 @@ +from importlib import resources +import json + + class MpasCore: """ The base class for housing all the tests for a given MPAS core, such as @@ -10,6 +14,11 @@ class MpasCore: test_groups : dict A dictionary of test groups for the MPAS core with their names as keys + + cached_files : dict + A dictionary that maps from output file names in test cases to cached + files in the ``compass_cache`` database for the MPAS core. These + file mappings are read in from ``cached_files.json`` in the MPAS core. """ def __init__(self, name): @@ -26,6 +35,9 @@ def __init__(self, name): # test groups are added with add_test_groups() self.test_groups = dict() + self.cached_files = dict() + self._read_cached_files() + def add_test_group(self, test_group): """ Add a test group to the MPAS core @@ -36,3 +48,16 @@ def add_test_group(self, test_group): the test group to add """ self.test_groups[test_group.name] = test_group + + def _read_cached_files(self): + """ Read in the dictionary of cached files from cached_files.json """ + + package = f'compass.{self.name}' + filename = 'cached_files.json' + try: + with resources.path(package, filename) as path: + with open(path) as data_file: + self.cached_files = json.load(data_file) + except FileNotFoundError: + # no cached files for this core + pass diff --git a/compass/setup.py b/compass/setup.py index 537a7ec622..9ed48804ee 100644 --- a/compass/setup.py +++ b/compass/setup.py @@ -3,6 +3,7 @@ import configparser import os import pickle +import warnings from compass.mpas_cores import get_mpas_cores from compass.config import add_config, ensure_absolute_paths @@ -12,7 +13,7 @@ def setup_cases(tests=None, numbers=None, config_file=None, machine=None, work_dir=None, baseline_dir=None, mpas_model_path=None, - suite_name='custom'): + suite_name='custom', cached=None): """ Set up one or more test cases @@ -21,8 +22,10 @@ def setup_cases(tests=None, numbers=None, config_file=None, machine=None, tests : list of str, optional Relative paths for a test cases to set up - numbers : list of int, optional - Case numbers to setup, as listed from ``compass list`` + numbers : list of str, optional + Case numbers to setup, as listed from ``compass list``, optionally with + a suffix ``c`` to indicate that all steps in that test case should be + cached config_file : str, optional Configuration file with custom options for setting up and running test @@ -46,6 +49,11 @@ def setup_cases(tests=None, numbers=None, config_file=None, machine=None, The name of the test suite if tests are being set up through a test suite or ``'custom'`` if not + cached : list of list of str, optional + For each test in ``tests``, which steps (if any) should be cached, + or a list with "_all" as the first entry if all steps in the test case + should be cached + Returns ------- test_cases : dict of compass.TestCase @@ -61,6 +69,14 @@ def setup_cases(tests=None, numbers=None, config_file=None, machine=None, if tests is None and numbers is None: raise ValueError('At least one of tests or numbers is needed.') + if cached is not None: + if tests is None: + warnings.warn('Ignoring "cached" argument becasue "tests" was ' + 'not provided') + elif len(cached) != len(tests): + raise ValueError('A list of cached steps must be provided for ' + 'each test in "tests"') + if work_dir is None: work_dir = os.getcwd() work_dir = os.path.abspath(work_dir) @@ -74,20 +90,34 @@ def setup_cases(tests=None, numbers=None, config_file=None, machine=None, all_test_cases[test_case.path] = test_case test_cases = dict() + cached_steps = dict() if numbers is not None: keys = list(all_test_cases) for number in numbers: + cache_all = False + if number.endswith('c'): + cache_all = True + number = int(number[:-1]) + else: + number = int(number) + if number >= len(keys): raise ValueError('test number {} is out of range. There are ' 'only {} tests.'.format(number, len(keys))) path = keys[number] + if cache_all: + cached_steps[path] = ['_all'] + else: + cached_steps[path] = list() test_cases[path] = all_test_cases[path] if tests is not None: - for path in tests: + for index, path in enumerate(tests): if path not in all_test_cases: raise ValueError('Test case with path {} is not in ' 'test_cases'.format(path)) + if cached is not None: + cached_steps[path] = cached[index] test_cases[path] = all_test_cases[path] # get the MPAS core of the first test case. We'll assume all tests are @@ -101,7 +131,8 @@ def setup_cases(tests=None, numbers=None, config_file=None, machine=None, print('Setting up test cases:') for path, test_case in test_cases.items(): setup_case(path, test_case, config_file, machine, work_dir, - baseline_dir, mpas_model_path) + baseline_dir, mpas_model_path, + cached_steps=cached_steps[path]) test_suite = {'name': suite_name, 'test_cases': test_cases, @@ -127,7 +158,7 @@ def setup_cases(tests=None, numbers=None, config_file=None, machine=None, def setup_case(path, test_case, config_file, machine, work_dir, baseline_dir, - mpas_model_path): + mpas_model_path, cached_steps): """ Set up one or more test cases @@ -156,6 +187,10 @@ def setup_case(path, test_case, config_file, machine, work_dir, baseline_dir, mpas_model_path : str The relative or absolute path to the root of a branch where the MPAS model has been built + + cached_steps : list of str + Which steps (if any) should be cached. If all steps should be cached, + the first entry is "_all" """ print(' {}'.format(path)) @@ -227,6 +262,14 @@ def setup_case(path, test_case, config_file, machine, work_dir, baseline_dir, with open(os.path.join(test_case_dir, test_case_config), 'w') as f: config.write(f) + if len(cached_steps) > 0 and cached_steps[0] == '_all': + cached_steps = list(test_case.steps.keys()) + if len(cached_steps) > 0: + print_steps = ' '.join(cached_steps) + print(f' steps with cached outputs: {print_steps}') + for step_name in cached_steps: + test_case.steps[step_name].cached = True + # iterate over steps for step in test_case.steps.values(): # make the step directory if it doesn't exist @@ -279,10 +322,12 @@ def main(): help="Relative path for a test case to set up", metavar="PATH") parser.add_argument("-n", "--case_number", nargs='+', dest="case_num", - type=int, + type=str, help="Case number(s) to setup, as listed from " "'compass list'. Can be a space-separated" - "list of case numbers.", metavar="NUM") + "list of case numbers. A suffix 'c' indicates" + "that all steps in the test should use cached" + "outputs.", metavar="NUM") parser.add_argument("-f", "--config_file", dest="config_file", help="Configuration file for test case setup", metavar="FILE") @@ -304,16 +349,25 @@ def main(): help="The name to use for the 'custom' test suite" "containing all setup test cases.", metavar="SUITE") + parser.add_argument("--cached", dest="cached", nargs='+', + help="A list of steps in the test case supplied with" + "--test that should use cached outputs, or " + "'_all' if all steps should be cached", + metavar="STEP") args = parser.parse_args(sys.argv[2:]) + cached = None if args.test is None: tests = None else: tests = [args.test] + if args.cached is not None: + cached = [args.cached] setup_cases(tests=tests, numbers=args.case_num, config_file=args.config_file, machine=args.machine, work_dir=args.work_dir, baseline_dir=args.baseline_dir, - mpas_model_path=args.mpas_model, suite_name=args.suite_name) + mpas_model_path=args.mpas_model, suite_name=args.suite_name, + cached=cached) def _get_required_cores(test_cases): diff --git a/compass/step.py b/compass/step.py index 55874a7a59..20984f7858 100644 --- a/compass/step.py +++ b/compass/step.py @@ -76,9 +76,9 @@ class Step: time or the step will raise an exception outputs : list of str - a list of absolute paths of output files produced by this step and - available as inputs to other test cases and steps. These files must - exist after the test has run or an exception will be raised + a list of absolute paths of output files produced by this step (or + cached) and available as inputs to other test cases and steps. These + files must exist after the test has run or an exception will be raised namelist_data : dict a dictionary used internally to keep track of updates to the default @@ -111,10 +111,14 @@ class Step: log_filename : str At run time, the name of a log file where output/errors from the step are being logged, or ``None`` if output is to stdout/stderr + + cached : bool + Whether to get all of the outputs for the step from the database of + cached outputs for this MPAS core """ def __init__(self, test_case, name, subdir=None, cores=1, min_cores=1, - threads=1, max_memory=1000, max_disk=1000): + threads=1, max_memory=1000, max_disk=1000, cached=False): """ Create a new test case @@ -150,6 +154,10 @@ def __init__(self, test_case, name, subdir=None, cores=1, min_cores=1, the amount of disk space that the step is allowed to use in MB. This is currently just a placeholder for later use with task parallelism + + cached : bool, optional + Whether to get all of the outputs for the step from the database of + cached outputs for this MPAS core """ self.name = name self.test_case = test_case @@ -186,6 +194,9 @@ def __init__(self, test_case, name, subdir=None, cores=1, min_cores=1, self.logger = None self.log_filename = None + # output caching + self.cached = cached + def setup(self): """ Set up the test case in the work directory, including downloading any @@ -454,6 +465,22 @@ def process_inputs_and_outputs(self): step_dir = self.work_dir config = self.config + # process the outputs first because cached outputs will add more inputs + if self.cached: + # forget about the inputs -- we won't used them, but we will add + # the cached outputs as inputs + self.input_data = list() + for output in self.outputs: + filename = os.path.join(self.path, output) + if filename not in self.mpas_core.cached_files: + raise ValueError(f'The file {filename} has not been added ' + f'to the cache database') + target = self.mpas_core.cached_files[filename] + self.add_input_file( + filename=output, + target=target, + database='compass_cache') + inputs = [] for entry in self.input_data: filename = entry['filename'] @@ -534,6 +561,10 @@ def _generate_namelists(self): by parsing the files and dictionaries in the step's ``namelist_data``. """ + if self.cached: + # no need for namelists + return + step_work_dir = self.work_dir config = self.config @@ -570,6 +601,9 @@ def _generate_streams(self): Writes out a streams file in the work directory with new values given by parsing the files and dictionaries in the step's ``streams_data``. """ + if self.cached: + # no need for streams + return step_work_dir = self.work_dir config = self.config diff --git a/compass/suite.py b/compass/suite.py index a93b9d4c55..5d0cbc7c37 100644 --- a/compass/suite.py +++ b/compass/suite.py @@ -34,7 +34,7 @@ def setup_suite(mpas_core, suite_name, config_file=None, machine=None, directories baseline_dir : str, optional - Location of baseslines that can be compared to + Location of baselines that can be compared to mpas_model_path : str, optional The relative or absolute path to the root of a branch where the MPAS @@ -42,16 +42,13 @@ def setup_suite(mpas_core, suite_name, config_file=None, machine=None, """ text = resources.read_text('compass.{}.suites'.format(mpas_core), '{}.txt'.format(suite_name)) - tests = list() - for test in text.split('\n'): - test = test.strip() - if (len(test) > 0 and test not in tests - and not test.startswith('#')): - tests.append(test) + + tests, cached = _parse_suite(text) setup_cases(tests, config_file=config_file, machine=machine, work_dir=work_dir, baseline_dir=baseline_dir, - mpas_model_path=mpas_model_path, suite_name=suite_name) + mpas_model_path=mpas_model_path, suite_name=suite_name, + cached=cached) def clean_suite(mpas_core, suite_name, work_dir=None): @@ -75,8 +72,8 @@ def clean_suite(mpas_core, suite_name, work_dir=None): text = resources.read_text('compass.{}.suites'.format(mpas_core), '{}.txt'.format(suite_name)) - tests = [test.strip() for test in text.split('\n') if - len(test.strip()) > 0 and not test.startswith('#')] + + tests, _ = _parse_suite(text) clean_cases(tests=tests, work_dir=work_dir, suite_name=suite_name) @@ -128,3 +125,26 @@ def main(): config_file=args.config_file, machine=args.machine, work_dir=args.work_dir, baseline_dir=args.baseline_dir, mpas_model_path=args.mpas_model) + + +def _parse_suite(text): + """ Parse the text of a file defining a test suite """ + + tests = list() + cached = list() + for test in text.split('\n'): + test = test.strip() + if len(test) == 0 or test.startswith('#'): + # a blank line or comment + continue + + if test == 'cached': + cached[-1] = ['_all'] + elif test.startswith('cached:'): + steps = test[len('cached:'):].strip().split(' ') + cached[-1].extend(steps) + else: + tests.append(test) + cached.append(list()) + + return tests, cached diff --git a/compass/testcase.py b/compass/testcase.py index 2350f8b518..365329a57d 100644 --- a/compass/testcase.py +++ b/compass/testcase.py @@ -145,6 +145,9 @@ def run(self): cwd = os.getcwd() for step_name in self.steps_to_run: step = self.steps[step_name] + if step.cached: + logger.info(' * Cached: {}'.format(step_name)) + continue step.config = self.config new_log_file = self.new_step_log_file if self.log_filename is not None: From 986352c69d3b1352bfd30c26933282873cb90f54 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Tue, 3 Aug 2021 09:57:33 -0500 Subject: [PATCH 3/6] Add "database" of ocean cached outputs So far, meshes and initial conditions for tests in the global ocean and global convergence test groups are included. --- compass/ocean/cached_files.json | 63 +++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 compass/ocean/cached_files.json diff --git a/compass/ocean/cached_files.json b/compass/ocean/cached_files.json new file mode 100644 index 0000000000..b8871eff1a --- /dev/null +++ b/compass/ocean/cached_files.json @@ -0,0 +1,63 @@ +{ + "ocean/global_ocean/QU240/mesh/mesh/culled_mesh.nc": "global_ocean/QU240/mesh/mesh/culled_mesh.210803.nc", + "ocean/global_ocean/QU240/mesh/mesh/culled_graph.info": "global_ocean/QU240/mesh/mesh/culled_graph.210803.info", + "ocean/global_ocean/QU240/mesh/mesh/critical_passages_mask_final.nc": "global_ocean/QU240/mesh/mesh/critical_passages_mask_final.210803.nc", + "ocean/global_ocean/QU240/PHC/init/initial_state/initial_state.nc": "global_ocean/QU240/PHC/init/initial_state/initial_state.210803.nc", + "ocean/global_ocean/QU240/PHC/init/initial_state/init_mode_forcing_data.nc": "global_ocean/QU240/PHC/init/initial_state/init_mode_forcing_data.210803.nc", + "ocean/global_ocean/QUwISC240/mesh/mesh/culled_mesh.nc": "global_ocean/QUwISC240/mesh/mesh/culled_mesh.210803.nc", + "ocean/global_ocean/QUwISC240/mesh/mesh/culled_graph.info": "global_ocean/QUwISC240/mesh/mesh/culled_graph.210803.info", + "ocean/global_ocean/QUwISC240/mesh/mesh/critical_passages_mask_final.nc": "global_ocean/QUwISC240/mesh/mesh/critical_passages_mask_final.210803.nc", + "ocean/global_ocean/QUwISC240/PHC/init/initial_state/initial_state.nc": "global_ocean/QUwISC240/PHC/init/initial_state/initial_state.210803.nc", + "ocean/global_ocean/QUwISC240/PHC/init/initial_state/init_mode_forcing_data.nc": "global_ocean/QUwISC240/PHC/init/initial_state/init_mode_forcing_data.210803.nc", + "ocean/global_ocean/QUwISC240/PHC/init/ssh_adjustment/adjusted_init.nc": "global_ocean/QUwISC240/PHC/init/ssh_adjustment/adjusted_init.210803.nc", + "ocean/global_ocean/EC30to60/mesh/mesh/culled_mesh.nc": "global_ocean/EC30to60/mesh/mesh/culled_mesh.210803.nc", + "ocean/global_ocean/EC30to60/mesh/mesh/culled_graph.info": "global_ocean/EC30to60/mesh/mesh/culled_graph.210803.info", + "ocean/global_ocean/EC30to60/mesh/mesh/critical_passages_mask_final.nc": "global_ocean/EC30to60/mesh/mesh/critical_passages_mask_final.210803.nc", + "ocean/global_ocean/EC30to60/PHC/init/initial_state/initial_state.nc": "global_ocean/EC30to60/PHC/init/initial_state/initial_state.210803.nc", + "ocean/global_ocean/EC30to60/PHC/init/initial_state/init_mode_forcing_data.nc": "global_ocean/EC30to60/PHC/init/initial_state/init_mode_forcing_data.210803.nc", + "ocean/global_ocean/WC14/mesh/mesh/culled_mesh.nc": "global_ocean/WC14/mesh/mesh/culled_mesh.210803.nc", + "ocean/global_ocean/WC14/mesh/mesh/culled_graph.info": "global_ocean/WC14/mesh/mesh/culled_graph.210803.info", + "ocean/global_ocean/WC14/mesh/mesh/critical_passages_mask_final.nc": "global_ocean/WC14/mesh/mesh/critical_passages_mask_final.210803.nc", + "ocean/global_ocean/WC14/PHC/init/initial_state/initial_state.nc": "global_ocean/WC14/PHC/init/initial_state/initial_state.210803.nc", + "ocean/global_ocean/WC14/PHC/init/initial_state/init_mode_forcing_data.nc": "global_ocean/WC14/PHC/init/initial_state/init_mode_forcing_data.210803.nc", + "ocean/global_ocean/ECwISC30to60/mesh/mesh/culled_mesh.nc": "global_ocean/ECwISC30to60/mesh/mesh/culled_mesh.210803.nc", + "ocean/global_ocean/ECwISC30to60/mesh/mesh/culled_graph.info": "global_ocean/ECwISC30to60/mesh/mesh/culled_graph.210803.info", + "ocean/global_ocean/ECwISC30to60/mesh/mesh/critical_passages_mask_final.nc": "global_ocean/ECwISC30to60/mesh/mesh/critical_passages_mask_final.210803.nc", + "ocean/global_ocean/ECwISC30to60/PHC/init/initial_state/initial_state.nc": "global_ocean/ECwISC30to60/PHC/init/initial_state/initial_state.210803.nc", + "ocean/global_ocean/ECwISC30to60/PHC/init/initial_state/init_mode_forcing_data.nc": "global_ocean/ECwISC30to60/PHC/init/initial_state/init_mode_forcing_data.210803.nc", + "ocean/global_ocean/ECwISC30to60/PHC/init/ssh_adjustment/adjusted_init.nc": "global_ocean/ECwISC30to60/PHC/init/ssh_adjustment/adjusted_init.210803.nc", + "ocean/global_ocean/SOwISC12to60/mesh/mesh/culled_mesh.nc": "global_ocean/SOwISC12to60/mesh/mesh/culled_mesh.210803.nc", + "ocean/global_ocean/SOwISC12to60/mesh/mesh/culled_graph.info": "global_ocean/SOwISC12to60/mesh/mesh/culled_graph.210803.info", + "ocean/global_ocean/SOwISC12to60/mesh/mesh/critical_passages_mask_final.nc": "global_ocean/SOwISC12to60/mesh/mesh/critical_passages_mask_final.210803.nc", + "ocean/global_ocean/SOwISC12to60/PHC/init/initial_state/initial_state.nc": "global_ocean/SOwISC12to60/PHC/init/initial_state/initial_state.210803.nc", + "ocean/global_ocean/SOwISC12to60/PHC/init/initial_state/init_mode_forcing_data.nc": "global_ocean/SOwISC12to60/PHC/init/initial_state/init_mode_forcing_data.210803.nc", + "ocean/global_ocean/SOwISC12to60/PHC/init/ssh_adjustment/adjusted_init.nc": "global_ocean/SOwISC12to60/PHC/init/ssh_adjustment/adjusted_init.210803.nc", + "ocean/global_convergence/cosine_bell/QU60/mesh/mesh.nc": "global_convergence/cosine_bell/QU60/mesh/mesh.210803.nc", + "ocean/global_convergence/cosine_bell/QU60/mesh/graph.info": "global_convergence/cosine_bell/QU60/mesh/graph.210803.info", + "ocean/global_convergence/cosine_bell/QU60/init/namelist.ocean": "global_convergence/cosine_bell/QU60/init/namelist.210803.ocean", + "ocean/global_convergence/cosine_bell/QU60/init/initial_state.nc": "global_convergence/cosine_bell/QU60/init/initial_state.210803.nc", + "ocean/global_convergence/cosine_bell/QU90/mesh/mesh.nc": "global_convergence/cosine_bell/QU90/mesh/mesh.210803.nc", + "ocean/global_convergence/cosine_bell/QU90/mesh/graph.info": "global_convergence/cosine_bell/QU90/mesh/graph.210803.info", + "ocean/global_convergence/cosine_bell/QU90/init/namelist.ocean": "global_convergence/cosine_bell/QU90/init/namelist.210803.ocean", + "ocean/global_convergence/cosine_bell/QU90/init/initial_state.nc": "global_convergence/cosine_bell/QU90/init/initial_state.210803.nc", + "ocean/global_convergence/cosine_bell/QU120/mesh/mesh.nc": "global_convergence/cosine_bell/QU120/mesh/mesh.210803.nc", + "ocean/global_convergence/cosine_bell/QU120/mesh/graph.info": "global_convergence/cosine_bell/QU120/mesh/graph.210803.info", + "ocean/global_convergence/cosine_bell/QU120/init/namelist.ocean": "global_convergence/cosine_bell/QU120/init/namelist.210803.ocean", + "ocean/global_convergence/cosine_bell/QU120/init/initial_state.nc": "global_convergence/cosine_bell/QU120/init/initial_state.210803.nc", + "ocean/global_convergence/cosine_bell/QU180/mesh/mesh.nc": "global_convergence/cosine_bell/QU180/mesh/mesh.210803.nc", + "ocean/global_convergence/cosine_bell/QU180/mesh/graph.info": "global_convergence/cosine_bell/QU180/mesh/graph.210803.info", + "ocean/global_convergence/cosine_bell/QU180/init/namelist.ocean": "global_convergence/cosine_bell/QU180/init/namelist.210803.ocean", + "ocean/global_convergence/cosine_bell/QU180/init/initial_state.nc": "global_convergence/cosine_bell/QU180/init/initial_state.210803.nc", + "ocean/global_convergence/cosine_bell/QU210/mesh/mesh.nc": "global_convergence/cosine_bell/QU210/mesh/mesh.210803.nc", + "ocean/global_convergence/cosine_bell/QU210/mesh/graph.info": "global_convergence/cosine_bell/QU210/mesh/graph.210803.info", + "ocean/global_convergence/cosine_bell/QU210/init/namelist.ocean": "global_convergence/cosine_bell/QU210/init/namelist.210803.ocean", + "ocean/global_convergence/cosine_bell/QU210/init/initial_state.nc": "global_convergence/cosine_bell/QU210/init/initial_state.210803.nc", + "ocean/global_convergence/cosine_bell/QU240/mesh/mesh.nc": "global_convergence/cosine_bell/QU240/mesh/mesh.210803.nc", + "ocean/global_convergence/cosine_bell/QU240/mesh/graph.info": "global_convergence/cosine_bell/QU240/mesh/graph.210803.info", + "ocean/global_convergence/cosine_bell/QU240/init/namelist.ocean": "global_convergence/cosine_bell/QU240/init/namelist.210803.ocean", + "ocean/global_convergence/cosine_bell/QU240/init/initial_state.nc": "global_convergence/cosine_bell/QU240/init/initial_state.210803.nc", + "ocean/global_convergence/cosine_bell/QU150/mesh/mesh.nc": "global_convergence/cosine_bell/QU150/mesh/mesh.210803.nc", + "ocean/global_convergence/cosine_bell/QU150/mesh/graph.info": "global_convergence/cosine_bell/QU150/mesh/graph.210803.info", + "ocean/global_convergence/cosine_bell/QU150/init/namelist.ocean": "global_convergence/cosine_bell/QU150/init/namelist.210803.ocean", + "ocean/global_convergence/cosine_bell/QU150/init/initial_state.nc": "global_convergence/cosine_bell/QU150/init/initial_state.210803.nc" +} \ No newline at end of file From 4435b2770c56b2b6711a3ec176c6f56db7b47caf Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Tue, 3 Aug 2021 13:31:56 -0500 Subject: [PATCH 4/6] Add test suite for cosine bell with cached init --- compass/ocean/suites/cosine_bell_cached_init.txt | 4 ++++ compass/ocean/suites/nightly.txt | 6 ++++++ 2 files changed, 10 insertions(+) create mode 100644 compass/ocean/suites/cosine_bell_cached_init.txt diff --git a/compass/ocean/suites/cosine_bell_cached_init.txt b/compass/ocean/suites/cosine_bell_cached_init.txt new file mode 100644 index 0000000000..a07fcb38f1 --- /dev/null +++ b/compass/ocean/suites/cosine_bell_cached_init.txt @@ -0,0 +1,4 @@ +ocean/global_convergence/cosine_bell + cached: QU60_mesh QU60_init QU90_mesh QU90_init QU120_mesh QU120_init + cached: QU150_mesh QU150_init QU180_mesh QU180_init QU210_mesh QU210_init + cached: QU240_mesh QU240_init diff --git a/compass/ocean/suites/nightly.txt b/compass/ocean/suites/nightly.txt index bc873ea8ab..18521637c8 100644 --- a/compass/ocean/suites/nightly.txt +++ b/compass/ocean/suites/nightly.txt @@ -22,6 +22,12 @@ ocean/global_ocean/QU240/EN4_1900/performance_test ocean/global_ocean/QU240/PHC_BGC/init ocean/global_ocean/QU240/PHC_BGC/performance_test +ocean/global_ocean/QUwISC240/mesh + cached +ocean/global_ocean/QUwISC240/PHC/init + cached +ocean/global_ocean/QUwISC240/PHC/performance_test + ocean/ice_shelf_2d/5km/z-star/restart_test ocean/ice_shelf_2d/5km/z-level/restart_test From f93a1f699093a4bad50fcb4961e12bbc08f59f5b Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Wed, 4 Aug 2021 15:06:32 +0200 Subject: [PATCH 5/6] Update docs to include cached outputs --- docs/developers_guide/api.rst | 11 +++ docs/developers_guide/command_line.rst | 54 +++++++++++++ docs/developers_guide/framework.rst | 13 ++++ docs/developers_guide/organization.rst | 102 +++++++++++++++++++++++-- 4 files changed, 173 insertions(+), 7 deletions(-) diff --git a/docs/developers_guide/api.rst b/docs/developers_guide/api.rst index 0d68038a49..8a2da246bf 100644 --- a/docs/developers_guide/api.rst +++ b/docs/developers_guide/api.rst @@ -90,6 +90,17 @@ run run_step +cache +~~~~~ + +.. currentmodule:: compass.cache + +.. autosummary:: + :toctree: generated/ + + update_cache + + Base Classes ^^^^^^^^^^^^ diff --git a/docs/developers_guide/command_line.rst b/docs/developers_guide/command_line.rst index f6620ab183..e041901dea 100644 --- a/docs/developers_guide/command_line.rst +++ b/docs/developers_guide/command_line.rst @@ -276,3 +276,57 @@ Would both accomplish the same thing in this example -- skipping the over the config option. See :ref:`dev_run` for more about the underlying framework. + +.. _dev_compass_cache: + +compass cache +------------- + +``compass`` supports caching outputs from any step in a special database +called ``compass_cache`` (see :ref:`dev_step_input_download`). Files in this +database have a directory structure similar to the work directory (but without +the MPAS core subdirectory, which is redundant). The files include a date stamp +so that new revisions can be added without removing older ones (supported by +older compass versions). See :ref:`dev_step_cached_output` for more details. + +A new command, ``compass cache`` has been added to aid in updating the file +``cached_files.json`` within an MPAS core. This command is only available on +Anvil and Chrysalis, since developers can only copy files from a compass work +directory onto the LCRC server from these two machines. Developers run +``compass cache`` from the base work directory, giving the relative paths of +the step whose outputs should be cached: + +.. code-block:: bash + + compass cache -i ocean/global_ocean/QU240/mesh/mesh \ + ocean/global_ocean/QU240/PHC/init/initial_state + +This will: + +1. copy the output files from the steps directories into the appropriate + ``compass_cache`` location on the LCRC server and + +2. add these files to a local ``ocean_cached_files.json`` that can then be + copied to ``compass/ocean`` as part of a PR to add a cached version of a + step. + +The resulting ``ocean_cached_files.json`` will look something like: + +.. code-block:: json + + { + "ocean/global_ocean/QU240/mesh/mesh/culled_mesh.nc": "global_ocean/QU240/mesh/mesh/culled_mesh.210803.nc", + "ocean/global_ocean/QU240/mesh/mesh/culled_graph.info": "global_ocean/QU240/mesh/mesh/culled_graph.210803.info", + "ocean/global_ocean/QU240/mesh/mesh/critical_passages_mask_final.nc": "global_ocean/QU240/mesh/mesh/critical_passages_mask_final.210803.nc", + "ocean/global_ocean/QU240/PHC/init/initial_state/initial_state.nc": "global_ocean/QU240/PHC/init/initial_state/initial_state.210803.nc", + "ocean/global_ocean/QU240/PHC/init/initial_state/init_mode_forcing_data.nc": "global_ocean/QU240/PHC/init/initial_state/init_mode_forcing_data.210803.nc" + } + +An optional flag ``--date_string`` lets the developer set the date string to +a date they choose. The default is today's date. + +The flag ``--dry_run`` can be used to sanity check the resulting ``json`` file +and the list of files printed to stdout without actually copying the files to +the LCRC server. + +See :ref:`dev_cache` for more about the underlying framework. diff --git a/docs/developers_guide/framework.rst b/docs/developers_guide/framework.rst index f979329df2..c843cc0881 100644 --- a/docs/developers_guide/framework.rst +++ b/docs/developers_guide/framework.rst @@ -99,6 +99,19 @@ from individual steps are stored in log files ``.log`` in the test case's work directory. The results of validation (if any) are displayed in the final stage of running the test case. +.. _dev_cache: + +cache module +~~~~~~~~~~~~ + +The :py:func:`compass.cache.update_cache()` function is used by +``compass cache`` to copy step outputs to the ``compass_cache`` database on +the LCRC server and to update ``_cached_files.json`` files that +contain a mapping between these cached files and the original outputs. This +functionality enables running steps with :ref:`dev_step_cached_output`, which +can be used to skip time-consuming initialization steps for faster development +and debugging. + .. _dev_config: Config files diff --git a/docs/developers_guide/organization.rst b/docs/developers_guide/organization.rst index ac575488c9..be6253c0fe 100644 --- a/docs/developers_guide/organization.rst +++ b/docs/developers_guide/organization.rst @@ -815,13 +815,6 @@ As was the case for test cases, the base class :py:class:`compass.Step` has a large number of attributes that are useful at different stages (init, setup and run) of the step. - logger : logging.Logger - A logger for output from the step - - log_filename : str - At run time, the name of a log file where output/errors from the step - are being logged, or ``None`` if output is to stdout/stderr - Some attributes are available after calling the base class' constructor ``super().__init__()``. These include: @@ -857,6 +850,10 @@ Some attributes are available after calling the base class' constructor ``self.threads`` the number of threads the step will use +``self.cached`` + Whether to get all of the outputs for the step from the database of + cached outputs for the MPAS core that this step belongs to + Another set of attributes is not useful until ``setup()`` is called by the ``compass`` framework: @@ -897,6 +894,10 @@ framework: methods and functions that use the logger to write their output to the log file. +``self.log_filename`` + The name of a log file where output/errors from the step are being logged, + or ``None`` if output is to stdout/stderr + The inputs and outputs should not be altered but they may be used to get file names to read or write. @@ -1544,6 +1545,93 @@ in the test case's :ref:`dev_test_case_init`. The relative path in ``filename`` is with respect to the step's work directory, and is converted to an absolute path internally before the step is run. +.. _dev_step_cached_output: + +Cached output files +~~~~~~~~~~~~~~~~~~~ + +Many ``compass`` test cases and steps are expensive enough that it can become +time consuming to run full workflows to produce meshes and initial conditions +in order to test simulations. Therefore, ``compass`` provides a mechanism for +caching the outputs of each step in a database so that they can be downloaded +and symlinked rather than being computed each time. + +Cached output files are be stored in the ``compass_cache`` database within each +MPAS core's space on that LCRC server (see :ref:`dev_step_input_download`). +If the "cached" version of a step is selected, as we will describe below, each +of the test case's outputs will have a corresponding "input" file added with +the ``target`` being a cache file on the LCRC server and the ``filename`` being +the output file. ``compass`` uses the ``cached_files.json`` database to know +which cache files correspond to which step outputs. + +A developer can indicate that ``compass`` test suite includes steps with cached +outputs in two ways. First, if all steps in a test case should have cached +output, the following notation should be used: + +.. code-block:: none + + ocean/global_ocean/QU240/mesh + cached + ocean/global_ocean/QU240/PHC/init + cached + +That is, the word ``cached`` should appear after the test case on its own line. +The indentation is for visual clarity and is not required. + + +Second, ff only some steps in a test case should have cached output, they need +to be listed explicitly, as follows: + +.. code-block:: none + + ocean/global_ocean/QUwISC240/mesh + cached: mesh + ocean/global_ocean/QUwISC240/PHC/init + cached: initial_state ssh_adjustment + +The line can be indented for visual clarity, but must begin with ``cached:``, +followed by a list of steps separated by a single space. + +Similarly, a user setting up test cases has two mechanisms for specifying which +test cases and steps should have cached outputs. If all steps in a test case +should have cached outputs, the suffix ``c`` can be added to the test number: + +.. code-block:: none + + compass setup -n 90c 91c 92 ... + +In this example, test cases 90 and 91 (``mesh`` and ``init`` test cases from +the ``SOwISC12to60`` global ocean mesh, in this case) are set up with cached +outputs in all steps and 92 (``performance_test``) is not. This approach is +efficient but does not provide any control of which steps use cached outputs +and which do not. + +A much more verbose approach is required if some steps use cached outputs and +others do not within a given test case. Each test case must be set up on its +own with the ``-t`` and ``--cached`` flags as follows: + + +.. code-block:: none + + compass setup -t ocean/global_ocean/QU240/mesh --cached mesh ... + compass setup -t ocean/global_ocean/QU240/PHC/init --cached initial_state ... + ... + +Cache files should be generated by first running the test case as normal, then +running the :ref:`dev_compass_cache` command-line tool at the base of the work +directory, providing the names of the steps whose outputs should be added to +the cache. The resulting ``_cached_files.json`` should be copied +to ``compass//cached_files.json`` in a ``compass`` branch. + +Calls to ``compass cache`` must be made on Chrysalis or Anvil. If outputs were +produced on another machine, they must be transferred to one of these two +machines before calling ``compass cache``. File can be added manually to the +LCRC server and the ``cached_files.json`` databases but this is not +recommended. + +More details on cached outputs are available in the design document +:ref:`design_doc_cached_outputs`. + .. _dev_step_namelists_and_streams: Adding namelist and streams files From 0fa5a4c543c1e4a372f3ff624449649017b54a58 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Mon, 9 Aug 2021 16:21:17 -0500 Subject: [PATCH 6/6] Update database of ocean cached files The latest version includes graph.info files for global ocean test cases. --- compass/ocean/cached_files.json | 74 ++++++++++++++++++--------------- 1 file changed, 40 insertions(+), 34 deletions(-) diff --git a/compass/ocean/cached_files.json b/compass/ocean/cached_files.json index b8871eff1a..9437342c67 100644 --- a/compass/ocean/cached_files.json +++ b/compass/ocean/cached_files.json @@ -1,37 +1,43 @@ { - "ocean/global_ocean/QU240/mesh/mesh/culled_mesh.nc": "global_ocean/QU240/mesh/mesh/culled_mesh.210803.nc", - "ocean/global_ocean/QU240/mesh/mesh/culled_graph.info": "global_ocean/QU240/mesh/mesh/culled_graph.210803.info", - "ocean/global_ocean/QU240/mesh/mesh/critical_passages_mask_final.nc": "global_ocean/QU240/mesh/mesh/critical_passages_mask_final.210803.nc", - "ocean/global_ocean/QU240/PHC/init/initial_state/initial_state.nc": "global_ocean/QU240/PHC/init/initial_state/initial_state.210803.nc", - "ocean/global_ocean/QU240/PHC/init/initial_state/init_mode_forcing_data.nc": "global_ocean/QU240/PHC/init/initial_state/init_mode_forcing_data.210803.nc", - "ocean/global_ocean/QUwISC240/mesh/mesh/culled_mesh.nc": "global_ocean/QUwISC240/mesh/mesh/culled_mesh.210803.nc", - "ocean/global_ocean/QUwISC240/mesh/mesh/culled_graph.info": "global_ocean/QUwISC240/mesh/mesh/culled_graph.210803.info", - "ocean/global_ocean/QUwISC240/mesh/mesh/critical_passages_mask_final.nc": "global_ocean/QUwISC240/mesh/mesh/critical_passages_mask_final.210803.nc", - "ocean/global_ocean/QUwISC240/PHC/init/initial_state/initial_state.nc": "global_ocean/QUwISC240/PHC/init/initial_state/initial_state.210803.nc", - "ocean/global_ocean/QUwISC240/PHC/init/initial_state/init_mode_forcing_data.nc": "global_ocean/QUwISC240/PHC/init/initial_state/init_mode_forcing_data.210803.nc", - "ocean/global_ocean/QUwISC240/PHC/init/ssh_adjustment/adjusted_init.nc": "global_ocean/QUwISC240/PHC/init/ssh_adjustment/adjusted_init.210803.nc", - "ocean/global_ocean/EC30to60/mesh/mesh/culled_mesh.nc": "global_ocean/EC30to60/mesh/mesh/culled_mesh.210803.nc", - "ocean/global_ocean/EC30to60/mesh/mesh/culled_graph.info": "global_ocean/EC30to60/mesh/mesh/culled_graph.210803.info", - "ocean/global_ocean/EC30to60/mesh/mesh/critical_passages_mask_final.nc": "global_ocean/EC30to60/mesh/mesh/critical_passages_mask_final.210803.nc", - "ocean/global_ocean/EC30to60/PHC/init/initial_state/initial_state.nc": "global_ocean/EC30to60/PHC/init/initial_state/initial_state.210803.nc", - "ocean/global_ocean/EC30to60/PHC/init/initial_state/init_mode_forcing_data.nc": "global_ocean/EC30to60/PHC/init/initial_state/init_mode_forcing_data.210803.nc", - "ocean/global_ocean/WC14/mesh/mesh/culled_mesh.nc": "global_ocean/WC14/mesh/mesh/culled_mesh.210803.nc", - "ocean/global_ocean/WC14/mesh/mesh/culled_graph.info": "global_ocean/WC14/mesh/mesh/culled_graph.210803.info", - "ocean/global_ocean/WC14/mesh/mesh/critical_passages_mask_final.nc": "global_ocean/WC14/mesh/mesh/critical_passages_mask_final.210803.nc", - "ocean/global_ocean/WC14/PHC/init/initial_state/initial_state.nc": "global_ocean/WC14/PHC/init/initial_state/initial_state.210803.nc", - "ocean/global_ocean/WC14/PHC/init/initial_state/init_mode_forcing_data.nc": "global_ocean/WC14/PHC/init/initial_state/init_mode_forcing_data.210803.nc", - "ocean/global_ocean/ECwISC30to60/mesh/mesh/culled_mesh.nc": "global_ocean/ECwISC30to60/mesh/mesh/culled_mesh.210803.nc", - "ocean/global_ocean/ECwISC30to60/mesh/mesh/culled_graph.info": "global_ocean/ECwISC30to60/mesh/mesh/culled_graph.210803.info", - "ocean/global_ocean/ECwISC30to60/mesh/mesh/critical_passages_mask_final.nc": "global_ocean/ECwISC30to60/mesh/mesh/critical_passages_mask_final.210803.nc", - "ocean/global_ocean/ECwISC30to60/PHC/init/initial_state/initial_state.nc": "global_ocean/ECwISC30to60/PHC/init/initial_state/initial_state.210803.nc", - "ocean/global_ocean/ECwISC30to60/PHC/init/initial_state/init_mode_forcing_data.nc": "global_ocean/ECwISC30to60/PHC/init/initial_state/init_mode_forcing_data.210803.nc", - "ocean/global_ocean/ECwISC30to60/PHC/init/ssh_adjustment/adjusted_init.nc": "global_ocean/ECwISC30to60/PHC/init/ssh_adjustment/adjusted_init.210803.nc", - "ocean/global_ocean/SOwISC12to60/mesh/mesh/culled_mesh.nc": "global_ocean/SOwISC12to60/mesh/mesh/culled_mesh.210803.nc", - "ocean/global_ocean/SOwISC12to60/mesh/mesh/culled_graph.info": "global_ocean/SOwISC12to60/mesh/mesh/culled_graph.210803.info", - "ocean/global_ocean/SOwISC12to60/mesh/mesh/critical_passages_mask_final.nc": "global_ocean/SOwISC12to60/mesh/mesh/critical_passages_mask_final.210803.nc", - "ocean/global_ocean/SOwISC12to60/PHC/init/initial_state/initial_state.nc": "global_ocean/SOwISC12to60/PHC/init/initial_state/initial_state.210803.nc", - "ocean/global_ocean/SOwISC12to60/PHC/init/initial_state/init_mode_forcing_data.nc": "global_ocean/SOwISC12to60/PHC/init/initial_state/init_mode_forcing_data.210803.nc", - "ocean/global_ocean/SOwISC12to60/PHC/init/ssh_adjustment/adjusted_init.nc": "global_ocean/SOwISC12to60/PHC/init/ssh_adjustment/adjusted_init.210803.nc", + "ocean/global_ocean/QU240/mesh/mesh/culled_mesh.nc": "global_ocean/QU240/mesh/mesh/culled_mesh.210809.nc", + "ocean/global_ocean/QU240/mesh/mesh/culled_graph.info": "global_ocean/QU240/mesh/mesh/culled_graph.210809.info", + "ocean/global_ocean/QU240/mesh/mesh/critical_passages_mask_final.nc": "global_ocean/QU240/mesh/mesh/critical_passages_mask_final.210809.nc", + "ocean/global_ocean/QU240/PHC/init/initial_state/initial_state.nc": "global_ocean/QU240/PHC/init/initial_state/initial_state.210809.nc", + "ocean/global_ocean/QU240/PHC/init/initial_state/init_mode_forcing_data.nc": "global_ocean/QU240/PHC/init/initial_state/init_mode_forcing_data.210809.nc", + "ocean/global_ocean/QU240/PHC/init/initial_state/graph.info": "global_ocean/QU240/PHC/init/initial_state/graph.210809.info", + "ocean/global_ocean/QUwISC240/mesh/mesh/culled_mesh.nc": "global_ocean/QUwISC240/mesh/mesh/culled_mesh.210809.nc", + "ocean/global_ocean/QUwISC240/mesh/mesh/culled_graph.info": "global_ocean/QUwISC240/mesh/mesh/culled_graph.210809.info", + "ocean/global_ocean/QUwISC240/mesh/mesh/critical_passages_mask_final.nc": "global_ocean/QUwISC240/mesh/mesh/critical_passages_mask_final.210809.nc", + "ocean/global_ocean/QUwISC240/PHC/init/initial_state/initial_state.nc": "global_ocean/QUwISC240/PHC/init/initial_state/initial_state.210809.nc", + "ocean/global_ocean/QUwISC240/PHC/init/initial_state/init_mode_forcing_data.nc": "global_ocean/QUwISC240/PHC/init/initial_state/init_mode_forcing_data.210809.nc", + "ocean/global_ocean/QUwISC240/PHC/init/initial_state/graph.info": "global_ocean/QUwISC240/PHC/init/initial_state/graph.210809.info", + "ocean/global_ocean/QUwISC240/PHC/init/ssh_adjustment/adjusted_init.nc": "global_ocean/QUwISC240/PHC/init/ssh_adjustment/adjusted_init.210809.nc", + "ocean/global_ocean/EC30to60/mesh/mesh/culled_mesh.nc": "global_ocean/EC30to60/mesh/mesh/culled_mesh.210809.nc", + "ocean/global_ocean/EC30to60/mesh/mesh/culled_graph.info": "global_ocean/EC30to60/mesh/mesh/culled_graph.210809.info", + "ocean/global_ocean/EC30to60/mesh/mesh/critical_passages_mask_final.nc": "global_ocean/EC30to60/mesh/mesh/critical_passages_mask_final.210809.nc", + "ocean/global_ocean/EC30to60/PHC/init/initial_state/initial_state.nc": "global_ocean/EC30to60/PHC/init/initial_state/initial_state.210809.nc", + "ocean/global_ocean/EC30to60/PHC/init/initial_state/init_mode_forcing_data.nc": "global_ocean/EC30to60/PHC/init/initial_state/init_mode_forcing_data.210809.nc", + "ocean/global_ocean/EC30to60/PHC/init/initial_state/graph.info": "global_ocean/EC30to60/PHC/init/initial_state/graph.210809.info", + "ocean/global_ocean/WC14/mesh/mesh/culled_mesh.nc": "global_ocean/WC14/mesh/mesh/culled_mesh.210809.nc", + "ocean/global_ocean/WC14/mesh/mesh/culled_graph.info": "global_ocean/WC14/mesh/mesh/culled_graph.210809.info", + "ocean/global_ocean/WC14/mesh/mesh/critical_passages_mask_final.nc": "global_ocean/WC14/mesh/mesh/critical_passages_mask_final.210809.nc", + "ocean/global_ocean/WC14/PHC/init/initial_state/initial_state.nc": "global_ocean/WC14/PHC/init/initial_state/initial_state.210809.nc", + "ocean/global_ocean/WC14/PHC/init/initial_state/init_mode_forcing_data.nc": "global_ocean/WC14/PHC/init/initial_state/init_mode_forcing_data.210809.nc", + "ocean/global_ocean/WC14/PHC/init/initial_state/graph.info": "global_ocean/WC14/PHC/init/initial_state/graph.210809.info", + "ocean/global_ocean/ECwISC30to60/mesh/mesh/culled_mesh.nc": "global_ocean/ECwISC30to60/mesh/mesh/culled_mesh.210809.nc", + "ocean/global_ocean/ECwISC30to60/mesh/mesh/culled_graph.info": "global_ocean/ECwISC30to60/mesh/mesh/culled_graph.210809.info", + "ocean/global_ocean/ECwISC30to60/mesh/mesh/critical_passages_mask_final.nc": "global_ocean/ECwISC30to60/mesh/mesh/critical_passages_mask_final.210809.nc", + "ocean/global_ocean/ECwISC30to60/PHC/init/initial_state/initial_state.nc": "global_ocean/ECwISC30to60/PHC/init/initial_state/initial_state.210809.nc", + "ocean/global_ocean/ECwISC30to60/PHC/init/initial_state/init_mode_forcing_data.nc": "global_ocean/ECwISC30to60/PHC/init/initial_state/init_mode_forcing_data.210809.nc", + "ocean/global_ocean/ECwISC30to60/PHC/init/ssh_adjustment/adjusted_init.nc": "global_ocean/ECwISC30to60/PHC/init/ssh_adjustment/adjusted_init.210809.nc", + "ocean/global_ocean/ECwISC30to60/PHC/init/initial_state/graph.info": "global_ocean/ECwISC30to60/PHC/init/initial_state/graph.210809.info", + "ocean/global_ocean/SOwISC12to60/mesh/mesh/culled_mesh.nc": "global_ocean/SOwISC12to60/mesh/mesh/culled_mesh.210810.nc", + "ocean/global_ocean/SOwISC12to60/mesh/mesh/culled_graph.info": "global_ocean/SOwISC12to60/mesh/mesh/culled_graph.210810.info", + "ocean/global_ocean/SOwISC12to60/mesh/mesh/critical_passages_mask_final.nc": "global_ocean/SOwISC12to60/mesh/mesh/critical_passages_mask_final.210810.nc", + "ocean/global_ocean/SOwISC12to60/PHC/init/initial_state/initial_state.nc": "global_ocean/SOwISC12to60/PHC/init/initial_state/initial_state.210810.nc", + "ocean/global_ocean/SOwISC12to60/PHC/init/initial_state/init_mode_forcing_data.nc": "global_ocean/SOwISC12to60/PHC/init/initial_state/init_mode_forcing_data.210810.nc", + "ocean/global_ocean/SOwISC12to60/PHC/init/ssh_adjustment/adjusted_init.nc": "global_ocean/SOwISC12to60/PHC/init/ssh_adjustment/adjusted_init.210810.nc", + "ocean/global_ocean/SOwISC12to60/PHC/init/initial_state/graph.info": "global_ocean/SOwISC12to60/PHC/init/initial_state/graph.210810.info", "ocean/global_convergence/cosine_bell/QU60/mesh/mesh.nc": "global_convergence/cosine_bell/QU60/mesh/mesh.210803.nc", "ocean/global_convergence/cosine_bell/QU60/mesh/graph.info": "global_convergence/cosine_bell/QU60/mesh/graph.210803.info", "ocean/global_convergence/cosine_bell/QU60/init/namelist.ocean": "global_convergence/cosine_bell/QU60/init/namelist.210803.ocean", @@ -60,4 +66,4 @@ "ocean/global_convergence/cosine_bell/QU150/mesh/graph.info": "global_convergence/cosine_bell/QU150/mesh/graph.210803.info", "ocean/global_convergence/cosine_bell/QU150/init/namelist.ocean": "global_convergence/cosine_bell/QU150/init/namelist.210803.ocean", "ocean/global_convergence/cosine_bell/QU150/init/initial_state.nc": "global_convergence/cosine_bell/QU150/init/initial_state.210803.nc" -} \ No newline at end of file +}