529 allcontributors

.all-contributorsrc

@@ -0,0 +1,4 @@
+{
+  "projectName": "MDTF-diagnostics",
+  "projectOwner": "aradhakrishnanGFDL"
+}

.github/workflows/docker-build-push-aws.yml

@@ -0,0 +1,53 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+name: Build and push image to Amazon ECR.
+
+on:
+  pull_request:
+    branches:
+      - main
+
+env:
+  AWS_REGION: us-east-1                  # set this to your preferred AWS region, e.g. us-west-1
+  ECR_REPOSITORY: mdtf/mdtf-test        # set this to your Amazon ECR repository name
+  ECS_SERVICE: custom-service                # MY_ECS_SERVICE   set this to your Amazon ECS service name
+  ECS_CLUSTER: mdtf-east1                 # MY_ECS_CLUSTER  set this to your Amazon ECS cluster name
+  ECS_TASK_DEFINITION: ecs-task-defn.json # set this to the path to your Amazon ECS task definition
+                                               # file, e.g. .aws/task-definition.json
+  CONTAINER_NAME: mdtf-test-a1r           # set this to the name of the container in the
+                                               # containerDefinitions section of your task definition MY_CONTAINER_NAME
+
+jobs:
+  deploy:
+    name: push-to-public-ecr-cache-test
+    runs-on: ubuntu-latest
+    environment: dev
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@13d241b293754004c80624b5567555c4a39ffbe3
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ env.AWS_REGION }}
+
+      - name: Build, tag, and push image to Amazon ECR
+        id: build-image
+        uses: aradhakrishnanGFDL/docker-build-with-cache-action@master #support-aws-ecr-public
+        env:
+         ECR_REGISTRY: public.ecr.aws
+         IMAGE_TAG: "latest"
+         ECS_IMAGE: $ECR_REGISTRY/ECR_REPOSITORY:$env.IMAGE_TAG
+        with:
+         registry: public.ecr.aws
+         username: "${{ secrets.AWS_ACCESS_KEY_ID }}"  
+         password: "${{ secrets.AWS_SECRET_ACCESS_KEY }}"  
+         image_name: c6h4h4s8/mdtf/mdtf-test
+         push_image_and_stages: true
+         image_tag: latest

.github/workflows/docker-run-test.yml

@@ -0,0 +1,18 @@
+name: docker-run-minitest1
+on:
+  pull_request:
+    branches: [ main ]
+jobs:
+  container-test-job:
+    runs-on: ubuntu-latest
+    container:
+      image: public.ecr.aws/c6h4h4s8/mdtf/mdtf-test:latest
+      env:
+        NODE_ENV: development
+      options: --cpus 1
+    steps:
+      - name: Check PATH
+        run: (echo $PATH) || (echo "No path")
+      - name: Activate environment
+        shell: bash 
+        run: (eval "$(micromamba shell hook -s bash -p ~/conda)"; micromamba activate; micromamba activate _MDTF_base;mdtf_framework.py --version)

Dockerfile

@@ -0,0 +1,36 @@
+# Base OS
+FROM ubuntu:latest
+FROM mambaorg/micromamba:latest as micromamba
+
+#micromamba set up
+
+USER root
+
+RUN micromamba info
+
+##
+# Container Metadata
+LABEL maintainer="mdtf-leads-AR"
+LABEL version="test-01"
+LABEL description="This is a docker image for the MDTF-diagnostics package"
+# Copy the MDTF-diagnostics package contents from local machine to image 
+ENV CODE_ROOT=/proj/MDTF-diagnostics
+COPY src ${CODE_ROOT}/src                          
+COPY data ${CODE_ROOT}/data
+#COPY diagnostics ${CODE_ROOT}/diagnostics
+COPY mdtf_framework.py ${CODE_ROOT}
+COPY shared ${CODE_ROOT}/shared
+COPY sites ${CODE_ROOT}/sites
+COPY tests ${CODE_ROOT}/tests
+# Install conda environments
+ENV CONDA_ROOT=/opt/conda/
+ENV CONDA_ENV_DIR=/opt/conda/envs
+
+#USER mambauser
+RUN micromamba create -f /proj/MDTF-diagnostics/src/conda/env_base.yml
+RUN micromamba create -f /proj/MDTF-diagnostics/src/conda/env_NCL_base.yml
+
+ENV PATH="${PATH}:/proj/MDTF-diagnostics/"
+#RUN eval "$(micromamba shell hook --shell bash )";micromamba activate _MDTF_base
+ENV PATH=$PATH:"/opt/conda/envs/_MDTF_NCL_base/bin/"
+

README.md

@@ -2,6 +2,8 @@
 
 [![MDTF_test](https://github.com/NOAA-GFDL/MDTF-diagnostics/actions/workflows/mdtf_tests.yml/badge.svg)](https://github.com/NOAA-GFDL/MDTF-diagnostics/actions/workflows/mdtf_tests.yml) [![CodeQL](https://github.com/NOAA-GFDL/MDTF-diagnostics/actions/workflows/codeql.yml/badge.svg)](https://github.com/NOAA-GFDL/MDTF-diagnostics/actions/workflows/codeql.yml) [![Documentation Status](https://readthedocs.org/projects/mdtf-diagnostics/badge/?version=main)](https://mdtf-diagnostics.readthedocs.io/en/main/?badge=main)
 
+This project follows the [all-contributors](https://github.com/all-contributors/all-contributors) specification. Contributions of any kind welcome!
+
 The MDTF-diagnostics package is a portable framework for running process-oriented diagnostics (PODs) on weather and
 climate model data.
 
@@ -282,3 +284,17 @@ Federal law. Any reference to specific commercial products, processes, or servic
 manufacturer, or otherwise, does not constitute or imply their endorsement, recommendation or favoring by the
 Department of Commerce. The Department of Commerce seal and logo, or the seal and logo of a DOC bureau, shall not be
 used in any manner to imply endorsement of any commercial product or activity by DOC or the United States Government.
+
+## Contributors
+
+<!-- ALL-CONTRIBUTORS-LIST:START - Do not remove or modify this section -->
+<!-- prettier-ignore-start -->
+<!-- markdownlint-disable -->
+
+<!-- markdownlint-restore -->
+<!-- prettier-ignore-end -->
+
+<!-- ALL-CONTRIBUTORS-LIST:END -->
+
+[![All Contributors](https://img.shields.io/github/aradhakrishnanGFDL/MDTF-diagnostics/all-contributors?color=ee8449&style=flat-square)](#contributors-)
+

src/conda/micromamba_init.sh

@@ -162,4 +162,3 @@ else
     fi
 fi
 unset __conda_setup
-

src/environment_manager.py

@@ -148,19 +148,11 @@ def __init__(self, config: util.NameSpace, log):
             cmd = f"{self.conda_dir}/conda_init.sh {config.conda_root}"
 
         try:
-            conda_info = util.run_shell_command(
-                cmd,
-                log=log
-            )
-            for line in conda_info:
-                key, val = line.split('=')
-                if key == '_CONDA_EXE':
-                    self.conda_exe = val
-                    assert os.path.exists(self.conda_exe)
-                elif key == '_CONDA_ROOT':
-                    self.conda_root = val
+                print('Looking for micromamba') 
+                self.conda_exe = 'micromamba'
+                self.conda_root = '/opt/conda' 
         except Exception as exc:
-            raise util.PodRuntimeError("Can't find conda.") from exc
+                raise util.PodRuntimeError("Can't find conda.") from exc
 
         # find where environments are installed
         self.conda_env_root = config.conda_env_root
@@ -234,17 +226,6 @@ def activate_env_commands(self, env_name):
         # conda_init for bash defines conda as a shell function; will get error
         # if we try to call the conda executable directly
         conda_prefix = os.path.join(self.conda_env_root, env_name)
-        if os.path.split(self.conda_exe)[-1] == 'micromamba':
-            return [
-                f'source {self.conda_dir}/micromamba_init.sh --micromamba_exe {self.conda_exe}'
-                f' --micromamba_root {self.conda_root}',
-                f'micromamba activate {conda_prefix}'
-            ]
-        else:
-            return [
-                f'source {self.conda_dir}/conda_init.sh {self.conda_root}',
-                f'conda activate {conda_prefix}'
-            ]
 
     def deactivate_env_commands(self, env_name):
         return []
@@ -280,9 +261,6 @@ class SubprocessRuntimePODWrapper:
     env_vars: dict = dataclasses.field(default_factory=dict)
     process: typing.Any = dataclasses.field(default=None, init=False)
 
-    def __init__(self, pod):
-        self.pod = pod
-
     def set_pod_env_vars(self, pod, cases: dict):
         """Sets all environment variables for the POD: paths and names of each
         variable and coordinate. Raise a :class:`~src.util.exceptions.WormKeyError`

tests/docker_test_set1.jsonc

@@ -0,0 +1,91 @@
+// Configuration for MDTF-diagnostics driver script self-test.
+// All text to the right of an unquoted "//" is a comment and ignored, as well
+// as blank lines (JSONC quasi-standard.)
+{
+    "case_list" : [
+      // The cases below correspond to the different sample model data sets. Note
+      // that the MDTF package does not currently support analyzing multiple
+      // models in a single invocation. Comment out or delete the first entry and
+      // uncomment the second to run NOAA-GFDL-AM4 only for the MJO_prop_amp POD,
+      // and likewise for the SM_ET_coupling POD.
+      {
+        "CASENAME" : "GFDL.Synthetic",
+        "convention" : "GFDL",
+        "FIRSTYR" : 1,
+        "LASTYR" : 10,
+        "pod_list": [
+            // Optional: PODs to run for this model only (defaults to all)
+          "MJO_prop_amp"
+        ]
+      }
+    ],
+    // PATHS ---------------------------------------------------------------------
+    // Location of supporting data downloaded when the framework was installed.
+
+    // If a relative path is given, it's resolved relative to the MDTF-diagnostics
+    // code directory. Environment variables (eg, $HOME) can be referenced with a
+    // "$" and will be expended to their current values when the framework runs.
+
+    // Parent directory containing observational data used by individual PODs.
+    //"OBS_DATA_ROOT": "../travis_test/obs_data",
+    "OBS_DATA_ROOT": "../inputdata/obs_data",
+
+    // Parent directory containing results from different models.
+    //"MODEL_DATA_ROOT": "../travis_test/model/",
+    "MODEL_DATA_ROOT": "/proj/inputdata/mdtf_test_data",
+
+    // Working directory. Defaults to working directory if blank.
+    "WORKING_DIR": "../wkdir",
+
+    // Directory to write output. The results of each run of the framework will be
+    // put in a subdirectory of this directory.
+    "OUTPUT_DIR": "../wkdir",
+
+    // Location of the Anaconda/miniconda installation to use for managing
+    // dependencies (path returned lsby running `conda info --base`.) If empty,
+    // framework will attempt to determine location of system's conda installation.
+    "conda_root": "/opt/conda",
+
+    // Directory containing the framework-specific conda environments. This should
+    // be equal to the "--env_dir" flag passed to conda_env_setup.sh. If left
+    // blank, the framework will look for its environments in the system default
+    // location.
+    "conda_env_root": "/opt/conda/envs/",
+
+    // SETTINGS ------------------------------------------------------------------
+    // Any command-line option recognized by the mdtf script (type `mdtf --help`)
+    // can be set here, in the form "flag name": "desired setting".
+
+    // Method used to fetch model data.
+    "data_manager": "Local_File",
+
+    // Method used to manage dependencies.
+    "environment_manager": "Conda",
+
+    // Settings affecting what output is generated:
+
+    // Set to true to have PODs save postscript figures in addition to bitmaps.
+    "save_ps": false,
+
+    // Set to true to have PODs save netCDF files of processed data.
+    "save_nc": false,
+
+    // Set to true to save HTML and bitmap plots in a .tar file.
+    "make_variab_tar": false,
+
+    // Set to true to overwrite results in OUTPUT_DIR; otherwise results saved
+    // under a unique name.
+    "overwrite": false,
+
+    // Settings used in debugging:
+
+    // Log verbosity level.
+    "verbose": 1,
+
+    // Set to true for framework test. Data is fetched but PODs are not run.
+    "test_mode": false,
+
+    // Set to true for framework test. No external commands are run and no remote
+    // data is copied. Implies test_mode.
+    "dry_run": false
+  }