Aradhakrishnan gfdl patch 2

.github/workflows/curl.yml

@@ -0,0 +1,57 @@
+name: curl-test
+on: push
+jobs:
+  start-runner:
+    name: Start self-hosted EC2 runner
+    runs-on: ubuntu-latest
+    outputs:
+      label: ${{ steps.start-ec2-runner.outputs.label }}
+      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION }}
+      - name: Start EC2 runner
+        id: start-ec2-runner
+        uses: machulav/ec2-github-runner@v2
+        with:
+          mode: start
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          ec2-image-id: ami-092a9e946def5be32
+          ec2-instance-type: t2.micro
+          subnet-id: subnet-003faaf9f1e386979
+          security-group-id: sg-016e543200ed56a0f
+          #iam-role-name: github-actions-role-a # optional, requires additional permissions
+
+  do-the-job:
+    name: Do the job on the runner
+    needs: start-runner # required to start the main job when the runner is ready
+    runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner
+    steps:
+      - name: Hello World
+        run:  curl --verbose --ipv4 --connect-timeout 8 --max-time 1200 --retry 128 --ftp-ssl --disable-epsv --ftp-pasv -u "anonymous:anonymous" ftp://ftp.gfdl.noaa.gov/perm/oar.gfdl.mdtf/convective_transition_diag_obs_data.tar --output convective_transition_diag_obs_data.tar
+
+  stop-runner:
+    name: Stop self-hosted EC2 runner
+    needs:
+      - start-runner # required to get output from the start-runner job
+      - do-the-job # required to wait when the main job is done
+    runs-on: ubuntu-latest
+    if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION }}
+      - name: Stop EC2 runner
+        uses: machulav/ec2-github-runner@v2
+        with:
+          mode: stop
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          label: ${{ needs.start-runner.outputs.label }}
+          ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }}

.github/workflows/curltest.yml

@@ -0,0 +1,57 @@
+name: do-the-curl
+on: pull_request
+jobs:
+  start-runner:
+    name: Start self-hosted EC2 runner
+    runs-on: ubuntu-latest
+    outputs:
+      label: ${{ steps.start-ec2-runner.outputs.label }}
+      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION }}
+      - name: Start EC2 runner
+        id: start-ec2-runner
+        uses: machulav/ec2-github-runner@v2
+        with:
+          mode: start
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          ec2-image-id: ami-092a9e946def5be32
+          ec2-instance-type: t2.micro
+          subnet-id: subnet-003faaf9f1e386979
+          security-group-id: sg-016e543200ed56a0f
+          #iam-role-name: github-actions-role-a # optional, requires additional permissions
+
+  do-the-curl:
+    name: Do the job on the runner
+    needs: start-runner # required to start the main job when the runner is ready
+    runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner
+    steps:
+      - name: Hello World
+        run:  curl --verbose --ipv4 --connect-timeout 8 --max-time 1200 --retry 128 --ftp-ssl --disable-epsv --ftp-pasv -u "anonymous:anonymous" ftp://ftp.gfdl.noaa.gov/perm/oar.gfdl.mdtf/convective_transition_diag_obs_data.tar --output convective_transition_diag_obs_data.tar
+
+  stop-runner:
+    name: Stop self-hosted EC2 runner
+    needs:
+      - start-runner # required to get output from the start-runner job
+      - do-the-curl # required to wait when the main job is done
+    runs-on: ubuntu-latest
+    if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION }}
+      - name: Stop EC2 runner
+        uses: machulav/ec2-github-runner@v2
+        with:
+          mode: stop
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          label: ${{ needs.start-runner.outputs.label }}
+          ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }}

.github/workflows/mdtf_test_cloud.yml

@@ -0,0 +1,175 @@
+# This file builds and runs a lightweight version of the MDTF test suite.
+# Note that the tests assess functionality of the diagnostics,
+# and do not evaluate data or scientific content.
+name: MDTF_test_cloud
+
+on:
+  push:
+    branches:
+      - main
+      - develop
+  pull_request:
+    branches:
+      - main
+      - develop
+defaults:
+  run:
+    shell: bash -l {0}
+jobs:
+  start-runner:
+    name: Start self-hosted EC2 runner
+    runs-on: ubuntu-latest
+    outputs:
+      label: ${{ steps.start-ec2-runner.outputs.label }}
+      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION }}
+      - name: Start EC2 runner
+        id: start-ec2-runner
+        uses: machulav/ec2-github-runner@v2
+        with:
+          mode: start
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          ec2-image-id: ami-0d9c66960c53029ef
+          ec2-instance-type: t2.xlarge
+          subnet-id: subnet-00d803f7c597997c8 #######subnet-003faaf9f1e386979
+          security-group-id: sg-016e543200ed56a0f
+          #iam-role-name: github-actions-role-a # optional, requires additional permissions
+
+  build:
+    name: MDTF core tests
+    needs: start-runner
+    runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner
+  #  continue-on-error: ${{ matrix.experimental }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        conda-root: [/actions-runner/_work/MDTF-diagnostics/MDTF-diagnostics/3]#, /usr/share/miniconda3, /Users/runner/miniconda3]
+        action-conda-root: [/actions-runner/_work/MDTF-diagnostics/MDTF-diagnostics/3]
+        json-file: ["tests/github_actions_test_cloud_set1.jsonc","tests/github_actions_test_macos_set1.jsonc"]
+        json-file-set2: ["tests/github_actions_test_ubuntu_set2.jsonc", "tests/github_actions_test_macos_set2.jsonc"]
+        json-file-set3: ["tests/github_actions_test_ubuntu_set3.jsonc", "tests/github_actions_test_macos_set3.jsonc"]
+        # if experimental is true, other jobs to run if one fails
+        experimental: [false]
+        exclude:
+          - os: ubuntu-latest
+            conda-root: /Users/runner/miniconda3
+          - os: ubuntu-latest
+            json-file: "tests/github_actions_test_macos_set1.jsonc"
+          - os: ubuntu-latest
+            json-file-set2: "tests/github_actions_test_macos_set2.jsonc"
+          - os: ubuntu-latest
+            json-file-set3: "tests/github_actions_test_macos_set3.jsonc"
+          - conda-root: /usr/share/miniconda3
+            json-file: "tests/github_actions_test_macos_set1.jsonc"
+          - conda-root: /Users/runner/minconda3
+            json-file: "tests/github_actions_test_ubuntu_set1.jsonc"
+          - conda-root: /usr/share/miniconda3
+            json-file-set2: "tests/github_actions_test_macos_set2.jsonc"
+          - conda-root: /Users/runner/minconda3
+            json-file-set2: "tests/github_actions_test_ubuntu_set2.jsonc"
+          - conda-root: /usr/share/miniconda3
+            json-file-set3: "tests/github_actions_test_macos_set3.jsonc"
+          - conda-root: /Users/runner/minconda3
+            json-file-set3: "tests/github_actions_test_ubuntu_set3.jsonc"
+      max-parallel: 2
+    steps:
+    - uses: actions/checkout@v2
+    - name: Download Miniconda 3
+      uses: conda-incubator/setup-miniconda@v2
+      with:
+        miniconda-version: "latest"
+        python-version: 3.8
+    - name: Verify miniconda
+      run: |
+        conda info -a
+        conda config --show
+    - name: Set environment variables
+      run: |
+        echo "POD_OUTPUT=$(echo $PWD/../wkdir/GFDL.Synthetic)" >> $GITHUB_ENV
+    - name: Install Conda Environments
+      run: |
+        # install mamba (https://github.com/mamba-org/mamba) for faster dependency resolution
+        echo "Installing Mamba"
+        conda install mamba -n base -c conda-forge
+        # create the synthetic data environment
+        echo "Creating the _MDTF_synthetic_data environment"
+        mamba env create --force -q -f ./src/conda/_env_synthetic_data.yml
+        echo "Installing Conda Environments"
+        # MDTF-specific setup: install all conda envs
+        ##${{matrix.conda-root}}
+        ./src/conda/conda_env_setup.sh --all --conda_root ${{matrix.action-conda-root}} 
+    - name: Generate Model Data
+      run: |
+        cd ../
+        conda activate _MDTF_synthetic_data
+        pip install mdtf-test-data
+        mkdir mdtf_test_data ; cd mdtf_test_data
+        # generate the data and run unit tests
+        mdtf_synthetic.py -c GFDL --startyear 1 --nyears 10 --unittest
+        mdtf_synthetic.py -c NCAR --startyear 1975 --nyears 7
+        mdtf_synthetic.py -c CMIP --startyear 1 --nyears 10
+        cd ../
+        mkdir wkdir
+    - name: Get Observational Data for Set 1
+      run: |
+        echo "${PWD}"
+        cd ../
+        echo "Available Space"
+        df -h
+        # attempt FTP data fetch
+        # allow 20 min for transfer before timeout; Github actions allows 6 hours for individual
+        # jobs, but we don't want to max out resources that are shared by the NOAA-GFDL repos.
+        curl --verbose --ipv4 --connect-timeout 8 --max-time 1200 --retry 128 --ftp-ssl --disable-epsv --ftp-pasv -u "anonymous:anonymous" ftp://ftp.gfdl.noaa.gov/perm/oar.gfdl.mdtf/convective_transition_diag_obs_data.tar --output convective_transition_diag_obs_data.tar
+        curl --verbose --ipv4 --connect-timeout 8 --max-time 1200 --retry 128 --ftp-ssl --disable-epsv --ftp-pasv -u "anonymous:anonymous" ftp://ftp.gfdl.noaa.gov/perm/oar.gfdl.mdtf/EOF_500hPa_obs_data.tar --output EOF_500hPa_obs_data.tar
+        curl --verbose --ipv4 --connect-timeout 8 --max-time 1200 --retry 128 --ftp-ssl --disable-epsv --ftp-pasv -u "anonymous:anonymous" ftp://ftp.gfdl.noaa.gov/perm/oar.gfdl.mdtf/Wheeler_Kiladis_obs_data.tar --output Wheeler_Kiladis_obs_data.tar
+        curl --verbose --ipv4 --connect-timeout 8 --max-time 1200 --retry 128 --ftp-ssl --disable-epsv --ftp-pasv -u "anonymous:anonymous" ftp://ftp.gfdl.noaa.gov/perm/oar.gfdl.mdtf/MJO_teleconnection_obs_data.tar --output MJO_teleconnection_obs_data.tar
+        curl --verbose --ipv4 --connect-timeout 8 --max-time 1200 --retry 128 --ftp-ssl --disable-epsv --ftp-pasv -u "anonymous:anonymous" ftp://ftp.gfdl.noaa.gov/perm/oar.gfdl.mdtf/MJO_suite_obs_data.tar --output MJO_suite_obs_data.tar
+        curl --verbose --ipv4 --connect-timeout 8 --max-time 1200 --retry 128 --ftp-ssl --disable-epsv --ftp-pasv -u "anonymous:anonymous" ftp://ftp.gfdl.noaa.gov/perm/oar.gfdl.mdtf/precip_diurnal_cycle_obs_data.tar --output precip_diurnal_cycle_obs_data.tar
+        ## make input data directories
+        mkdir -p inputdata/obs_data
+        echo "Untarring test files"
+        tar -xvf convective_transition_diag_obs_data.tar
+        tar -xvf EOF_500hPa_obs_data.tar
+        tar -xvf precip_diurnal_cycle_obs_data.tar
+        tar -xvf MJO_teleconnection_obs_data.tar
+        tar -xvf MJO_suite_obs_data.tar
+        tar -xvf Wheeler_Kiladis_obs_data.tar
+        # clean up tarballs
+        rm -f *.tar
+    - name: Run diagnostic tests set 1
+      run: |
+        echo "POD_OUTPUT is: "
+        echo "${POD_OUTPUT}"
+        conda activate _MDTF_base
+        cat ./mdtf
+        # trivial check that install script worked
+        ./mdtf --version
+        # run the test PODs
+        ./mdtf -v -f ${{matrix.json-file}}
+  stop-runner:
+    name: Stop self-hosted EC2 runner
+    needs:
+      - start-runner # required to get output from the start-runner job
+      - build # required to wait when the main job is done
+    runs-on: ubuntu-latest
+    if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION }}
+      - name: Stop EC2 runner
+        uses: machulav/ec2-github-runner@v2
+        with:
+          mode: stop
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          label: ${{ needs.start-runner.outputs.label }}
+          ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }}     

tests/github_actions_test_cloud_set1.jsonc

@@ -0,0 +1,94 @@
+// Configuration for MDTF-diagnostics driver script self-test.
+// All text to the right of an unquoted "//" is a comment and ignored, as well
+// as blank lines (JSONC quasi-standard.)
+{
+  "case_list" : [
+    // The cases below correspond to the different sample model data sets. Note
+    // that the MDTF package does not currently support analyzing multiple
+    // models in a single invocation. Comment out or delete the first entry and
+    // uncomment the second to run NOAA-GFDL-AM4 only for the MJO_prop_amp POD,
+    // and likewise for the SM_ET_coupling POD.
+    {
+      "CASENAME" : "NCAR.Synthetic",
+      "convention" : "CESM",
+      "FIRSTYR" : 1975,
+      "LASTYR" : 1981,
+      "pod_list": [
+          // Optional: PODs to run for this model only (defaults to all)
+        "convective_transition_diag",
+        "Wheeler_Kiladis",
+        "MJO_suite",
+        "MJO_teleconnection",
+        "precip_diurnal_cycle",
+        "EOF_500hPa"
+      ]
+    }
+  ],
+  // PATHS ---------------------------------------------------------------------
+  // Location of supporting data downloaded when the framework was installed.
+
+  // If a relative path is given, it's resolved relative to the MDTF-diagnostics
+  // code directory. Environment variables (eg, $HOME) can be referenced with a
+  // "$" and will be expended to their current values when the framework runs.
+
+  // Parent directory containing observational data used by individual PODs.
+  "OBS_DATA_ROOT": "../inputdata/obs_data",
+
+  // Parent directory containing results from different models.
+  "MODEL_DATA_ROOT": "../mdtf_test_data",
+
+  // Working directory. Defaults to working directory if blank.
+  "WORKING_DIR": "../wkdir",
+
+  // Directory to write output. The results of each run of the framework will be
+  // put in a subdirectory of this directory.
+  "OUTPUT_DIR": "../wkdir",
+
+  // Location of the Anaconda/miniconda installation to use for managing
+  // dependencies (path returned lsby running `conda info --base`.) If empty,
+  // framework will attempt to determine location of system's conda installation.
+  "conda_root": "/actions-runner/_work/MDTF-diagnostics/MDTF-diagnostics/3",
+
+  // Directory containing the framework-specific conda environments. This should
+  // be equal to the "--env_dir" flag passed to conda_env_setup.sh. If left
+  // blank, the framework will look for its environments in the system default
+  // location.
+  "conda_env_root": "/actions-runner/_work/MDTF-diagnostics/MDTF-diagnostics/3/envs/",
+
+  // SETTINGS ------------------------------------------------------------------
+  // Any command-line option recognized by the mdtf script (type `mdtf --help`)
+  // can be set here, in the form "flag name": "desired setting".
+
+  // Method used to fetch model data.
+  "data_manager": "Local_File",
+
+  // Method used to manage dependencies.
+  "environment_manager": "Conda",
+
+  // Settings affecting what output is generated:
+
+  // Set to true to have PODs save postscript figures in addition to bitmaps.
+  "save_ps": false,
+
+  // Set to true to have PODs save netCDF files of processed data.
+  "save_nc": false,
+
+  // Set to true to save HTML and bitmap plots in a .tar file.
+  "make_variab_tar": false,
+
+  // Set to true to overwrite results in OUTPUT_DIR; otherwise results saved
+  // under a unique name.
+  "overwrite": false,
+
+  // Settings used in debugging:
+
+  // Log verbosity level.
+  "verbose": 1,
+
+  // Set to true for framework test. Data is fetched but PODs are not run.
+  "test_mode": false,
+
+  // Set to true for framework test. No external commands are run and no remote
+  // data is copied. Implies test_mode.
+  "dry_run": false
+}