From 9ffbb9e4035835d01407297a582935de20af5817 Mon Sep 17 00:00:00 2001
From: Aparna Radhakrishnan <aparna.radhakrishnan@noaa.gov>
Date: Fri, 19 Mar 2021 16:25:14 -0400
Subject: [PATCH 01/34] Create main.yml

quick test to see if our docker image works
---
 .github/workflows/main.yml | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 .github/workflows/main.yml

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
new file mode 100644
index 000000000..8f4804bbe
--- /dev/null
+++ b/.github/workflows/main.yml
@@ -0,0 +1,10 @@
+on: [push]
+
+jobs:
+  mdtf_docker_test:
+    runs-on: ubuntu-latest
+    name: A job to print mdtf version
+    steps:
+    - name: Hello world action step
+      id: hello
+      uses: aradhakrishnanGFDL/MDTF-diagnostics@dev

From 4660afc3d091e064dffec547e979df7c4d162e08 Mon Sep 17 00:00:00 2001
From: Aparna Radhakrishnan <aparna.radhakrishnan@noaa.gov>
Date: Fri, 19 Mar 2021 16:27:48 -0400
Subject: [PATCH 02/34] Update main.yml

---
 .github/workflows/main.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 8f4804bbe..6f795dbdb 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -5,6 +5,5 @@ jobs:
     runs-on: ubuntu-latest
     name: A job to print mdtf version
     steps:
-    - name: Hello world action step
-      id: hello
+    - name: build image
       uses: aradhakrishnanGFDL/MDTF-diagnostics@dev

From 2efda9603b5bfec20d2c25760ea82964153fa96b Mon Sep 17 00:00:00 2001
From: Aparna Radhakrishnan <aparna.radhakrishnan@noaa.gov>
Date: Wed, 15 Sep 2021 11:34:53 -0400
Subject: [PATCH 03/34] Create actions-test1.yml

testing actions
---
 .github/workflows/actions-test1.yml | 56 +++++++++++++++++++++++++++++
 1 file changed, 56 insertions(+)
 create mode 100644 .github/workflows/actions-test1.yml

diff --git a/.github/workflows/actions-test1.yml b/.github/workflows/actions-test1.yml
new file mode 100644
index 000000000..e0d331df2
--- /dev/null
+++ b/.github/workflows/actions-test1.yml
@@ -0,0 +1,56 @@
+name: do-the-job
+on: pull_request
+jobs:
+  start-runner:
+    name: Start self-hosted EC2 runner
+    runs-on: ubuntu-latest
+    outputs:
+      label: ${{ steps.start-ec2-runner.outputs.label }}
+      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION }}
+      - name: Start EC2 runner
+        id: start-ec2-runner
+        uses: machulav/ec2-github-runner@v2
+        with:
+          mode: start
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          ec2-image-id: ami-092a9e946def5be32
+          ec2-instance-type: t2.micro
+          subnet-id: subnet-003faaf9f1e386979
+          security-group-id: sg-016e543200ed56a0f
+          #iam-role-name: github-actions-role-a # optional, requires additional permissions
+          
+  do-the-job:
+    name: Do the job on the runner
+    needs: start-runner # required to start the main job when the runner is ready
+    runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner
+    steps:
+      - name: Hello World
+        run: echo 'Hello MDTF!'
+  stop-runner:
+    name: Stop self-hosted EC2 runner
+    needs:
+      - start-runner # required to get output from the start-runner job
+      - do-the-job # required to wait when the main job is done
+    runs-on: ubuntu-latest
+    if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION }}
+      - name: Stop EC2 runner
+        uses: machulav/ec2-github-runner@v2
+        with:
+          mode: stop
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          label: ${{ needs.start-runner.outputs.label }}
+          ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }}

From 5441911fee975948d942d30b784e1943a4a87ae4 Mon Sep 17 00:00:00 2001
From: Aparna Radhakrishnan <aparna.radhakrishnan@noaa.gov>
Date: Wed, 15 Sep 2021 11:59:46 -0400
Subject: [PATCH 04/34] Update main.yml

---
 .github/workflows/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 6f795dbdb..884cec1fb 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -6,4 +6,4 @@ jobs:
     name: A job to print mdtf version
     steps:
     - name: build image
-      uses: aradhakrishnanGFDL/MDTF-diagnostics@dev
+      uses: aradhakrishnanGFDL/MDTF-diagnostics@main

From bf90b187d596f7e26fddf49c68beeda0d5deecb2 Mon Sep 17 00:00:00 2001
From: Aparna Radhakrishnan <aparna.radhakrishnan@noaa.gov>
Date: Wed, 15 Sep 2021 12:41:50 -0400
Subject: [PATCH 05/34] Update main.yml

---
 .github/workflows/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 884cec1fb..da3b0fb91 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -6,4 +6,4 @@ jobs:
     name: A job to print mdtf version
     steps:
     - name: build image
-      uses: aradhakrishnanGFDL/MDTF-diagnostics@main
+      uses: aradhakrishnanGFDL/MDTF-diagnostics@aradhakrishnanGFDL-patch-2

From f2a4feb42401e54e1acb2c9a3c33ce5c1dd3925d Mon Sep 17 00:00:00 2001
From: Aparna Radhakrishnan <aparna.radhakrishnan@noaa.gov>
Date: Wed, 15 Sep 2021 12:46:51 -0400
Subject: [PATCH 06/34] Delete main.yml

---
 .github/workflows/main.yml | 9 ---------
 1 file changed, 9 deletions(-)
 delete mode 100644 .github/workflows/main.yml

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
deleted file mode 100644
index da3b0fb91..000000000
--- a/.github/workflows/main.yml
+++ /dev/null
@@ -1,9 +0,0 @@
-on: [push]
-
-jobs:
-  mdtf_docker_test:
-    runs-on: ubuntu-latest
-    name: A job to print mdtf version
-    steps:
-    - name: build image
-      uses: aradhakrishnanGFDL/MDTF-diagnostics@aradhakrishnanGFDL-patch-2

From 80e920398fdca9d3333b1c2e1d2fe9a0c5a008fa Mon Sep 17 00:00:00 2001
From: Aparna Radhakrishnan <aparna.radhakrishnan@noaa.gov>
Date: Wed, 15 Sep 2021 13:12:26 -0400
Subject: [PATCH 07/34] Create mdtf_test_cloud.yml

---
 .github/workflows/mdtf_test_cloud.yml | 202 ++++++++++++++++++++++++++
 1 file changed, 202 insertions(+)
 create mode 100644 .github/workflows/mdtf_test_cloud.yml

diff --git a/.github/workflows/mdtf_test_cloud.yml b/.github/workflows/mdtf_test_cloud.yml
new file mode 100644
index 000000000..ad6aebbd1
--- /dev/null
+++ b/.github/workflows/mdtf_test_cloud.yml
@@ -0,0 +1,202 @@
+# This file builds and runs a lightweight version of the MDTF test suite.
+# Note that the tests assess functionality of the diagnostics,
+# and do not evaluate data or scientific content.
+name: MDTF_test
+
+on:
+  push:
+    branches:
+      - main
+      - develop
+  pull_request:
+    branches:
+      - main
+      - develop
+defaults:
+  run:
+    shell: bash -l {0}
+jobs:
+  start-runner:
+    name: Start self-hosted EC2 runner
+    runs-on: ubuntu-latest
+    outputs:
+      label: ${{ steps.start-ec2-runner.outputs.label }}
+      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION }}
+      - name: Start EC2 runner
+        id: start-ec2-runner
+        uses: machulav/ec2-github-runner@v2
+        with:
+          mode: start
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          ec2-image-id: ami-092a9e946def5be32
+          ec2-instance-type: t2.micro
+          subnet-id: subnet-003faaf9f1e386979
+          security-group-id: sg-016e543200ed56a0f
+          #iam-role-name: github-actions-role-a # optional, requires additional permissions
+          
+  build:
+    runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner
+    continue-on-error: ${{ matrix.experimental }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        conda-root: [/usr/share/miniconda3, /Users/runner/miniconda3]
+        json-file: ["tests/github_actions_test_ubuntu_set1.jsonc","tests/github_actions_test_macos_set1.jsonc"]
+        json-file-set2: ["tests/github_actions_test_ubuntu_set2.jsonc", "tests/github_actions_test_macos_set2.jsonc"]
+        json-file-set3: ["tests/github_actions_test_ubuntu_set3.jsonc", "tests/github_actions_test_macos_set3.jsonc"]
+        # if experimental is true, other jobs to run if one fails
+        experimental: [false]
+        exclude:
+          - os: ubuntu-latest
+            conda-root: /Users/runner/miniconda3
+          - os: ubuntu-latest
+            json-file: "tests/github_actions_test_macos_set1.jsonc"
+          - os: ubuntu-latest
+            json-file-set2: "tests/github_actions_test_macos_set2.jsonc"
+          - os: ubuntu-latest
+            json-file-set3: "tests/github_actions_test_macos_set3.jsonc"
+          - os: macos-latest
+            conda-root: /usr/share/miniconda3
+          - os: macos-latest
+            json-file: "tests/github_actions_test_ubuntu_set1.jsonc"
+          - os: macos-latest
+            json-file-set2: "tests/github_actions_test_ubuntu_set2.jsonc"
+          - os: macos-latest
+            json-file-set3: "tests/github_actions_test_ubuntu_set3.jsonc"
+          - conda-root: /usr/share/miniconda3
+            json-file: "tests/github_actions_test_macos_set1.jsonc"
+          - conda-root: /Users/runner/minconda3
+            json-file: "tests/github_actions_test_ubuntu_set1.jsonc"
+          - conda-root: /usr/share/miniconda3
+            json-file-set2: "tests/github_actions_test_macos_set2.jsonc"
+          - conda-root: /Users/runner/minconda3
+            json-file-set2: "tests/github_actions_test_ubuntu_set2.jsonc"
+          - conda-root: /usr/share/miniconda3
+            json-file-set3: "tests/github_actions_test_macos_set3.jsonc"
+          - conda-root: /Users/runner/minconda3
+            json-file-set3: "tests/github_actions_test_ubuntu_set3.jsonc"
+      max-parallel: 2
+    steps:
+    - uses: actions/checkout@v2
+    - name: Download Miniconda 3
+      uses: conda-incubator/setup-miniconda@v2
+      with:
+        miniconda-version: "latest"
+        python-version: 3.8
+    - name: Verify miniconda
+      run: |
+        conda info -a
+    - name: Install XQuartz if macOS
+      if: ${{ matrix.os == 'macos-latest' && matrix.conda-root == '/Users/runner/miniconda3' }}
+      run: |
+        echo "Installing XQuartz"
+        brew install --cask xquartz
+    - name: Set environment variables
+      run: |
+        echo "POD_OUTPUT=$(echo $PWD/../wkdir/GFDL.Synthetic)" >> $GITHUB_ENV
+    - name: Install Conda Environments
+      run: |
+        # install mamba (https://github.com/mamba-org/mamba) for faster dependency resolution
+        echo "Installing Mamba"
+        conda install mamba -n base -c conda-forge
+        # create the synthetic data environment
+        echo "Creating the _MDTF_synthetic_data environment"
+        mamba env create --force -q -f ./src/conda/_env_synthetic_data.yml
+        echo "Installing Conda Environments"
+        # MDTF-specific setup: install all conda envs
+        ./src/conda/conda_env_setup.sh --all --conda_root ${{matrix.conda-root}}
+    - name: Generate Model Data
+      run: |
+        cd ../
+        conda activate _MDTF_synthetic_data
+        pip install mdtf-test-data
+        mkdir mdtf_test_data ; cd mdtf_test_data
+        # generate the data and run unit tests
+        mdtf_synthetic.py -c GFDL --startyear 1 --nyears 10 --unittest
+        mdtf_synthetic.py -c NCAR --startyear 1975 --nyears 7
+        mdtf_synthetic.py -c CMIP --startyear 1 --nyears 10
+        cd ../
+        mkdir wkdir
+    - name: Get Observational Data for Set 1
+      run: |
+        echo "${PWD}"
+        cd ../
+        echo "Available Space"
+        df -h
+        # attempt FTP data fetch
+        # allow 20 min for transfer before timeout; Github actions allows 6 hours for individual
+        # jobs, but we don't want to max out resources that are shared by the NOAA-GFDL repos.
+        curl --verbose --ipv4 --connect-timeout 8 --max-time 1200 --retry 128 --ftp-ssl --disable-epsv --ftp-pasv -u "anonymous:anonymous" ftp://ftp.gfdl.noaa.gov/perm/oar.gfdl.mdtf/convective_transition_diag_obs_data.tar --output convective_transition_diag_obs_data.tar
+        curl --verbose --ipv4 --connect-timeout 8 --max-time 1200 --retry 128 --ftp-ssl --disable-epsv --ftp-pasv -u "anonymous:anonymous" ftp://ftp.gfdl.noaa.gov/perm/oar.gfdl.mdtf/EOF_500hPa_obs_data.tar --output EOF_500hPa_obs_data.tar
+        curl --verbose --ipv4 --connect-timeout 8 --max-time 1200 --retry 128 --ftp-ssl --disable-epsv --ftp-pasv -u "anonymous:anonymous" ftp://ftp.gfdl.noaa.gov/perm/oar.gfdl.mdtf/Wheeler_Kiladis_obs_data.tar --output Wheeler_Kiladis_obs_data.tar
+        curl --verbose --ipv4 --connect-timeout 8 --max-time 1200 --retry 128 --ftp-ssl --disable-epsv --ftp-pasv -u "anonymous:anonymous" ftp://ftp.gfdl.noaa.gov/perm/oar.gfdl.mdtf/MJO_teleconnection_obs_data.tar --output MJO_teleconnection_obs_data.tar
+        curl --verbose --ipv4 --connect-timeout 8 --max-time 1200 --retry 128 --ftp-ssl --disable-epsv --ftp-pasv -u "anonymous:anonymous" ftp://ftp.gfdl.noaa.gov/perm/oar.gfdl.mdtf/MJO_suite_obs_data.tar --output MJO_suite_obs_data.tar
+        curl --verbose --ipv4 --connect-timeout 8 --max-time 1200 --retry 128 --ftp-ssl --disable-epsv --ftp-pasv -u "anonymous:anonymous" ftp://ftp.gfdl.noaa.gov/perm/oar.gfdl.mdtf/precip_diurnal_cycle_obs_data.tar --output precip_diurnal_cycle_obs_data.tar
+        ## make input data directories
+        mkdir -p inputdata/obs_data
+        echo "Untarring test files"
+        tar -xvf convective_transition_diag_obs_data.tar
+        tar -xvf EOF_500hPa_obs_data.tar
+        tar -xvf precip_diurnal_cycle_obs_data.tar
+        tar -xvf MJO_teleconnection_obs_data.tar
+        tar -xvf MJO_suite_obs_data.tar
+        tar -xvf Wheeler_Kiladis_obs_data.tar
+        # clean up tarballs
+        rm -f *.tar
+    - name: Run diagnostic tests set 1
+      run: |
+        echo "POD_OUTPUT is: "
+        echo "${POD_OUTPUT}"
+        conda activate _MDTF_base
+        cat ./mdtf
+        # trivial check that install script worked
+        ./mdtf --version
+        # run the test PODs
+        ./mdtf -v -f ${{matrix.json-file}}
+    - name: Get observational data for set 2
+      run: |
+        echo "${PWD}"
+        # remove data from previous run
+        # Actions moves you to the root repo directory in every step, so need to cd again
+        cd ../inputdata/obs_data
+        echo "deleting obs data from set 1"
+        rm -rf *
+        cd ../../
+        echo "Available Space"
+        df -h
+        # attempt FTP data fetch
+        # allow 20 min for transfer before timeout; Github actions allows 6 hours for individual
+        # jobs, but we don't want to max out resources that are shared by the NOAA-GFDL repos.
+        curl --verbose --ipv4 --connect-timeout 8 --max-time 1200 --retry 128 --ftp-ssl --disable-epsv --ftp-pasv -u "anonymous:anonymous" ftp://ftp.gfdl.noaa.gov/perm/oar.gfdl.mdtf/MJO_prop_amp_obs_data.tar --output MJO_prop_amp_obs_data.tar
+        echo "Untarring set 2 test files"
+        tar -xvf MJO_prop_amp_obs_data.tar
+        # clean up tarballs
+        rm -f *.tar
+  stop-runner:
+    name: Stop self-hosted EC2 runner
+    needs:
+      - start-runner # required to get output from the start-runner job
+      - build # required to wait when the main job is done
+    runs-on: ubuntu-latest
+    if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION }}
+      - name: Stop EC2 runner
+        uses: machulav/ec2-github-runner@v2
+        with:
+          mode: stop
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          label: ${{ needs.start-runner.outputs.label }}
+          ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }}     

From 6269acf00136157eb02537a97457d1303eb82393 Mon Sep 17 00:00:00 2001
From: Aparna Radhakrishnan <aparna.radhakrishnan@noaa.gov>
Date: Wed, 15 Sep 2021 13:21:56 -0400
Subject: [PATCH 08/34] Update mdtf_test_cloud.yml

---
 .github/workflows/mdtf_test_cloud.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/mdtf_test_cloud.yml b/.github/workflows/mdtf_test_cloud.yml
index ad6aebbd1..df6d293d0 100644
--- a/.github/workflows/mdtf_test_cloud.yml
+++ b/.github/workflows/mdtf_test_cloud.yml
@@ -1,7 +1,7 @@
 # This file builds and runs a lightweight version of the MDTF test suite.
 # Note that the tests assess functionality of the diagnostics,
 # and do not evaluate data or scientific content.
-name: MDTF_test
+name: MDTF_test_cloud
 
 on:
   push:

From 4413a7e5fe6c6331a3beeac91359d270af221cfc Mon Sep 17 00:00:00 2001
From: Aparna Radhakrishnan <aparna.radhakrishnan@noaa.gov>
Date: Wed, 15 Sep 2021 13:22:59 -0400
Subject: [PATCH 09/34] Delete main.yml

---
 .github/workflows/main.yml | 9 ---------
 1 file changed, 9 deletions(-)
 delete mode 100644 .github/workflows/main.yml

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
deleted file mode 100644
index 6f795dbdb..000000000
--- a/.github/workflows/main.yml
+++ /dev/null
@@ -1,9 +0,0 @@
-on: [push]
-
-jobs:
-  mdtf_docker_test:
-    runs-on: ubuntu-latest
-    name: A job to print mdtf version
-    steps:
-    - name: build image
-      uses: aradhakrishnanGFDL/MDTF-diagnostics@dev

From c4a6140d59f86148e50f3e7b5e965d3f3f80b77a Mon Sep 17 00:00:00 2001
From: Aparna Radhakrishnan <aparna.radhakrishnan@noaa.gov>
Date: Wed, 15 Sep 2021 13:37:48 -0400
Subject: [PATCH 10/34] Update mdtf_test_cloud.yml

---
 .github/workflows/mdtf_test_cloud.yml | 17 ++---------------
 1 file changed, 2 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/mdtf_test_cloud.yml b/.github/workflows/mdtf_test_cloud.yml
index df6d293d0..830170fa8 100644
--- a/.github/workflows/mdtf_test_cloud.yml
+++ b/.github/workflows/mdtf_test_cloud.yml
@@ -43,10 +43,10 @@ jobs:
           
   build:
     runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner
-    continue-on-error: ${{ matrix.experimental }}
+  #  continue-on-error: ${{ matrix.experimental }}
     strategy:
       matrix:
-        os: [ubuntu-latest, macos-latest]
+        os: [ubuntu-latest]
         conda-root: [/usr/share/miniconda3, /Users/runner/miniconda3]
         json-file: ["tests/github_actions_test_ubuntu_set1.jsonc","tests/github_actions_test_macos_set1.jsonc"]
         json-file-set2: ["tests/github_actions_test_ubuntu_set2.jsonc", "tests/github_actions_test_macos_set2.jsonc"]
@@ -62,14 +62,6 @@ jobs:
             json-file-set2: "tests/github_actions_test_macos_set2.jsonc"
           - os: ubuntu-latest
             json-file-set3: "tests/github_actions_test_macos_set3.jsonc"
-          - os: macos-latest
-            conda-root: /usr/share/miniconda3
-          - os: macos-latest
-            json-file: "tests/github_actions_test_ubuntu_set1.jsonc"
-          - os: macos-latest
-            json-file-set2: "tests/github_actions_test_ubuntu_set2.jsonc"
-          - os: macos-latest
-            json-file-set3: "tests/github_actions_test_ubuntu_set3.jsonc"
           - conda-root: /usr/share/miniconda3
             json-file: "tests/github_actions_test_macos_set1.jsonc"
           - conda-root: /Users/runner/minconda3
@@ -93,11 +85,6 @@ jobs:
     - name: Verify miniconda
       run: |
         conda info -a
-    - name: Install XQuartz if macOS
-      if: ${{ matrix.os == 'macos-latest' && matrix.conda-root == '/Users/runner/miniconda3' }}
-      run: |
-        echo "Installing XQuartz"
-        brew install --cask xquartz
     - name: Set environment variables
       run: |
         echo "POD_OUTPUT=$(echo $PWD/../wkdir/GFDL.Synthetic)" >> $GITHUB_ENV

From 5829bc325b86df85aec2d9c666f06ab3f5a3bd95 Mon Sep 17 00:00:00 2001
From: Aparna Radhakrishnan <aparna.radhakrishnan@noaa.gov>
Date: Wed, 15 Sep 2021 13:43:26 -0400
Subject: [PATCH 11/34] Update mdtf_test_cloud.yml

---
 .github/workflows/mdtf_test_cloud.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/mdtf_test_cloud.yml b/.github/workflows/mdtf_test_cloud.yml
index 830170fa8..8f835b727 100644
--- a/.github/workflows/mdtf_test_cloud.yml
+++ b/.github/workflows/mdtf_test_cloud.yml
@@ -42,6 +42,8 @@ jobs:
           #iam-role-name: github-actions-role-a # optional, requires additional permissions
           
   build:
+    name: MDTF core tests
+    needs: start-runner
     runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner
   #  continue-on-error: ${{ matrix.experimental }}
     strategy:

From b5cefb5f5cc21e2e812b4a2022b05f0931b1b0f2 Mon Sep 17 00:00:00 2001
From: Aparna Radhakrishnan <aparna.radhakrishnan@noaa.gov>
Date: Wed, 15 Sep 2021 17:36:48 -0400
Subject: [PATCH 12/34] Update mdtf_test_cloud.yml

---
 .github/workflows/mdtf_test_cloud.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/mdtf_test_cloud.yml b/.github/workflows/mdtf_test_cloud.yml
index 8f835b727..baca8452c 100644
--- a/.github/workflows/mdtf_test_cloud.yml
+++ b/.github/workflows/mdtf_test_cloud.yml
@@ -87,6 +87,10 @@ jobs:
     - name: Verify miniconda
       run: |
         conda info -a
+        conda config --show
+        which conda
+        which miniconda3
+        
     - name: Set environment variables
       run: |
         echo "POD_OUTPUT=$(echo $PWD/../wkdir/GFDL.Synthetic)" >> $GITHUB_ENV

From a6f2e26e213c3ff407b75ca001e034bf572242f5 Mon Sep 17 00:00:00 2001
From: Aparna Radhakrishnan <aparna.radhakrishnan@noaa.gov>
Date: Wed, 15 Sep 2021 17:41:18 -0400
Subject: [PATCH 13/34] Update mdtf_test_cloud.yml

---
 .github/workflows/mdtf_test_cloud.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/mdtf_test_cloud.yml b/.github/workflows/mdtf_test_cloud.yml
index baca8452c..249552132 100644
--- a/.github/workflows/mdtf_test_cloud.yml
+++ b/.github/workflows/mdtf_test_cloud.yml
@@ -49,7 +49,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        conda-root: [/usr/share/miniconda3, /Users/runner/miniconda3]
+        conda-root: [/actions-runner/_work/MDTF-diagnostics/MDTF-diagnostics/3/condabin/conda, /usr/share/miniconda3, /Users/runner/miniconda3]
         json-file: ["tests/github_actions_test_ubuntu_set1.jsonc","tests/github_actions_test_macos_set1.jsonc"]
         json-file-set2: ["tests/github_actions_test_ubuntu_set2.jsonc", "tests/github_actions_test_macos_set2.jsonc"]
         json-file-set3: ["tests/github_actions_test_ubuntu_set3.jsonc", "tests/github_actions_test_macos_set3.jsonc"]

From 3b351a8051d9b4ac65c09dd3fdc704313e416db0 Mon Sep 17 00:00:00 2001
From: Aparna Radhakrishnan <aparna.radhakrishnan@noaa.gov>
Date: Wed, 15 Sep 2021 17:45:12 -0400
Subject: [PATCH 14/34] Update mdtf_test_cloud.yml

---
 .github/workflows/mdtf_test_cloud.yml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/.github/workflows/mdtf_test_cloud.yml b/.github/workflows/mdtf_test_cloud.yml
index 249552132..0b054a3ae 100644
--- a/.github/workflows/mdtf_test_cloud.yml
+++ b/.github/workflows/mdtf_test_cloud.yml
@@ -88,9 +88,6 @@ jobs:
       run: |
         conda info -a
         conda config --show
-        which conda
-        which miniconda3
-        
     - name: Set environment variables
       run: |
         echo "POD_OUTPUT=$(echo $PWD/../wkdir/GFDL.Synthetic)" >> $GITHUB_ENV

From 0738d885c44c84057087fb23918db8fa28710689 Mon Sep 17 00:00:00 2001
From: Aparna Radhakrishnan <aparna.radhakrishnan@noaa.gov>
Date: Wed, 15 Sep 2021 17:57:39 -0400
Subject: [PATCH 15/34] Update mdtf_test_cloud.yml

temporary change to {{matrix.action-conda-root}} and see if it works
---
 .github/workflows/mdtf_test_cloud.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/mdtf_test_cloud.yml b/.github/workflows/mdtf_test_cloud.yml
index 0b054a3ae..58ae13ac9 100644
--- a/.github/workflows/mdtf_test_cloud.yml
+++ b/.github/workflows/mdtf_test_cloud.yml
@@ -50,6 +50,7 @@ jobs:
       matrix:
         os: [ubuntu-latest]
         conda-root: [/actions-runner/_work/MDTF-diagnostics/MDTF-diagnostics/3/condabin/conda, /usr/share/miniconda3, /Users/runner/miniconda3]
+        action-conda-root: [/actions-runner/_work/MDTF-diagnostics/MDTF-diagnostics/3/condabin/conda]
         json-file: ["tests/github_actions_test_ubuntu_set1.jsonc","tests/github_actions_test_macos_set1.jsonc"]
         json-file-set2: ["tests/github_actions_test_ubuntu_set2.jsonc", "tests/github_actions_test_macos_set2.jsonc"]
         json-file-set3: ["tests/github_actions_test_ubuntu_set3.jsonc", "tests/github_actions_test_macos_set3.jsonc"]
@@ -101,7 +102,7 @@ jobs:
         mamba env create --force -q -f ./src/conda/_env_synthetic_data.yml
         echo "Installing Conda Environments"
         # MDTF-specific setup: install all conda envs
-        ./src/conda/conda_env_setup.sh --all --conda_root ${{matrix.conda-root}}
+        ./src/conda/conda_env_setup.sh --all --conda_root {{matrix.action-conda-root}} #${{matrix.conda-root}}
     - name: Generate Model Data
       run: |
         cd ../

From 95da998e9acc69046ad06f9cd2b073d375da2639 Mon Sep 17 00:00:00 2001
From: Aparna Radhakrishnan <aparna.radhakrishnan@noaa.gov>
Date: Wed, 15 Sep 2021 18:51:33 -0400
Subject: [PATCH 16/34] Update mdtf_test_cloud.yml

---
 .github/workflows/mdtf_test_cloud.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/mdtf_test_cloud.yml b/.github/workflows/mdtf_test_cloud.yml
index 58ae13ac9..32a3d9495 100644
--- a/.github/workflows/mdtf_test_cloud.yml
+++ b/.github/workflows/mdtf_test_cloud.yml
@@ -49,7 +49,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        conda-root: [/actions-runner/_work/MDTF-diagnostics/MDTF-diagnostics/3/condabin/conda, /usr/share/miniconda3, /Users/runner/miniconda3]
+        conda-root: [/actions-runner/_work/MDTF-diagnostics/MDTF-diagnostics/3/condabin/conda]#, /usr/share/miniconda3, /Users/runner/miniconda3]
         action-conda-root: [/actions-runner/_work/MDTF-diagnostics/MDTF-diagnostics/3/condabin/conda]
         json-file: ["tests/github_actions_test_ubuntu_set1.jsonc","tests/github_actions_test_macos_set1.jsonc"]
         json-file-set2: ["tests/github_actions_test_ubuntu_set2.jsonc", "tests/github_actions_test_macos_set2.jsonc"]
@@ -102,7 +102,8 @@ jobs:
         mamba env create --force -q -f ./src/conda/_env_synthetic_data.yml
         echo "Installing Conda Environments"
         # MDTF-specific setup: install all conda envs
-        ./src/conda/conda_env_setup.sh --all --conda_root {{matrix.action-conda-root}} #${{matrix.conda-root}}
+        ##${{matrix.conda-root}}
+        ./src/conda/conda_env_setup.sh --all --conda_root {{matrix.action-conda-root}} 
     - name: Generate Model Data
       run: |
         cd ../

From 7bdda165e8f442302713ebe711ac4c812045d387 Mon Sep 17 00:00:00 2001
From: Aparna Radhakrishnan <aparna.radhakrishnan@noaa.gov>
Date: Thu, 16 Sep 2021 09:20:34 -0400
Subject: [PATCH 17/34] Update mdtf_test_cloud.yml

---
 .github/workflows/mdtf_test_cloud.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/mdtf_test_cloud.yml b/.github/workflows/mdtf_test_cloud.yml
index 32a3d9495..62898b753 100644
--- a/.github/workflows/mdtf_test_cloud.yml
+++ b/.github/workflows/mdtf_test_cloud.yml
@@ -103,7 +103,7 @@ jobs:
         echo "Installing Conda Environments"
         # MDTF-specific setup: install all conda envs
         ##${{matrix.conda-root}}
-        ./src/conda/conda_env_setup.sh --all --conda_root {{matrix.action-conda-root}} 
+        ./src/conda/conda_env_setup.sh --all --conda_root ${{matrix.action-conda-root}} 
     - name: Generate Model Data
       run: |
         cd ../

From cb3283119dcbe97025357475d7adca037b20d5ce Mon Sep 17 00:00:00 2001
From: Aparna Radhakrishnan <aparna.radhakrishnan@noaa.gov>
Date: Thu, 16 Sep 2021 09:30:44 -0400
Subject: [PATCH 18/34] Update mdtf_test_cloud.yml

---
 .github/workflows/mdtf_test_cloud.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/mdtf_test_cloud.yml b/.github/workflows/mdtf_test_cloud.yml
index 62898b753..a75fb351e 100644
--- a/.github/workflows/mdtf_test_cloud.yml
+++ b/.github/workflows/mdtf_test_cloud.yml
@@ -49,7 +49,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        conda-root: [/actions-runner/_work/MDTF-diagnostics/MDTF-diagnostics/3/condabin/conda]#, /usr/share/miniconda3, /Users/runner/miniconda3]
+        conda-root: [/actions-runner/_work/MDTF-diagnostics/MDTF-diagnostics/3]#, /usr/share/miniconda3, /Users/runner/miniconda3]
         action-conda-root: [/actions-runner/_work/MDTF-diagnostics/MDTF-diagnostics/3/condabin/conda]
         json-file: ["tests/github_actions_test_ubuntu_set1.jsonc","tests/github_actions_test_macos_set1.jsonc"]
         json-file-set2: ["tests/github_actions_test_ubuntu_set2.jsonc", "tests/github_actions_test_macos_set2.jsonc"]

From de8140563da8e504b2a39a6ab441ce6adf198218 Mon Sep 17 00:00:00 2001
From: Aparna Radhakrishnan <aparna.radhakrishnan@noaa.gov>
Date: Thu, 16 Sep 2021 09:46:22 -0400
Subject: [PATCH 19/34] Update mdtf_test_cloud.yml

---
 .github/workflows/mdtf_test_cloud.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/mdtf_test_cloud.yml b/.github/workflows/mdtf_test_cloud.yml
index a75fb351e..a2a7f9c29 100644
--- a/.github/workflows/mdtf_test_cloud.yml
+++ b/.github/workflows/mdtf_test_cloud.yml
@@ -50,7 +50,7 @@ jobs:
       matrix:
         os: [ubuntu-latest]
         conda-root: [/actions-runner/_work/MDTF-diagnostics/MDTF-diagnostics/3]#, /usr/share/miniconda3, /Users/runner/miniconda3]
-        action-conda-root: [/actions-runner/_work/MDTF-diagnostics/MDTF-diagnostics/3/condabin/conda]
+        action-conda-root: [/actions-runner/_work/MDTF-diagnostics/MDTF-diagnostics/3]
         json-file: ["tests/github_actions_test_ubuntu_set1.jsonc","tests/github_actions_test_macos_set1.jsonc"]
         json-file-set2: ["tests/github_actions_test_ubuntu_set2.jsonc", "tests/github_actions_test_macos_set2.jsonc"]
         json-file-set3: ["tests/github_actions_test_ubuntu_set3.jsonc", "tests/github_actions_test_macos_set3.jsonc"]

From 07618e53c726a53c0590e1be01e5f027edcccdad Mon Sep 17 00:00:00 2001
From: Aparna Radhakrishnan <aparna.radhakrishnan@noaa.gov>
Date: Thu, 16 Sep 2021 13:12:17 -0400
Subject: [PATCH 20/34] Update mdtf_test_cloud.yml

---
 .github/workflows/mdtf_test_cloud.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/mdtf_test_cloud.yml b/.github/workflows/mdtf_test_cloud.yml
index a2a7f9c29..41da1e7e6 100644
--- a/.github/workflows/mdtf_test_cloud.yml
+++ b/.github/workflows/mdtf_test_cloud.yml
@@ -36,7 +36,7 @@ jobs:
           mode: start
           github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
           ec2-image-id: ami-092a9e946def5be32
-          ec2-instance-type: t2.micro
+          ec2-instance-type: t2.medium
           subnet-id: subnet-003faaf9f1e386979
           security-group-id: sg-016e543200ed56a0f
           #iam-role-name: github-actions-role-a # optional, requires additional permissions

From f06106e05e4517c9c41fb2368387014cdb4446ba Mon Sep 17 00:00:00 2001
From: Aparna Radhakrishnan <aparna.radhakrishnan@noaa.gov>
Date: Thu, 16 Sep 2021 13:45:57 -0400
Subject: [PATCH 21/34] Update mdtf_test_cloud.yml

---
 .github/workflows/mdtf_test_cloud.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/mdtf_test_cloud.yml b/.github/workflows/mdtf_test_cloud.yml
index 41da1e7e6..ad7a81f63 100644
--- a/.github/workflows/mdtf_test_cloud.yml
+++ b/.github/workflows/mdtf_test_cloud.yml
@@ -36,7 +36,8 @@ jobs:
           mode: start
           github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
           ec2-image-id: ami-092a9e946def5be32
-          ec2-instance-type: t2.medium
+          ec2-instance-type: t2.xlarge
+          ec2-size: 20GB
           subnet-id: subnet-003faaf9f1e386979
           security-group-id: sg-016e543200ed56a0f
           #iam-role-name: github-actions-role-a # optional, requires additional permissions

From 7526d76dce3169dfe462159d827472c62f053f0d Mon Sep 17 00:00:00 2001
From: Aparna Radhakrishnan <aparna.radhakrishnan@noaa.gov>
Date: Thu, 16 Sep 2021 14:04:35 -0400
Subject: [PATCH 22/34] Update mdtf_test_cloud.yml

---
 .github/workflows/mdtf_test_cloud.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/mdtf_test_cloud.yml b/.github/workflows/mdtf_test_cloud.yml
index ad7a81f63..0c7411457 100644
--- a/.github/workflows/mdtf_test_cloud.yml
+++ b/.github/workflows/mdtf_test_cloud.yml
@@ -35,9 +35,8 @@ jobs:
         with:
           mode: start
           github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
-          ec2-image-id: ami-092a9e946def5be32
+          ec2-image-id: ami-0d9c66960c53029ef
           ec2-instance-type: t2.xlarge
-          ec2-size: 20GB
           subnet-id: subnet-003faaf9f1e386979
           security-group-id: sg-016e543200ed56a0f
           #iam-role-name: github-actions-role-a # optional, requires additional permissions

From f30c295e9b59e399c4185348cc13c6202de8a1e9 Mon Sep 17 00:00:00 2001
From: Aparna Radhakrishnan <aparna.radhakrishnan@noaa.gov>
Date: Thu, 16 Sep 2021 21:22:46 -0400
Subject: [PATCH 23/34] Update mdtf_test_cloud.yml

---
 .github/workflows/mdtf_test_cloud.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/mdtf_test_cloud.yml b/.github/workflows/mdtf_test_cloud.yml
index 0c7411457..107d164b0 100644
--- a/.github/workflows/mdtf_test_cloud.yml
+++ b/.github/workflows/mdtf_test_cloud.yml
@@ -37,7 +37,7 @@ jobs:
           github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
           ec2-image-id: ami-0d9c66960c53029ef
           ec2-instance-type: t2.xlarge
-          subnet-id: subnet-003faaf9f1e386979
+          subnet-id: subnet-00d803f7c597997c8 #######subnet-003faaf9f1e386979
           security-group-id: sg-016e543200ed56a0f
           #iam-role-name: github-actions-role-a # optional, requires additional permissions
           

From 7b277bd60d86c0b32ea9bd1081faa63b0e2dab7b Mon Sep 17 00:00:00 2001
From: Aparna Radhakrishnan <aparna.radhakrishnan@noaa.gov>
Date: Thu, 16 Sep 2021 21:40:20 -0400
Subject: [PATCH 24/34] Update actions-test1.yml

---
 .github/workflows/actions-test1.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/actions-test1.yml b/.github/workflows/actions-test1.yml
index e0d331df2..9a971c038 100644
--- a/.github/workflows/actions-test1.yml
+++ b/.github/workflows/actions-test1.yml
@@ -32,7 +32,8 @@ jobs:
     runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner
     steps:
       - name: Hello World
-        run: echo 'Hello MDTF!'
+        run:  curl --verbose --ipv4 --connect-timeout 8 --max-time 1200 --retry 128 --ftp-ssl --disable-epsv --ftp-pasv -u "anonymous:anonymous" ftp://ftp.gfdl.noaa.gov/perm/oar.gfdl.mdtf/convective_transition_diag_obs_data.tar --output convective_transition_diag_obs_data.tar
+
   stop-runner:
     name: Stop self-hosted EC2 runner
     needs:

From daca08a89a36745cf91b7a1c49fc454179f183b8 Mon Sep 17 00:00:00 2001
From: Aparna Radhakrishnan <aparna.radhakrishnan@noaa.gov>
Date: Thu, 16 Sep 2021 21:43:28 -0400
Subject: [PATCH 25/34] Create curltest.yml

---
 .github/workflows/curltest.yml | 57 ++++++++++++++++++++++++++++++++++
 1 file changed, 57 insertions(+)
 create mode 100644 .github/workflows/curltest.yml

diff --git a/.github/workflows/curltest.yml b/.github/workflows/curltest.yml
new file mode 100644
index 000000000..9a971c038
--- /dev/null
+++ b/.github/workflows/curltest.yml
@@ -0,0 +1,57 @@
+name: do-the-job
+on: pull_request
+jobs:
+  start-runner:
+    name: Start self-hosted EC2 runner
+    runs-on: ubuntu-latest
+    outputs:
+      label: ${{ steps.start-ec2-runner.outputs.label }}
+      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION }}
+      - name: Start EC2 runner
+        id: start-ec2-runner
+        uses: machulav/ec2-github-runner@v2
+        with:
+          mode: start
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          ec2-image-id: ami-092a9e946def5be32
+          ec2-instance-type: t2.micro
+          subnet-id: subnet-003faaf9f1e386979
+          security-group-id: sg-016e543200ed56a0f
+          #iam-role-name: github-actions-role-a # optional, requires additional permissions
+          
+  do-the-job:
+    name: Do the job on the runner
+    needs: start-runner # required to start the main job when the runner is ready
+    runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner
+    steps:
+      - name: Hello World
+        run:  curl --verbose --ipv4 --connect-timeout 8 --max-time 1200 --retry 128 --ftp-ssl --disable-epsv --ftp-pasv -u "anonymous:anonymous" ftp://ftp.gfdl.noaa.gov/perm/oar.gfdl.mdtf/convective_transition_diag_obs_data.tar --output convective_transition_diag_obs_data.tar
+
+  stop-runner:
+    name: Stop self-hosted EC2 runner
+    needs:
+      - start-runner # required to get output from the start-runner job
+      - do-the-job # required to wait when the main job is done
+    runs-on: ubuntu-latest
+    if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION }}
+      - name: Stop EC2 runner
+        uses: machulav/ec2-github-runner@v2
+        with:
+          mode: stop
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          label: ${{ needs.start-runner.outputs.label }}
+          ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }}

From 1e143b086ea7103d889ea457479e34c3fee50eba Mon Sep 17 00:00:00 2001
From: Aparna Radhakrishnan <aparna.radhakrishnan@noaa.gov>
Date: Thu, 16 Sep 2021 21:45:09 -0400
Subject: [PATCH 26/34] Update and rename actions-test1.yml to curltest.yml

---
 .github/workflows/{actions-test1.yml => curltest.yml} | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 rename .github/workflows/{actions-test1.yml => curltest.yml} (99%)

diff --git a/.github/workflows/actions-test1.yml b/.github/workflows/curltest.yml
similarity index 99%
rename from .github/workflows/actions-test1.yml
rename to .github/workflows/curltest.yml
index 9a971c038..2e11d6b43 100644
--- a/.github/workflows/actions-test1.yml
+++ b/.github/workflows/curltest.yml
@@ -1,4 +1,4 @@
-name: do-the-job
+name: do-the-curl
 on: pull_request
 jobs:
   start-runner:

From 0ffd75ecddca647f0c3172abc0d9481570f15de3 Mon Sep 17 00:00:00 2001
From: Aparna Radhakrishnan <aparna.radhakrishnan@noaa.gov>
Date: Thu, 16 Sep 2021 21:46:21 -0400
Subject: [PATCH 27/34] Create curl.yml

---
 .github/workflows/curl.yml | 57 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 57 insertions(+)
 create mode 100644 .github/workflows/curl.yml

diff --git a/.github/workflows/curl.yml b/.github/workflows/curl.yml
new file mode 100644
index 000000000..398d6c4ce
--- /dev/null
+++ b/.github/workflows/curl.yml
@@ -0,0 +1,57 @@
+name: curl-test
+on: pull_request
+jobs:
+  start-runner:
+    name: Start self-hosted EC2 runner
+    runs-on: ubuntu-latest
+    outputs:
+      label: ${{ steps.start-ec2-runner.outputs.label }}
+      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION }}
+      - name: Start EC2 runner
+        id: start-ec2-runner
+        uses: machulav/ec2-github-runner@v2
+        with:
+          mode: start
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          ec2-image-id: ami-092a9e946def5be32
+          ec2-instance-type: t2.micro
+          subnet-id: subnet-003faaf9f1e386979
+          security-group-id: sg-016e543200ed56a0f
+          #iam-role-name: github-actions-role-a # optional, requires additional permissions
+          
+  do-the-job:
+    name: Do the job on the runner
+    needs: start-runner # required to start the main job when the runner is ready
+    runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner
+    steps:
+      - name: Hello World
+        run:  curl --verbose --ipv4 --connect-timeout 8 --max-time 1200 --retry 128 --ftp-ssl --disable-epsv --ftp-pasv -u "anonymous:anonymous" ftp://ftp.gfdl.noaa.gov/perm/oar.gfdl.mdtf/convective_transition_diag_obs_data.tar --output convective_transition_diag_obs_data.tar
+
+  stop-runner:
+    name: Stop self-hosted EC2 runner
+    needs:
+      - start-runner # required to get output from the start-runner job
+      - do-the-job # required to wait when the main job is done
+    runs-on: ubuntu-latest
+    if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION }}
+      - name: Stop EC2 runner
+        uses: machulav/ec2-github-runner@v2
+        with:
+          mode: stop
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          label: ${{ needs.start-runner.outputs.label }}
+          ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }}

From 7a59a1b41eb68ad498f7013a77b021541dec87d9 Mon Sep 17 00:00:00 2001
From: Aparna Radhakrishnan <aparna.radhakrishnan@noaa.gov>
Date: Thu, 16 Sep 2021 21:48:08 -0400
Subject: [PATCH 28/34] Update curl.yml

---
 .github/workflows/curl.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/curl.yml b/.github/workflows/curl.yml
index 398d6c4ce..439cd5325 100644
--- a/.github/workflows/curl.yml
+++ b/.github/workflows/curl.yml
@@ -1,5 +1,5 @@
 name: curl-test
-on: pull_request
+on: commit
 jobs:
   start-runner:
     name: Start self-hosted EC2 runner

From 50d607124925fc746e75b49993aeda61f78ff02b Mon Sep 17 00:00:00 2001
From: Aparna Radhakrishnan <aparna.radhakrishnan@noaa.gov>
Date: Thu, 16 Sep 2021 21:49:10 -0400
Subject: [PATCH 29/34] Update curl.yml

---
 .github/workflows/curl.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/curl.yml b/.github/workflows/curl.yml
index 439cd5325..b598bb7d7 100644
--- a/.github/workflows/curl.yml
+++ b/.github/workflows/curl.yml
@@ -1,5 +1,5 @@
 name: curl-test
-on: commit
+on: push
 jobs:
   start-runner:
     name: Start self-hosted EC2 runner

From afc6849f66d94ae9d41e73a4bb96dcb94d9b36ad Mon Sep 17 00:00:00 2001
From: Aparna Radhakrishnan <aparna.radhakrishnan@noaa.gov>
Date: Fri, 17 Sep 2021 16:25:43 -0400
Subject: [PATCH 30/34] Create github_actions_test_cloud_set1.jsonc

conda root for cloud actions runner
---
 tests/github_actions_test_cloud_set1.jsonc | 94 ++++++++++++++++++++++
 1 file changed, 94 insertions(+)
 create mode 100644 tests/github_actions_test_cloud_set1.jsonc

diff --git a/tests/github_actions_test_cloud_set1.jsonc b/tests/github_actions_test_cloud_set1.jsonc
new file mode 100644
index 000000000..38d2b8dc8
--- /dev/null
+++ b/tests/github_actions_test_cloud_set1.jsonc
@@ -0,0 +1,94 @@
+// Configuration for MDTF-diagnostics driver script self-test.
+// All text to the right of an unquoted "//" is a comment and ignored, as well
+// as blank lines (JSONC quasi-standard.)
+{
+  "case_list" : [
+    // The cases below correspond to the different sample model data sets. Note
+    // that the MDTF package does not currently support analyzing multiple
+    // models in a single invocation. Comment out or delete the first entry and
+    // uncomment the second to run NOAA-GFDL-AM4 only for the MJO_prop_amp POD,
+    // and likewise for the SM_ET_coupling POD.
+    {
+      "CASENAME" : "NCAR.Synthetic",
+      "convention" : "CESM",
+      "FIRSTYR" : 1975,
+      "LASTYR" : 1981,
+      "pod_list": [
+          // Optional: PODs to run for this model only (defaults to all)
+        "convective_transition_diag",
+        "Wheeler_Kiladis",
+        "MJO_suite",
+        "MJO_teleconnection",
+        "precip_diurnal_cycle",
+        "EOF_500hPa"
+      ]
+    }
+  ],
+  // PATHS ---------------------------------------------------------------------
+  // Location of supporting data downloaded when the framework was installed.
+
+  // If a relative path is given, it's resolved relative to the MDTF-diagnostics
+  // code directory. Environment variables (eg, $HOME) can be referenced with a
+  // "$" and will be expended to their current values when the framework runs.
+
+  // Parent directory containing observational data used by individual PODs.
+  "OBS_DATA_ROOT": "../inputdata/obs_data",
+
+  // Parent directory containing results from different models.
+  "MODEL_DATA_ROOT": "../mdtf_test_data",
+
+  // Working directory. Defaults to working directory if blank.
+  "WORKING_DIR": "../wkdir",
+
+  // Directory to write output. The results of each run of the framework will be
+  // put in a subdirectory of this directory.
+  "OUTPUT_DIR": "../wkdir",
+
+  // Location of the Anaconda/miniconda installation to use for managing
+  // dependencies (path returned lsby running `conda info --base`.) If empty,
+  // framework will attempt to determine location of system's conda installation.
+  "conda_root": "actions-runner/_work/MDTF-diagnostics/MDTF-diagnostics/3",
+
+  // Directory containing the framework-specific conda environments. This should
+  // be equal to the "--env_dir" flag passed to conda_env_setup.sh. If left
+  // blank, the framework will look for its environments in the system default
+  // location.
+  "conda_env_root": "/actions-runner/_work/MDTF-diagnostics/MDTF-diagnostics/3/envs/",
+
+  // SETTINGS ------------------------------------------------------------------
+  // Any command-line option recognized by the mdtf script (type `mdtf --help`)
+  // can be set here, in the form "flag name": "desired setting".
+
+  // Method used to fetch model data.
+  "data_manager": "Local_File",
+
+  // Method used to manage dependencies.
+  "environment_manager": "Conda",
+
+  // Settings affecting what output is generated:
+
+  // Set to true to have PODs save postscript figures in addition to bitmaps.
+  "save_ps": false,
+
+  // Set to true to have PODs save netCDF files of processed data.
+  "save_nc": false,
+
+  // Set to true to save HTML and bitmap plots in a .tar file.
+  "make_variab_tar": false,
+
+  // Set to true to overwrite results in OUTPUT_DIR; otherwise results saved
+  // under a unique name.
+  "overwrite": false,
+
+  // Settings used in debugging:
+
+  // Log verbosity level.
+  "verbose": 1,
+
+  // Set to true for framework test. Data is fetched but PODs are not run.
+  "test_mode": false,
+
+  // Set to true for framework test. No external commands are run and no remote
+  // data is copied. Implies test_mode.
+  "dry_run": false
+}

From 57ae092013e798f3f3b9043b02eb9e7f6b6ec533 Mon Sep 17 00:00:00 2001
From: Aparna Radhakrishnan <aparna.radhakrishnan@noaa.gov>
Date: Fri, 17 Sep 2021 16:26:38 -0400
Subject: [PATCH 31/34] Update mdtf_test_cloud.yml

---
 .github/workflows/mdtf_test_cloud.yml | 19 -------------------
 1 file changed, 19 deletions(-)

diff --git a/.github/workflows/mdtf_test_cloud.yml b/.github/workflows/mdtf_test_cloud.yml
index 107d164b0..21b06d081 100644
--- a/.github/workflows/mdtf_test_cloud.yml
+++ b/.github/workflows/mdtf_test_cloud.yml
@@ -152,25 +152,6 @@ jobs:
         ./mdtf --version
         # run the test PODs
         ./mdtf -v -f ${{matrix.json-file}}
-    - name: Get observational data for set 2
-      run: |
-        echo "${PWD}"
-        # remove data from previous run
-        # Actions moves you to the root repo directory in every step, so need to cd again
-        cd ../inputdata/obs_data
-        echo "deleting obs data from set 1"
-        rm -rf *
-        cd ../../
-        echo "Available Space"
-        df -h
-        # attempt FTP data fetch
-        # allow 20 min for transfer before timeout; Github actions allows 6 hours for individual
-        # jobs, but we don't want to max out resources that are shared by the NOAA-GFDL repos.
-        curl --verbose --ipv4 --connect-timeout 8 --max-time 1200 --retry 128 --ftp-ssl --disable-epsv --ftp-pasv -u "anonymous:anonymous" ftp://ftp.gfdl.noaa.gov/perm/oar.gfdl.mdtf/MJO_prop_amp_obs_data.tar --output MJO_prop_amp_obs_data.tar
-        echo "Untarring set 2 test files"
-        tar -xvf MJO_prop_amp_obs_data.tar
-        # clean up tarballs
-        rm -f *.tar
   stop-runner:
     name: Stop self-hosted EC2 runner
     needs:

From bdefd70faabf888e3cc3f3ba1daa151b87a4cca4 Mon Sep 17 00:00:00 2001
From: Aparna Radhakrishnan <aparna.radhakrishnan@noaa.gov>
Date: Fri, 17 Sep 2021 17:05:50 -0400
Subject: [PATCH 32/34] Update mdtf_test_cloud.yml

---
 .github/workflows/mdtf_test_cloud.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/mdtf_test_cloud.yml b/.github/workflows/mdtf_test_cloud.yml
index 21b06d081..6dfddef86 100644
--- a/.github/workflows/mdtf_test_cloud.yml
+++ b/.github/workflows/mdtf_test_cloud.yml
@@ -51,7 +51,7 @@ jobs:
         os: [ubuntu-latest]
         conda-root: [/actions-runner/_work/MDTF-diagnostics/MDTF-diagnostics/3]#, /usr/share/miniconda3, /Users/runner/miniconda3]
         action-conda-root: [/actions-runner/_work/MDTF-diagnostics/MDTF-diagnostics/3]
-        json-file: ["tests/github_actions_test_ubuntu_set1.jsonc","tests/github_actions_test_macos_set1.jsonc"]
+        json-file: ["tests/github_actions_test_cloud_set1.jsonc","tests/github_actions_test_macos_set1.jsonc"]
         json-file-set2: ["tests/github_actions_test_ubuntu_set2.jsonc", "tests/github_actions_test_macos_set2.jsonc"]
         json-file-set3: ["tests/github_actions_test_ubuntu_set3.jsonc", "tests/github_actions_test_macos_set3.jsonc"]
         # if experimental is true, other jobs to run if one fails

From 1b92c209a1a1395ca0eda292c74e6474311c073f Mon Sep 17 00:00:00 2001
From: Aparna Radhakrishnan <aparna.radhakrishnan@noaa.gov>
Date: Fri, 17 Sep 2021 17:21:09 -0400
Subject: [PATCH 33/34] Update github_actions_test_cloud_set1.jsonc

---
 tests/github_actions_test_cloud_set1.jsonc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/github_actions_test_cloud_set1.jsonc b/tests/github_actions_test_cloud_set1.jsonc
index 38d2b8dc8..8b590fdf0 100644
--- a/tests/github_actions_test_cloud_set1.jsonc
+++ b/tests/github_actions_test_cloud_set1.jsonc
@@ -47,7 +47,7 @@
   // Location of the Anaconda/miniconda installation to use for managing
   // dependencies (path returned lsby running `conda info --base`.) If empty,
   // framework will attempt to determine location of system's conda installation.
-  "conda_root": "actions-runner/_work/MDTF-diagnostics/MDTF-diagnostics/3",
+  "conda_root": "/actions-runner/_work/MDTF-diagnostics/MDTF-diagnostics/3",
 
   // Directory containing the framework-specific conda environments. This should
   // be equal to the "--env_dir" flag passed to conda_env_setup.sh. If left

From 1b35449b51cfc2fd21ace19709e233010b638876 Mon Sep 17 00:00:00 2001
From: Aparna Radhakrishnan <aparna.radhakrishnan@noaa.gov>
Date: Tue, 23 Nov 2021 08:48:52 -0500
Subject: [PATCH 34/34] Update curltest.yml

---
 .github/workflows/curltest.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/curltest.yml b/.github/workflows/curltest.yml
index 1406752a5..de200393f 100644
--- a/.github/workflows/curltest.yml
+++ b/.github/workflows/curltest.yml
@@ -38,7 +38,7 @@ jobs:
     name: Stop self-hosted EC2 runner
     needs:
       - start-runner # required to get output from the start-runner job
-      - do-the-job # required to wait when the main job is done
+      - do-the-curl # required to wait when the main job is done
     runs-on: ubuntu-latest
     if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
     steps: