diff --git a/.github/workflows/curl.yml b/.github/workflows/curl.yml new file mode 100644 index 000000000..b598bb7d7 --- /dev/null +++ b/.github/workflows/curl.yml @@ -0,0 +1,57 @@ +name: curl-test +on: push +jobs: + start-runner: + name: Start self-hosted EC2 runner + runs-on: ubuntu-latest + outputs: + label: ${{ steps.start-ec2-runner.outputs.label }} + ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_REGION }} + - name: Start EC2 runner + id: start-ec2-runner + uses: machulav/ec2-github-runner@v2 + with: + mode: start + github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + ec2-image-id: ami-092a9e946def5be32 + ec2-instance-type: t2.micro + subnet-id: subnet-003faaf9f1e386979 + security-group-id: sg-016e543200ed56a0f + #iam-role-name: github-actions-role-a # optional, requires additional permissions + + do-the-job: + name: Do the job on the runner + needs: start-runner # required to start the main job when the runner is ready + runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner + steps: + - name: Hello World + run: curl --verbose --ipv4 --connect-timeout 8 --max-time 1200 --retry 128 --ftp-ssl --disable-epsv --ftp-pasv -u "anonymous:anonymous" ftp://ftp.gfdl.noaa.gov/perm/oar.gfdl.mdtf/convective_transition_diag_obs_data.tar --output convective_transition_diag_obs_data.tar + + stop-runner: + name: Stop self-hosted EC2 runner + needs: + - start-runner # required to get output from the start-runner job + - do-the-job # required to wait when the main job is done + runs-on: ubuntu-latest + if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_REGION }} + - name: Stop EC2 runner + uses: machulav/ec2-github-runner@v2 + with: + mode: stop + github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + label: ${{ needs.start-runner.outputs.label }} + ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }} diff --git a/.github/workflows/curltest.yml b/.github/workflows/curltest.yml new file mode 100644 index 000000000..de200393f --- /dev/null +++ b/.github/workflows/curltest.yml @@ -0,0 +1,57 @@ +name: do-the-curl +on: pull_request +jobs: + start-runner: + name: Start self-hosted EC2 runner + runs-on: ubuntu-latest + outputs: + label: ${{ steps.start-ec2-runner.outputs.label }} + ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_REGION }} + - name: Start EC2 runner + id: start-ec2-runner + uses: machulav/ec2-github-runner@v2 + with: + mode: start + github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + ec2-image-id: ami-092a9e946def5be32 + ec2-instance-type: t2.micro + subnet-id: subnet-003faaf9f1e386979 + security-group-id: sg-016e543200ed56a0f + #iam-role-name: github-actions-role-a # optional, requires additional permissions + + do-the-curl: + name: Do the job on the runner + needs: start-runner # required to start the main job when the runner is ready + runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner + steps: + - name: Hello World + run: curl --verbose --ipv4 --connect-timeout 8 --max-time 1200 --retry 128 --ftp-ssl --disable-epsv --ftp-pasv -u "anonymous:anonymous" ftp://ftp.gfdl.noaa.gov/perm/oar.gfdl.mdtf/convective_transition_diag_obs_data.tar --output convective_transition_diag_obs_data.tar + + stop-runner: + name: Stop self-hosted EC2 runner + needs: + - start-runner # required to get output from the start-runner job + - do-the-curl # required to wait when the main job is done + runs-on: ubuntu-latest + if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_REGION }} + - name: Stop EC2 runner + uses: machulav/ec2-github-runner@v2 + with: + mode: stop + github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + label: ${{ needs.start-runner.outputs.label }} + ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }} diff --git a/.github/workflows/mdtf_test_cloud.yml b/.github/workflows/mdtf_test_cloud.yml new file mode 100644 index 000000000..6dfddef86 --- /dev/null +++ b/.github/workflows/mdtf_test_cloud.yml @@ -0,0 +1,175 @@ +# This file builds and runs a lightweight version of the MDTF test suite. +# Note that the tests assess functionality of the diagnostics, +# and do not evaluate data or scientific content. +name: MDTF_test_cloud + +on: + push: + branches: + - main + - develop + pull_request: + branches: + - main + - develop +defaults: + run: + shell: bash -l {0} +jobs: + start-runner: + name: Start self-hosted EC2 runner + runs-on: ubuntu-latest + outputs: + label: ${{ steps.start-ec2-runner.outputs.label }} + ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_REGION }} + - name: Start EC2 runner + id: start-ec2-runner + uses: machulav/ec2-github-runner@v2 + with: + mode: start + github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + ec2-image-id: ami-0d9c66960c53029ef + ec2-instance-type: t2.xlarge + subnet-id: subnet-00d803f7c597997c8 #######subnet-003faaf9f1e386979 + security-group-id: sg-016e543200ed56a0f + #iam-role-name: github-actions-role-a # optional, requires additional permissions + + build: + name: MDTF core tests + needs: start-runner + runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner + # continue-on-error: ${{ matrix.experimental }} + strategy: + matrix: + os: [ubuntu-latest] + conda-root: [/actions-runner/_work/MDTF-diagnostics/MDTF-diagnostics/3]#, /usr/share/miniconda3, /Users/runner/miniconda3] + action-conda-root: [/actions-runner/_work/MDTF-diagnostics/MDTF-diagnostics/3] + json-file: ["tests/github_actions_test_cloud_set1.jsonc","tests/github_actions_test_macos_set1.jsonc"] + json-file-set2: ["tests/github_actions_test_ubuntu_set2.jsonc", "tests/github_actions_test_macos_set2.jsonc"] + json-file-set3: ["tests/github_actions_test_ubuntu_set3.jsonc", "tests/github_actions_test_macos_set3.jsonc"] + # if experimental is true, other jobs to run if one fails + experimental: [false] + exclude: + - os: ubuntu-latest + conda-root: /Users/runner/miniconda3 + - os: ubuntu-latest + json-file: "tests/github_actions_test_macos_set1.jsonc" + - os: ubuntu-latest + json-file-set2: "tests/github_actions_test_macos_set2.jsonc" + - os: ubuntu-latest + json-file-set3: "tests/github_actions_test_macos_set3.jsonc" + - conda-root: /usr/share/miniconda3 + json-file: "tests/github_actions_test_macos_set1.jsonc" + - conda-root: /Users/runner/minconda3 + json-file: "tests/github_actions_test_ubuntu_set1.jsonc" + - conda-root: /usr/share/miniconda3 + json-file-set2: "tests/github_actions_test_macos_set2.jsonc" + - conda-root: /Users/runner/minconda3 + json-file-set2: "tests/github_actions_test_ubuntu_set2.jsonc" + - conda-root: /usr/share/miniconda3 + json-file-set3: "tests/github_actions_test_macos_set3.jsonc" + - conda-root: /Users/runner/minconda3 + json-file-set3: "tests/github_actions_test_ubuntu_set3.jsonc" + max-parallel: 2 + steps: + - uses: actions/checkout@v2 + - name: Download Miniconda 3 + uses: conda-incubator/setup-miniconda@v2 + with: + miniconda-version: "latest" + python-version: 3.8 + - name: Verify miniconda + run: | + conda info -a + conda config --show + - name: Set environment variables + run: | + echo "POD_OUTPUT=$(echo $PWD/../wkdir/GFDL.Synthetic)" >> $GITHUB_ENV + - name: Install Conda Environments + run: | + # install mamba (https://github.com/mamba-org/mamba) for faster dependency resolution + echo "Installing Mamba" + conda install mamba -n base -c conda-forge + # create the synthetic data environment + echo "Creating the _MDTF_synthetic_data environment" + mamba env create --force -q -f ./src/conda/_env_synthetic_data.yml + echo "Installing Conda Environments" + # MDTF-specific setup: install all conda envs + ##${{matrix.conda-root}} + ./src/conda/conda_env_setup.sh --all --conda_root ${{matrix.action-conda-root}} + - name: Generate Model Data + run: | + cd ../ + conda activate _MDTF_synthetic_data + pip install mdtf-test-data + mkdir mdtf_test_data ; cd mdtf_test_data + # generate the data and run unit tests + mdtf_synthetic.py -c GFDL --startyear 1 --nyears 10 --unittest + mdtf_synthetic.py -c NCAR --startyear 1975 --nyears 7 + mdtf_synthetic.py -c CMIP --startyear 1 --nyears 10 + cd ../ + mkdir wkdir + - name: Get Observational Data for Set 1 + run: | + echo "${PWD}" + cd ../ + echo "Available Space" + df -h + # attempt FTP data fetch + # allow 20 min for transfer before timeout; Github actions allows 6 hours for individual + # jobs, but we don't want to max out resources that are shared by the NOAA-GFDL repos. + curl --verbose --ipv4 --connect-timeout 8 --max-time 1200 --retry 128 --ftp-ssl --disable-epsv --ftp-pasv -u "anonymous:anonymous" ftp://ftp.gfdl.noaa.gov/perm/oar.gfdl.mdtf/convective_transition_diag_obs_data.tar --output convective_transition_diag_obs_data.tar + curl --verbose --ipv4 --connect-timeout 8 --max-time 1200 --retry 128 --ftp-ssl --disable-epsv --ftp-pasv -u "anonymous:anonymous" ftp://ftp.gfdl.noaa.gov/perm/oar.gfdl.mdtf/EOF_500hPa_obs_data.tar --output EOF_500hPa_obs_data.tar + curl --verbose --ipv4 --connect-timeout 8 --max-time 1200 --retry 128 --ftp-ssl --disable-epsv --ftp-pasv -u "anonymous:anonymous" ftp://ftp.gfdl.noaa.gov/perm/oar.gfdl.mdtf/Wheeler_Kiladis_obs_data.tar --output Wheeler_Kiladis_obs_data.tar + curl --verbose --ipv4 --connect-timeout 8 --max-time 1200 --retry 128 --ftp-ssl --disable-epsv --ftp-pasv -u "anonymous:anonymous" ftp://ftp.gfdl.noaa.gov/perm/oar.gfdl.mdtf/MJO_teleconnection_obs_data.tar --output MJO_teleconnection_obs_data.tar + curl --verbose --ipv4 --connect-timeout 8 --max-time 1200 --retry 128 --ftp-ssl --disable-epsv --ftp-pasv -u "anonymous:anonymous" ftp://ftp.gfdl.noaa.gov/perm/oar.gfdl.mdtf/MJO_suite_obs_data.tar --output MJO_suite_obs_data.tar + curl --verbose --ipv4 --connect-timeout 8 --max-time 1200 --retry 128 --ftp-ssl --disable-epsv --ftp-pasv -u "anonymous:anonymous" ftp://ftp.gfdl.noaa.gov/perm/oar.gfdl.mdtf/precip_diurnal_cycle_obs_data.tar --output precip_diurnal_cycle_obs_data.tar + ## make input data directories + mkdir -p inputdata/obs_data + echo "Untarring test files" + tar -xvf convective_transition_diag_obs_data.tar + tar -xvf EOF_500hPa_obs_data.tar + tar -xvf precip_diurnal_cycle_obs_data.tar + tar -xvf MJO_teleconnection_obs_data.tar + tar -xvf MJO_suite_obs_data.tar + tar -xvf Wheeler_Kiladis_obs_data.tar + # clean up tarballs + rm -f *.tar + - name: Run diagnostic tests set 1 + run: | + echo "POD_OUTPUT is: " + echo "${POD_OUTPUT}" + conda activate _MDTF_base + cat ./mdtf + # trivial check that install script worked + ./mdtf --version + # run the test PODs + ./mdtf -v -f ${{matrix.json-file}} + stop-runner: + name: Stop self-hosted EC2 runner + needs: + - start-runner # required to get output from the start-runner job + - build # required to wait when the main job is done + runs-on: ubuntu-latest + if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_REGION }} + - name: Stop EC2 runner + uses: machulav/ec2-github-runner@v2 + with: + mode: stop + github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + label: ${{ needs.start-runner.outputs.label }} + ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }} diff --git a/tests/github_actions_test_cloud_set1.jsonc b/tests/github_actions_test_cloud_set1.jsonc new file mode 100644 index 000000000..8b590fdf0 --- /dev/null +++ b/tests/github_actions_test_cloud_set1.jsonc @@ -0,0 +1,94 @@ +// Configuration for MDTF-diagnostics driver script self-test. +// All text to the right of an unquoted "//" is a comment and ignored, as well +// as blank lines (JSONC quasi-standard.) +{ + "case_list" : [ + // The cases below correspond to the different sample model data sets. Note + // that the MDTF package does not currently support analyzing multiple + // models in a single invocation. Comment out or delete the first entry and + // uncomment the second to run NOAA-GFDL-AM4 only for the MJO_prop_amp POD, + // and likewise for the SM_ET_coupling POD. + { + "CASENAME" : "NCAR.Synthetic", + "convention" : "CESM", + "FIRSTYR" : 1975, + "LASTYR" : 1981, + "pod_list": [ + // Optional: PODs to run for this model only (defaults to all) + "convective_transition_diag", + "Wheeler_Kiladis", + "MJO_suite", + "MJO_teleconnection", + "precip_diurnal_cycle", + "EOF_500hPa" + ] + } + ], + // PATHS --------------------------------------------------------------------- + // Location of supporting data downloaded when the framework was installed. + + // If a relative path is given, it's resolved relative to the MDTF-diagnostics + // code directory. Environment variables (eg, $HOME) can be referenced with a + // "$" and will be expended to their current values when the framework runs. + + // Parent directory containing observational data used by individual PODs. + "OBS_DATA_ROOT": "../inputdata/obs_data", + + // Parent directory containing results from different models. + "MODEL_DATA_ROOT": "../mdtf_test_data", + + // Working directory. Defaults to working directory if blank. + "WORKING_DIR": "../wkdir", + + // Directory to write output. The results of each run of the framework will be + // put in a subdirectory of this directory. + "OUTPUT_DIR": "../wkdir", + + // Location of the Anaconda/miniconda installation to use for managing + // dependencies (path returned lsby running `conda info --base`.) If empty, + // framework will attempt to determine location of system's conda installation. + "conda_root": "/actions-runner/_work/MDTF-diagnostics/MDTF-diagnostics/3", + + // Directory containing the framework-specific conda environments. This should + // be equal to the "--env_dir" flag passed to conda_env_setup.sh. If left + // blank, the framework will look for its environments in the system default + // location. + "conda_env_root": "/actions-runner/_work/MDTF-diagnostics/MDTF-diagnostics/3/envs/", + + // SETTINGS ------------------------------------------------------------------ + // Any command-line option recognized by the mdtf script (type `mdtf --help`) + // can be set here, in the form "flag name": "desired setting". + + // Method used to fetch model data. + "data_manager": "Local_File", + + // Method used to manage dependencies. + "environment_manager": "Conda", + + // Settings affecting what output is generated: + + // Set to true to have PODs save postscript figures in addition to bitmaps. + "save_ps": false, + + // Set to true to have PODs save netCDF files of processed data. + "save_nc": false, + + // Set to true to save HTML and bitmap plots in a .tar file. + "make_variab_tar": false, + + // Set to true to overwrite results in OUTPUT_DIR; otherwise results saved + // under a unique name. + "overwrite": false, + + // Settings used in debugging: + + // Log verbosity level. + "verbose": 1, + + // Set to true for framework test. Data is fetched but PODs are not run. + "test_mode": false, + + // Set to true for framework test. No external commands are run and no remote + // data is copied. Implies test_mode. + "dry_run": false +}