diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000..9d0b0d4 Binary files /dev/null and b/.DS_Store differ diff --git a/.github/workflows/github_actions_CD.yml b/.github/workflows/github_actions_CD.yml index a8c077a..0c0aea5 100644 --- a/.github/workflows/github_actions_CD.yml +++ b/.github/workflows/github_actions_CD.yml @@ -8,7 +8,7 @@ #The purpose of this file is to give instructions to GitHub on how to do the #image deployment to Docker Hub. #Author: -#Cedric H. David, 2022-2023. +#Cedric H. David, 2022-2024. #******************************************************************************* @@ -84,6 +84,7 @@ jobs: - name: Build and push uses: docker/build-push-action@v5 with: + context: . push: true platforms: linux/amd64,linux/arm64 tags: | diff --git a/README.md b/README.md index 4292ab5..625d3aa 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,7 @@ The beauty of Docker is that there is **no need to install anymore packages**. RRR is ready to go! To run it, just use: ``` -$ docker run --rm -it chdavid/rrr +$ docker run --rm -e EARTHDATA_USERNAME="yourUsername" -e EARTHDATA_PASSWORD="yourPassword" -it chdavid/rrr ``` ## Testing with Docker diff --git a/drv/.DS_Store b/drv/.DS_Store new file mode 100644 index 0000000..97c2af1 Binary files /dev/null and b/drv/.DS_Store differ diff --git a/drv/rrr_drv_MERIT_Hydro_v07_Basins_v01_GLDAS_v20.py b/drv/rrr_drv_MERIT_Hydro_v07_Basins_v01_GLDAS_v20.py old mode 100755 new mode 100644 index 83b63a8..6c95fd2 --- a/drv/rrr_drv_MERIT_Hydro_v07_Basins_v01_GLDAS_v20.py +++ b/drv/rrr_drv_MERIT_Hydro_v07_Basins_v01_GLDAS_v20.py @@ -1,23 +1,23 @@ #!/usr/bin/env python3 -#******************************************************************************* -#rrr_drv_MERIT_Hydro_v07_Basins_v01_GLDAS_v20.py -#******************************************************************************* - -#Purpose: -#This program is designed to produce input data for RAPID for one entire month, -#and can be run with a simple set of instructions. The focus here ins on runoff -#from GLDAS v2.0 and hydrography from MERIT Hydro v0.1 Basins v0.1 . The details -#of the execution are included in a simple python class: -#rrr = RRR('74', 'VIC', '3H', '2000-01') -#This class combines the pfaf_level_02 code, the Land Surface Model (LSM) name, -#the LSM temporal resolution, and the month in yyyy-mm format. -#Authors: -#Cedric H. David, Kevin Marlis, 2023-2023 - - -#******************************************************************************* -#Import Python modules -#******************************************************************************* +# ***************************************************************************** +# rrr_drv_MERIT_Hydro_v07_Basins_v01_GLDAS_v20.py +# ***************************************************************************** + +# Purpose: +# This program is designed to produce input data for RAPID for one entire month +# and can be run with a simple set of instructions. The focus here is on runoff +# from GLDAS v2.0 and hydrography from MERITHydro v0.1 Basins v0.1. The details +# of the execution are included in a simple python class: +# rrr = RRR('74', 'VIC', '3H', '2000-01') +# This class combines the pfaf_level_02 code, the Land Surface Model (LSM) name +# the LSM temporal resolution, and the month in yyyy-mm format. +# Authors: +# Cedric H. David, Kevin Marlis, Manu Tom 2023-24 + + +# ***************************************************************************** +# Import Python modules +# ***************************************************************************** import os import subprocess import datetime @@ -25,317 +25,338 @@ import glob -#******************************************************************************* -#Define nomenclature inside Python class -#******************************************************************************* +# ***************************************************************************** +# Define nomenclature inside Python class +# ***************************************************************************** class RRR(object): - basn_id: str - lsm_mod: str - lsm_stp: str - yyyy_mm: str - - def __init__(self, basn_id, lsm_mod, lsm_stp, yyyy_mm) -> None: - self.basn_id = basn_id - self.lsm_mod = lsm_mod - self.lsm_stp = lsm_stp - self.yyyy_mm = yyyy_mm - - self.set_paths() - - def set_paths(self): - #Uses “formatted string literals” also known as f-strings. - - #--------------------------------------------------------------------- - #Directories - #--------------------------------------------------------------------- - self.inp_dir = f'/tmp/input/' - os.makedirs(self.inp_dir, exist_ok=True) - self.hyd_dir = f'/tmp/input/MH07B01/' - os.makedirs(self.hyd_dir, exist_ok=True) - self.lsm_dir = f'/tmp/input/GLDAS20/{self.lsm_mod}/{self.yyyy_mm}/' - os.makedirs(self.lsm_dir, exist_ok=True) - - self.out_dir = f'/tmp/output/' - os.makedirs(self.out_dir, exist_ok=True) - - - #--------------------------------------------------------------------- - #Hydrography {basn_id} - #--------------------------------------------------------------------- - self.riv_shp = f'riv_pfaf_{self.basn_id}_MERIT_Hydro_v07_Basins_v01.shp' - self.cat_shp = f'cat_pfaf_{self.basn_id}_MERIT_Hydro_v07_Basins_v01.shp' - - self.kfc_csv = f'kfac_pfaf_{self.basn_id}_1km_hour.csv' - self.xfc_csv = f'xfac_pfaf_{self.basn_id}_0.1.csv' - self.klo_csv = f'k_pfaf_{self.basn_id}_low.csv' - self.xlo_csv = f'x_pfaf_{self.basn_id}_low.csv' - self.knr_csv = f'k_pfaf_{self.basn_id}_nrm.csv' - self.xnr_csv = f'x_pfaf_{self.basn_id}_nrm.csv' - self.khi_csv = f'k_pfaf_{self.basn_id}_hig.csv' - self.xhi_csv = f'x_pfaf_{self.basn_id}_hig.csv' - - self.srt_csv = f'sort_pfaf_{self.basn_id}_topo.csv' - self.crd_csv = f'coords_pfaf_{self.basn_id}.csv' - self.bas_csv = f'riv_bas_id_pfaf_{self.basn_id}_topo.csv' - self.con_csv = f'rapid_connect_pfaf_{self.basn_id}.csv' - self.cat_csv = f'rapid_catchment_pfaf_{self.basn_id}.csv' - - #--------------------------------------------------------------------- - #Times {yyyy_mm} - #--------------------------------------------------------------------- - self.dat_str = datetime.datetime.strptime(self.yyyy_mm,'%Y-%m') - self.dat_end = self.dat_str \ - + dateutil.relativedelta.relativedelta(months=1) \ - + dateutil.relativedelta.relativedelta(seconds=-1) - self.iso_str = self.dat_str.isoformat() - self.iso_end = self.dat_end.isoformat() - - #--------------------------------------------------------------------- - #Land Surface Model {lsm_mod} {lsm_stp} {yyyy_mm} - #--------------------------------------------------------------------- - self.lsm_tmp = f'GLDAS_{self.lsm_mod}_{self.lsm_stp}_{self.yyyy_mm}_utc_tmp.nc4' - self.lsm_ncf = f'GLDAS_{self.lsm_mod}_{self.lsm_stp}_{self.yyyy_mm}_utc.nc4' - - #--------------------------------------------------------------------- - #Coupling {basn_id} - #--------------------------------------------------------------------- - self.cpl_csv = f'rapid_coupling_pfaf_{self.basn_id}_GLDAS.csv' - - #--------------------------------------------------------------------- - #Lateral inflow {basn_id} {lsm_mod} {lsm_stp} {yyyy_mm} - #--------------------------------------------------------------------- - self.m3r_ncf = f'm3_riv_pfaf_{self.basn_id}_GLDAS_{self.lsm_mod}_{self.lsm_stp}_{self.yyyy_mm}_utc.nc4' - - -#******************************************************************************* -#Driver for downloading -#******************************************************************************* + basn_id: str + lsm_mod: str + lsm_stp: str + yyyy_mm: str + + def __init__(self, basn_id, lsm_mod, lsm_stp, yyyy_mm) -> None: + self.basn_id = basn_id + self.lsm_mod = lsm_mod + self.lsm_stp = lsm_stp + self.yyyy_mm = yyyy_mm + + self.set_paths() + + def set_paths(self): + # Uses “formatted string literals” also known as f-strings. + + # --------------------------------------------------------------------- + # Directories + # --------------------------------------------------------------------- + self.inp_dir = '/tmp/input/' + os.makedirs(self.inp_dir, exist_ok=True) + self.hyd_dir = '/rrr/input/pfaf_{self.basn_id}/' + self.lsm_dir = ( + f'/tmp/input/pfaf_{self.basn_id}/GLDAS20/{self.lsm_mod}/' + f'{self.lsm_stp}/{self.yyyy_mm}/' + ) + os.makedirs(self.lsm_dir, exist_ok=True) + self.out_dir = '/tmp/' + os.makedirs(self.out_dir, exist_ok=True) + + # --------------------------------------------------------------------- + # Hydrography {basn_id} + # --------------------------------------------------------------------- + self.riv_shp = ( + f"riv_pfaf_{self.basn_id}_MERIT_Hydro_v07_Basins_v01_" + f"GLDAS_ENS.shp" + ) + self.cat_shp = ( + f"cat_pfaf_{self.basn_id}_MERIT_Hydro_v07_Basins_v01.shp" + ) + self.kfc_csv = f'kfac_pfaf_{self.basn_id}_1km_hour.csv' + self.xfc_csv = f'xfac_pfaf_{self.basn_id}_0.1.csv' + self.klo_csv = f'k_pfaf_{self.basn_id}_low.csv' + self.xlo_csv = f'x_pfaf_{self.basn_id}_low.csv' + self.knr_csv = f'k_pfaf_{self.basn_id}_nrm.csv' + self.xnr_csv = f'x_pfaf_{self.basn_id}_nrm.csv' + self.khi_csv = f'k_pfaf_{self.basn_id}_hig.csv' + self.xhi_csv = f'x_pfaf_{self.basn_id}_hig.csv' + + self.srt_csv = f'sort_pfaf_{self.basn_id}_topo.csv' + self.crd_csv = f'coords_pfaf_{self.basn_id}.csv' + self.bas_csv = f'riv_bas_id_pfaf_{self.basn_id}_topo.csv' + self.con_csv = f'rapid_connect_pfaf_{self.basn_id}.csv' + self.cat_csv = f'rapid_catchment_pfaf_{self.basn_id}.csv' + + # --------------------------------------------------------------------- + # Times {yyyy_mm} + # --------------------------------------------------------------------- + self.dat_str = datetime.datetime.strptime(self.yyyy_mm, '%Y-%m') + self.dat_end = self.dat_str \ + + dateutil.relativedelta.relativedelta(months=1) \ + + dateutil.relativedelta.relativedelta(seconds=-1) + self.iso_str = self.dat_str.isoformat() + self.iso_end = self.dat_end.isoformat() + + # --------------------------------------------------------------------- + # Land Surface Model {lsm_mod} {lsm_stp} {yyyy_mm} + # --------------------------------------------------------------------- + self.lsm_tmp = ( + f"GLDAS_{self.lsm_mod}_" + f"{self.lsm_stp}_" + f"{self.yyyy_mm}_utc_tmp.nc4" + ) + self.lsm_ncf = ( + f"GLDAS_{self.lsm_mod}_" + f"{self.lsm_stp}_" + f"{self.yyyy_mm}_utc.nc4" + ) + + # --------------------------------------------------------------------- + # Coupling {basn_id} + # --------------------------------------------------------------------- + self.cpl_csv = f'rapid_coupling_pfaf_{self.basn_id}_GLDAS.csv' + + # --------------------------------------------------------------------- + # Lateral inflow {basn_id} {lsm_mod} {lsm_stp} {yyyy_mm} + # --------------------------------------------------------------------- + self.m3r_ncf = ( + f"m3_riv_pfaf_{self.basn_id}_" + f"GLDAS_{self.lsm_mod}_" + f"{self.lsm_stp}_" + f"{self.yyyy_mm}_utc.nc4" + ) + + +# ***************************************************************************** +# Driver for downloading +# ***************************************************************************** def drv_dwn(rrr: RRR): - print('Driver for downloading') - - #-------------------------------------------------------------------------- - #Download GLDAS files - #-------------------------------------------------------------------------- - print('- Download GLDAS files') - comnd=['../src/rrr_lsm_tot_ldas.py'] \ - +['GLDAS'] \ - +[rrr.lsm_mod] \ - +[rrr.lsm_stp] \ - +[rrr.iso_str] \ - +[rrr.iso_end] \ - +[rrr.lsm_dir] \ - +['org_no'] - subprocess.run(comnd, capture_output=True, check=True) - - -#******************************************************************************* -#Driver fof processing of hydrography -#******************************************************************************* + print('Driver for downloading') + + # ------------------------------------------------------------------------- + # Download GLDAS files + # ------------------------------------------------------------------------- + print('- Download GLDAS files') + comnd = ['../src/rrr_lsm_tot_ldas.py'] \ + + ['GLDAS'] \ + + [rrr.lsm_mod] \ + + [rrr.lsm_stp] \ + + [rrr.iso_str] \ + + [rrr.iso_end] \ + + [rrr.lsm_dir] \ + + ['org_no'] + subprocess.run(comnd, capture_output=True, check=True) + + +# ***************************************************************************** +# Driver fof processing of hydrography +# ***************************************************************************** def drv_hyd(rrr: RRR): - print('Driver for processing of hydrography') - - #-------------------------------------------------------------------------- - #Connectivity, base parameters, coordinates, sort - #-------------------------------------------------------------------------- - print('- Connectivity, base parameters, coordinates, sort') - comnd=['../src/rrr_riv_tot_gen_all_meritbasins.py'] \ - +[rrr.hyd_dir + rrr.riv_shp] \ - +['5'] \ - +[rrr.out_dir + rrr.con_csv] \ - +[rrr.out_dir + rrr.kfc_csv] \ - +[rrr.out_dir + rrr.xfc_csv] \ - +[rrr.out_dir + rrr.srt_csv] \ - +[rrr.out_dir + rrr.crd_csv] - subprocess.run(comnd, capture_output=True, check=True) - - #-------------------------------------------------------------------------- - #Parameters - #-------------------------------------------------------------------------- - print('- Parameters') - comnd=['../src/rrr_riv_tot_scl_prm.py'] \ - +[rrr.out_dir + rrr.kfc_csv] \ - +[rrr.out_dir + rrr.xfc_csv] \ - +['0.20'] \ - +['0.00'] \ - +[rrr.out_dir + rrr.klo_csv] \ - +[rrr.out_dir + rrr.xlo_csv] - subprocess.run(comnd, capture_output=True, check=True) - - comnd=['../src/rrr_riv_tot_scl_prm.py'] \ - +[rrr.out_dir + rrr.kfc_csv] \ - +[rrr.out_dir + rrr.xfc_csv] \ - +['0.35'] \ - +['3.00'] \ - +[rrr.out_dir + rrr.knr_csv] \ - +[rrr.out_dir + rrr.xnr_csv] - subprocess.run(comnd, capture_output=True, check=True) - - comnd=['../src/rrr_riv_tot_scl_prm.py'] \ - +[rrr.out_dir + rrr.kfc_csv] \ - +[rrr.out_dir + rrr.xfc_csv] \ - +['0.50'] \ - +['5.00'] \ - +[rrr.out_dir + rrr.khi_csv] \ - +[rrr.out_dir + rrr.xhi_csv] - subprocess.run(comnd, capture_output=True, check=True) - - #-------------------------------------------------------------------------- - #Sorted subset - #-------------------------------------------------------------------------- - print('- Sorted subset') - comnd=['../src/rrr_riv_bas_gen_one_meritbasins.py'] \ - +[rrr.hyd_dir + rrr.riv_shp] \ - +[rrr.out_dir + rrr.con_csv] \ - +[rrr.out_dir + rrr.srt_csv] \ - +[rrr.out_dir + rrr.bas_csv] - subprocess.run(comnd, capture_output=True, check=True) - - #-------------------------------------------------------------------------- - #Contributing catchment information - #-------------------------------------------------------------------------- - print('- Contributing catchment information') - comnd=['../src/rrr_cat_tot_gen_one_meritbasins.py'] \ - +[rrr.hyd_dir + rrr.cat_shp] \ - +[rrr.out_dir + rrr.cat_csv] - subprocess.run(comnd, capture_output=True, check=True) - - -#******************************************************************************* -#Driver for processing land surface model outputs -#******************************************************************************* + print('Driver for processing of hydrography') + + # ------------------------------------------------------------------------- + # Connectivity, base parameters, coordinates, sort + # ------------------------------------------------------------------------- + print('- Connectivity, base parameters, coordinates, sort') + comnd = ['../src/rrr_riv_tot_gen_all_meritbasins.py'] \ + + [rrr.hyd_dir + rrr.riv_shp] \ + + ['5'] \ + + [rrr.out_dir + rrr.con_csv] \ + + [rrr.out_dir + rrr.kfc_csv] \ + + [rrr.out_dir + rrr.xfc_csv] \ + + [rrr.out_dir + rrr.srt_csv] \ + + [rrr.out_dir + rrr.crd_csv] + subprocess.run(comnd, capture_output=True, check=True) + + # ------------------------------------------------------------------------- + # Parameters + # ------------------------------------------------------------------------- + print('- Parameters') + comnd = ['../src/rrr_riv_tot_scl_prm.py'] \ + + [rrr.out_dir + rrr.kfc_csv] \ + + [rrr.out_dir + rrr.xfc_csv] \ + + ['0.20'] \ + + ['0.00'] \ + + [rrr.out_dir + rrr.klo_csv] \ + + [rrr.out_dir + rrr.xlo_csv] + subprocess.run(comnd, capture_output=True, check=True) + + comnd = ['../src/rrr_riv_tot_scl_prm.py'] \ + + [rrr.out_dir + rrr.kfc_csv] \ + + [rrr.out_dir + rrr.xfc_csv] \ + + ['0.35'] \ + + ['3.00'] \ + + [rrr.out_dir + rrr.knr_csv] \ + + [rrr.out_dir + rrr.xnr_csv] + subprocess.run(comnd, capture_output=True, check=True) + + comnd = ['../src/rrr_riv_tot_scl_prm.py'] \ + + [rrr.out_dir + rrr.kfc_csv] \ + + [rrr.out_dir + rrr.xfc_csv] \ + + ['0.50'] \ + + ['5.00'] \ + + [rrr.out_dir + rrr.khi_csv] \ + + [rrr.out_dir + rrr.xhi_csv] + subprocess.run(comnd, capture_output=True, check=True) + + # ------------------------------------------------------------------------- + # Sorted subset + # ------------------------------------------------------------------------- + print('- Sorted subset') + comnd = ['../src/rrr_riv_bas_gen_one_meritbasins.py'] \ + + [rrr.hyd_dir + rrr.riv_shp] \ + + [rrr.out_dir + rrr.con_csv] \ + + [rrr.out_dir + rrr.srt_csv] \ + + [rrr.out_dir + rrr.bas_csv] + subprocess.run(comnd, capture_output=True, check=True) + + # ------------------------------------------------------------------------- + # Contributing catchment information + # ------------------------------------------------------------------------- + print('- Contributing catchment information') + comnd = ['../src/rrr_cat_tot_gen_one_meritbasins.py'] \ + + [rrr.hyd_dir + rrr.cat_shp] \ + + [rrr.out_dir + rrr.cat_csv] + subprocess.run(comnd, capture_output=True, check=True) + + +# ***************************************************************************** +# Driver for processing land surface model outputs +# ***************************************************************************** def drv_lsm(rrr: RRR): - print('Driver for processing land surface model outputs') - - #-------------------------------------------------------------------------- - #Combine and accumulate multiple files - #-------------------------------------------------------------------------- - print('- Combine and accumulate multiple files') - all_nc4=sorted(glob.glob(rrr.lsm_dir + '*.nc4')) - comnd=['/bin/bash'] \ - +['../src/rrr_lsm_tot_cmb_acc.sh'] \ - +all_nc4 \ - +['1'] \ - +[rrr.out_dir + rrr.lsm_tmp] - subprocess.run(comnd, capture_output=True, check=True) - - #-------------------------------------------------------------------------- - #Make file CF compliant - #-------------------------------------------------------------------------- - print('- Make file CF compliant') - comnd=['../src/rrr_lsm_tot_add_cfc.py'] \ - +[rrr.out_dir + rrr.lsm_tmp] \ - +[rrr.iso_str] \ - +['10800'] \ - +['1.0'] \ - +[rrr.out_dir + rrr.lsm_ncf] - subprocess.run(comnd, capture_output=True, check=True) - - #-------------------------------------------------------------------------- - #Delete temporary file - #-------------------------------------------------------------------------- - print('- Delete temporary file') - comnd=['rm'] \ - +[rrr.out_dir + rrr.lsm_tmp] - subprocess.run(comnd, capture_output=True, check=True) - - -#******************************************************************************* -#Driver for coupling -#******************************************************************************* + print('Driver for processing land surface model outputs') + + # ------------------------------------------------------------------------- + # Combine and accumulate multiple files + # ------------------------------------------------------------------------- + print('- Combine and accumulate multiple files') + all_nc4 = sorted(glob.glob(rrr.lsm_dir + '*.nc4')) + comnd = ['/bin/bash'] \ + + ['../src/rrr_lsm_tot_cmb_acc.sh'] \ + + all_nc4 \ + + ['1'] \ + + [rrr.out_dir + rrr.lsm_tmp] + subprocess.run(comnd, capture_output=True, check=True) + + # ------------------------------------------------------------------------- + # Make file CF compliant + # ------------------------------------------------------------------------- + print('- Make file CF compliant') + comnd = ['../src/rrr_lsm_tot_add_cfc.py'] \ + + [rrr.out_dir + rrr.lsm_tmp] \ + + [rrr.iso_str] \ + + ['10800'] \ + + ['1.0'] \ + + [rrr.out_dir + rrr.lsm_ncf] + subprocess.run(comnd, capture_output=True, check=True) + + # ------------------------------------------------------------------------- + # Delete temporary file + # ------------------------------------------------------------------------- + print('- Delete temporary file') + comnd = ['rm'] \ + + [rrr.out_dir + rrr.lsm_tmp] + subprocess.run(comnd, capture_output=True, check=True) + + +# ***************************************************************************** +# Driver for coupling +# ***************************************************************************** def drv_cpl(rrr: RRR): - print('Driver for coupling') + print('Driver for coupling') - #-------------------------------------------------------------------------- - #Create coupling file - #-------------------------------------------------------------------------- - print('- Create coupling file') - comnd=['../src/rrr_cpl_riv_lsm_lnk.py'] \ - +[rrr.out_dir + rrr.con_csv] \ - +[rrr.out_dir + rrr.cat_csv] \ - +[rrr.out_dir + rrr.lsm_ncf] \ - +[rrr.out_dir + rrr.cpl_csv] - subprocess.run(comnd, capture_output=True, check=True) + # ------------------------------------------------------------------------ + # Create coupling file + # ------------------------------------------------------------------------ + print('- Create coupling file') + comnd = ['../src/rrr_cpl_riv_lsm_lnk.py'] \ + + [rrr.out_dir + rrr.con_csv] \ + + [rrr.out_dir + rrr.cat_csv] \ + + [rrr.out_dir + rrr.lsm_ncf] \ + + [rrr.out_dir + rrr.cpl_csv] + subprocess.run(comnd, capture_output=True, check=True) -#******************************************************************************* -#Driver for volume -#******************************************************************************* +# ***************************************************************************** +# Driver for volume +# ***************************************************************************** def drv_vol(rrr: RRR): - print('Driver for volume') - - #-------------------------------------------------------------------------- - #Create volume file - #-------------------------------------------------------------------------- - print('- Create volume file') - comnd=['../src/rrr_cpl_riv_lsm_vol.py'] \ - +[rrr.out_dir + rrr.con_csv] \ - +[rrr.out_dir + rrr.crd_csv] \ - +[rrr.out_dir + rrr.lsm_ncf] \ - +[rrr.out_dir + rrr.cpl_csv] \ - +[rrr.out_dir + rrr.m3r_ncf] - subprocess.run(comnd, capture_output=True, check=True) - - #-------------------------------------------------------------------------- - #Update netCDF attributes - #-------------------------------------------------------------------------- - print('- Update netCDF attributes') - comnd=['../src/rrr_cpl_riv_lsm_att.py'] \ - +[rrr.out_dir + rrr.m3r_ncf] \ - +['RRR data corresponding to MERIT Hydro 07 Basin 01 pfaf_' \ - +rrr.basn_id+', GLDAS '+rrr.lsm_mod ] \ - +['Jet Propulsion Laboratory, California Institute of Technology'] \ - +[''] \ - +['6378137'] \ - +['298.257222101'] - subprocess.run(comnd, capture_output=True, check=True) - - -#******************************************************************************* -#Driver for all -#******************************************************************************* + print('Driver for volume') + + # ------------------------------------------------------------------------- + # Create volume file + # ------------------------------------------------------------------------- + print('- Create volume file') + comnd = ['../src/rrr_cpl_riv_lsm_vol.py'] \ + + [rrr.out_dir + rrr.con_csv] \ + + [rrr.out_dir + rrr.crd_csv] \ + + [rrr.out_dir + rrr.lsm_ncf] \ + + [rrr.out_dir + rrr.cpl_csv] \ + + [rrr.out_dir + rrr.m3r_ncf] + subprocess.run(comnd, capture_output=True, check=False, text=True) + + # -------------------------------------------------------------------------- + # Update netCDF attributes + # -------------------------------------------------------------------------- + print('- Update netCDF attributes') + comnd = [ + '../src/rrr_cpl_riv_lsm_att.py', + rrr.out_dir + rrr.m3r_ncf, + ( + f"RRR data corresponding to MERIT Hydro 07 Basin 01 pfaf_" + f"{rrr.basn_id}, GLDAS {rrr.lsm_mod}" + ), + "Jet Propulsion Laboratory, California Institute of Technology", + "", + "6378137", + "298.257222101", + ] + subprocess.run(comnd, capture_output=True, check=True) + + +# ***************************************************************************** +# Driver for all +# ***************************************************************************** def drv_all(rrr: RRR): - #-------------------------------------------------------------------------- - #Running all drivers - #-------------------------------------------------------------------------- - drv_dwn(rrr) - drv_hyd(rrr) - drv_lsm(rrr) - drv_cpl(rrr) - drv_vol(rrr) + # ------------------------------------------------------------------------- + # Running all drivers + # ------------------------------------------------------------------------- + drv_dwn(rrr) + drv_hyd(rrr) + drv_lsm(rrr) + drv_cpl(rrr) + drv_vol(rrr) -##******************************************************************************* -##For testing purposes -##******************************************************************************* +# ***************************************************************************** +# For testing purposes +# ***************************************************************************** # -##------------------------------------------------------------------------------- -##Execution of subprocess -##------------------------------------------------------------------------------- -#comnd=[] -#comnd.append('echo') -#comnd.append('TEST') -#subprocess.run(comnd, capture_output=True, check=True) -##subprocess.run(comnd, check=True) -##subprocess.Popen(comnd) +# #---------------------------------------------------------------------------- +# #Execution of subprocess +# #---------------------------------------------------------------------------- +# comnd=[] +# comnd.append('echo') +# comnd.append('TEST') +# subprocess.run(comnd, capture_output=True, check=True) +# #subprocess.run(comnd, check=True) +# #subprocess.Popen(comnd) # -##------------------------------------------------------------------------------- -##Running drivers -##------------------------------------------------------------------------------- -#rrr = RRR('74', 'VIC', '3H', '2000-01') -#drv_dwn(rrr) -#drv_hyd(rrr) -#drv_lsm(rrr) -#drv_cpl(rrr) -#drv_vol(rrr) - - -#******************************************************************************* -#End -#******************************************************************************* +# #---------------------------------------------------------------------------- +# #Running drivers +# #---------------------------------------------------------------------------- +# rrr = RRR('74', 'VIC', '3H', '2000-01') +# drv_dwn(rrr) +# drv_hyd(rrr) +# drv_lsm(rrr) +# drv_cpl(rrr) +# drv_vol(rrr) + + +# ***************************************************************************** +# End +# ***************************************************************************** diff --git a/requirements.apt b/requirements.apt index a73ad5a..9e30340 100644 --- a/requirements.apt +++ b/requirements.apt @@ -45,6 +45,8 @@ ffmpeg #multimedia file transcoding nco #netCDF operators +cdo +#climate data operators (for NLDAS GRIB to netCDF conversion) #------------------------------------------------------------------------------- #Code testing diff --git a/requirements.pip b/requirements.pip index 0c2283c..9c30b4b 100644 --- a/requirements.pip +++ b/requirements.pip @@ -45,6 +45,7 @@ requests==2.28.1 Rtree==1.0.0 scipy==1.7.3 Shapely==1.8.2 +earthaccess==0.9.0 #******************************************************************************* diff --git a/src/rrr_lsm_tot_ldas.py b/src/rrr_lsm_tot_ldas.py index 8f317fa..d4de58f 100755 --- a/src/rrr_lsm_tot_ldas.py +++ b/src/rrr_lsm_tot_ldas.py @@ -1,60 +1,64 @@ #!/usr/bin/env python3 -#******************************************************************************* -#rrr_lsm_tot_ldas.py -#******************************************************************************* +# ***************************************************************************** +# rrr_lsm_tot_ldas.py +# ***************************************************************************** -#Purpose: -#Given and model name, a temporal frequency key, a start date, an end date, and -#a folder path, this script downloads LDAS data from GES-DISC using the -#NASA EarthData credentials stored locally in '~/.netrc' file. -#Author: -#Cedric H. David, 2018-2023 +# Purpose: +# Given and model name, a temporal frequency key, a start date, an end date, +# and a folder path, this script downloads LDAS data from GES-DISC using the +# earthaccess library +# Author: +# Cedric H. David, Manu Tom 2018-24 -#******************************************************************************* -#Import Python modules -#******************************************************************************* +# ***************************************************************************** +# Import Python modules +# ***************************************************************************** import sys import os.path import datetime +import earthaccess +import glob +import subprocess import requests +import simplejson -#******************************************************************************* -#Declaration of variables (given as command line arguments) -#******************************************************************************* +# ***************************************************************************** +# Declaration of variables (given as command line arguments) +# ***************************************************************************** # 1 - rrr_lsm_exp # 2 - rrr_lsm_mod # 3 - rrr_lsm_frq # 4 - rrr_iso_beg # 5 - rrr_iso_end # 6 - rrr_lsm_dir -#(7)- rrr_lsm_org +# (7)- rrr_lsm_org -#******************************************************************************* -#Get command line arguments -#******************************************************************************* -IS_arg=len(sys.argv) +# ***************************************************************************** +# Get command line arguments +# ***************************************************************************** +IS_arg = len(sys.argv) if IS_arg < 7 or IS_arg > 8: - print('ERROR - A minimum of 6 and a maximum of 7 arguments can be used') - raise SystemExit(22) - -rrr_lsm_exp=sys.argv[1] -rrr_lsm_mod=sys.argv[2] -rrr_lsm_frq=sys.argv[3] -rrr_iso_beg=sys.argv[4] -rrr_iso_end=sys.argv[5] -rrr_lsm_dir=sys.argv[6] -if IS_arg==8: - rrr_lsm_org=sys.argv[7] + print('ERROR - A minimum of 6 and a maximum of 7 arguments can be used') + raise SystemExit(22) + +rrr_lsm_exp = sys.argv[1] +rrr_lsm_mod = sys.argv[2] +rrr_lsm_frq = sys.argv[3] +rrr_iso_beg = sys.argv[4] +rrr_iso_end = sys.argv[5] +rrr_lsm_dir = sys.argv[6] +if IS_arg == 8: + rrr_lsm_org = sys.argv[7] else: - rrr_lsm_org='' + rrr_lsm_org = '' -#******************************************************************************* -#Print input information -#******************************************************************************* +# ***************************************************************************** +# Print input information +# ***************************************************************************** print('Command line inputs') print('- '+rrr_lsm_exp) print('- '+rrr_lsm_mod) @@ -65,644 +69,229 @@ print('- '+rrr_lsm_org) -#******************************************************************************* -#Check if directory exists -#******************************************************************************* -rrr_lsm_dir=os.path.join(rrr_lsm_dir,'') -#add trailing slash if it is not there +# ***************************************************************************** +# Check if directory exists +# ***************************************************************************** +rrr_lsm_dir = os.path.join(rrr_lsm_dir, '') +# add trailing slash if it is not there if not os.path.isdir(rrr_lsm_dir): - os.mkdir(rrr_lsm_dir) + os.mkdir(rrr_lsm_dir) -#******************************************************************************* -#Check LDAS arguments -#******************************************************************************* +# ***************************************************************************** +# Check LDAS arguments +# ***************************************************************************** print('Check LDAS arguments') -if rrr_lsm_exp=='NLDAS' or rrr_lsm_exp=='GLDAS': - print('- LDAS name is valid') +if rrr_lsm_exp == 'NLDAS' or rrr_lsm_exp == 'GLDAS': + print('- LDAS name is valid') else: - print('ERROR - Invalid LDAS name') - raise SystemExit(22) + print('ERROR - Invalid LDAS name') + raise SystemExit(22) -if rrr_lsm_mod=='VIC' or rrr_lsm_mod=='NOAH' or rrr_lsm_mod=='MOS' \ - or rrr_lsm_mod=='CLSM': - print('- Model name is valid') +if rrr_lsm_mod == 'VIC' or rrr_lsm_mod == 'NOAH' or rrr_lsm_mod == 'MOS' \ + or rrr_lsm_mod == 'CLSM': + print('- Model name is valid') else: - print('ERROR - Invalid model name') - raise SystemExit(22) + print('ERROR - Invalid model name') + raise SystemExit(22) -if rrr_lsm_frq=='H' or rrr_lsm_frq=='M' or rrr_lsm_frq=='3H': - print('- Data frequency is valid') +if rrr_lsm_frq == 'H' or rrr_lsm_frq == 'M' or rrr_lsm_frq == '3H': + print('- Data frequency is valid') else: - print('ERROR - Invalid data frequency') - raise SystemExit(22) + print('ERROR - Invalid data frequency') + raise SystemExit(22) -#******************************************************************************* -#Check temporal information -#******************************************************************************* +# ***************************************************************************** +# Check temporal information +# ***************************************************************************** print('Check temporal information') -rrr_dat_beg=datetime.datetime.strptime(rrr_iso_beg,'%Y-%m-%dT%H:%M:%S') -rrr_dat_end=datetime.datetime.strptime(rrr_iso_end,'%Y-%m-%dT%H:%M:%S') +rrr_dat_beg = datetime.datetime.strptime(rrr_iso_beg, '%Y-%m-%dT%H:%M:%S') +rrr_dat_end = datetime.datetime.strptime(rrr_iso_end, '%Y-%m-%dT%H:%M:%S') -if rrr_dat_end>=rrr_dat_beg: - print('- Beginning of interval is before end of interval') +if rrr_dat_end >= rrr_dat_beg: + print('- Beginning of interval is before end of interval') else: - print('ERROR - Beginning of interval is NOT before end of interval') - raise SystemExit(22) - -rrr_dat_stp=rrr_dat_beg -IS_count=0 -#Initialized when to stop downloading and the number of files to download - -#------------------------------------------------------------------------------- -#If requesting hourly data -#------------------------------------------------------------------------------- -if rrr_lsm_frq=='H': - if rrr_dat_beg.minute==0 and rrr_dat_beg.second==0: - print('- Interval starts at the top of an hour') - else: - print('ERROR - The interval does NOT start at the top of an hour: ' \ - +rrr_iso_beg) - raise SystemExit(22) - - while rrr_dat_stp<=rrr_dat_end: - rrr_dat_stp=rrr_dat_stp+datetime.timedelta(hours=1) - #Adding one hour - IS_count=IS_count+1 - print('- The number of files to be downloaded is: '+str(IS_count)) - -#------------------------------------------------------------------------------- -#If requesting 3-hourly data -#------------------------------------------------------------------------------- -if rrr_lsm_frq=='3H': - if rrr_dat_beg.hour %3==0 and rrr_dat_beg.minute==0 and \ - rrr_dat_beg.second==0: - print('- Interval starts at the top of a 3-hour') - else: - print('ERROR - The interval does NOT start at the top of a 3-hour: ' \ - +rrr_iso_beg) - raise SystemExit(22) - - while rrr_dat_stp<=rrr_dat_end: - rrr_dat_stp=rrr_dat_stp+datetime.timedelta(hours=3) - #Adding one hour - IS_count=IS_count+1 - print('- The number of files to be downloaded is: '+str(IS_count)) - -#------------------------------------------------------------------------------- -#If requesting monthly data -#------------------------------------------------------------------------------- -if rrr_lsm_frq=='M': - if rrr_dat_beg.day==1 and rrr_dat_beg.hour==0 and \ - rrr_dat_beg.minute==0 and rrr_dat_beg.second==0: - print('- Interval starts at the top of a month') - else: - print('ERROR - The interval does NOT start at the top of a month: ' \ - +rrr_iso_beg) - raise SystemExit(22) - - while rrr_dat_stp<=rrr_dat_end: - rrr_dat_stp=(rrr_dat_stp+datetime.timedelta(days=32)).replace(day=1) - #Adding one month done by adding 32 days and replacing the day by 1 - IS_count=IS_count+1 - print('- The number of files to be downloaded is: '+str(IS_count)) - - -#******************************************************************************* -#Check if retaining LDAS directory structure -#******************************************************************************* + print('ERROR - Beginning of interval is NOT before end of interval') + raise SystemExit(22) + +rrr_dat_stp = rrr_dat_beg +IS_count = 0 +# Initialized when to stop downloading and the number of files to download + +# ----------------------------------------------------------------------------- +# If requesting hourly data +# ----------------------------------------------------------------------------- +if rrr_lsm_frq == 'H': + if rrr_dat_beg.minute == 0 and rrr_dat_beg.second == 0: + print('- Interval starts at the top of an hour') + else: + print('ERROR - The interval does NOT start at the top of an hour: ' + + rrr_iso_beg) + raise SystemExit(22) + + while rrr_dat_stp <= rrr_dat_end: + rrr_dat_stp = rrr_dat_stp + datetime.timedelta(hours=1) + # Adding one hour + IS_count = IS_count + 1 + print('- The number of files to be downloaded is: '+str(IS_count)) + +# ----------------------------------------------------------------------------- +# If requesting 3-hourly data +# ----------------------------------------------------------------------------- +if rrr_lsm_frq == '3H': + if rrr_dat_beg.hour % 3 == 0 and rrr_dat_beg.minute == 0 and \ + rrr_dat_beg.second == 0: + print('- Interval starts at the top of a 3-hour') + else: + print('ERROR - The interval does NOT start at the top of a 3-hour: ' + + rrr_iso_beg) + raise SystemExit(22) + + while rrr_dat_stp <= rrr_dat_end: + rrr_dat_stp = rrr_dat_stp+datetime.timedelta(hours=3) + # Adding one hour + IS_count = IS_count+1 + print('- The number of files to be downloaded is: ' + str(IS_count)) + +# ----------------------------------------------------------------------------- +# If requesting monthly data +# ----------------------------------------------------------------------------- +if rrr_lsm_frq == 'M': + if rrr_dat_beg.day == 1 and rrr_dat_beg.hour == 0 \ + and rrr_dat_beg.minute == 0 and rrr_dat_beg.second == 0: + print('- Interval starts at the top of a month') + else: + print('ERROR - The interval does NOT start at the top of a month: ' + + rrr_iso_beg) + raise SystemExit(22) + + while rrr_dat_stp <= rrr_dat_end: + rrr_dat_stp = (rrr_dat_stp+datetime.timedelta(days=32)).replace(day=1) + # Adding one month done by adding 32 days and replacing the day by 1 + IS_count = IS_count+1 + print('- The number of files to be downloaded is: '+str(IS_count)) + +# ***************************************************************************** +# Check if retaining LDAS directory structure +# ***************************************************************************** print('Check if retaining LDAS directory structure') -if rrr_lsm_org=='org_no': - print('- Not retaining LDAS directory structure') - -if (rrr_lsm_org!='org_no') and (rrr_lsm_org!=''): - print('ERROR: '+rrr_lsm_org+' does not match org_no') - raise SystemExit(22) - - -#******************************************************************************* -#Obtaining credentials for the server from a local file -#******************************************************************************* -print('Obtaining credentials for the server from a local file') - -url='https://urs.earthdata.nasa.gov' -print('- '+url) - -cred=requests.utils.get_netrc_auth(url) -if cred!=None: - print('- The credentials were obtained from ~/.netrc file') +if rrr_lsm_org == 'org_no': + print('- Not retaining LDAS directory structure') + +if (rrr_lsm_org != 'org_no') and (rrr_lsm_org != ''): + print('ERROR: '+rrr_lsm_org+' does not match org_no') + raise SystemExit(22) + +# ***************************************************************************** +# Earthaccess download parameters +# ***************************************************************************** +if (rrr_lsm_exp == "GLDAS"): + spatial_res = "10" + boundingBox = (-180, -60, 180, 90) +elif (rrr_lsm_exp == "NLDAS"): + spatial_res = "0125" + boundingBox = (-125, 25, -67, 53) else: - print('ERROR - Credentials not available in ~/.netrc file') - raise SystemExit(22) - -cred_obj=requests.auth.HTTPDigestAuth(cred[0],cred[1]) - - -#******************************************************************************* -#Checking that service and credentials work for one known file -#******************************************************************************* - -#------------------------------------------------------------------------------- -#If requesting NLDAS hourly data -#------------------------------------------------------------------------------- -if rrr_lsm_exp=='NLDAS' and rrr_lsm_frq=='H': - print('Checking that service and credentials work for one known file') - - url='https://hydro1.gesdisc.eosdis.nasa.gov/daac-bin/OTF/HTTP_services.cgi' - payload={} - payload['FILENAME']='/data/NLDAS/NLDAS_VIC0125_H.002/2000/001/' \ - +'NLDAS_VIC0125_H.A20000101.0000.002.grb' - payload['FORMAT']='bmM0Lw' - payload['BBOX']='25,-125,53,-67' - payload['LABEL']='NLDAS_VIC0125_H.A20000101.0000.002.grb.SUB.nc4' - payload['SHORTNAME']='NLDAS_VIC0125_H' - payload['SERVICE']='L34RS_LDAS' - payload['VERSION']='1.02' - payload['DATASET_VERSION']='002' - payload['VARIABLES']='BGRUN,SSRUN' - - print('- Requesting a subset of NLDAS_VIC0125_H.A20000101.0000.002.grb') - r=requests.get(url, params=payload, auth=cred_obj) - #Downloads data from: - #https://hydro1.gesdisc.eosdis.nasa.gov/daac-bin/OTF/HTTP_services.cgi - # ?FILENAME=/data/NLDAS/NLDAS_VIC0125_H.002/2000/001/ - # NLDAS_VIC0125_H.A20000101.0000.002.grb - # &FORMAT=bmM0Lw - # &BBOX=25,-125,53,-67 - # &LABEL=NLDAS_VIC0125_H.A20000101.0000.002.grb.SUB.nc4 - # &SHORTNAME=NLDAS_VIC0125_H - # &SERVICE=L34RS_LDAS - # &VERSION=1.02 - # &DATASET_VERSION=002 - # &VARIABLES=BGRUN,SSRUN' - #requests.get() actually downloads the file into memory and also saves some - #associated download metadata - if r.ok: - print('- The request was successful') - else: - print('ERROR - Status code '+str(r.status_code)) - raise SystemExit(22) - -#------------------------------------------------------------------------------- -#If requesting NLDAS monthly data -#------------------------------------------------------------------------------- -if rrr_lsm_exp=='NLDAS' and rrr_lsm_frq=='M': - print('Checking that service and credentials work for one known file') - - url='https://hydro1.gesdisc.eosdis.nasa.gov/daac-bin/OTF/HTTP_services.cgi' - payload={} - payload['FILENAME']='/data/NLDAS/NLDAS_VIC0125_M.002/2000/' \ - +'NLDAS_VIC0125_M.A200001.002.grb' - payload['FORMAT']='bmM0Lw' - payload['BBOX']='25,-125,53,-67' - payload['LABEL']='NLDAS_VIC0125_M.A200001.002.grb.SUB.nc4' - payload['SHORTNAME']='NLDAS_VIC0125_M' - payload['SERVICE']='L34RS_LDAS' - payload['VERSION']='1.02' - payload['DATASET_VERSION']='002' - payload['VARIABLES']='BGRUN,SSRUN' - - print('- Requesting a subset of NLDAS_VIC0125_M.A200001.002.grb') - r=requests.get(url, params=payload, auth=cred_obj) - #Downloads data from: - #https://hydro1.gesdisc.eosdis.nasa.gov/daac-bin/OTF/HTTP_services.cgi - # ?FILENAME=/data/NLDAS/NLDAS_VIC0125_M.002/2000/ - # NLDAS_VIC0125_M.A200001.002.grb - # &FORMAT=bmM0Lw - # &BBOX=25,-125,53,-67 - # &LABEL=NLDAS_VIC0125_M.A200001.002.grb.SUB.nc4 - # &SHORTNAME=NLDAS_VIC0125_M - # &SERVICE=L34RS_LDAS - # &VERSION=1.02 - # &DATASET_VERSION=002 - # &VARIABLES=BGRUN,SSRUN' - #requests.get() actually downloads the file into memory and also saves some - #associated download metadata - if r.ok: - print('- The request was successful') - else: - print('ERROR - Status code '+str(r.status_code)) - raise SystemExit(22) - -#------------------------------------------------------------------------------- -#If requesting GLDAS 3-hourly data -#------------------------------------------------------------------------------- -if rrr_lsm_exp=='GLDAS' and rrr_lsm_frq=='3H': - print('Checking that service and credentials work for one known file') - - url='https://hydro1.gesdisc.eosdis.nasa.gov/daac-bin/OTF/HTTP_services.cgi' - payload={} - payload['FILENAME']='/data/GLDAS/GLDAS_VIC10_3H.2.0/2000/001/' \ - +'GLDAS_VIC10_3H.A20000101.0000.020.nc4' - payload['FORMAT']='bmM0Lw' - payload['BBOX']='-60,-180,90,180' - payload['LABEL']='GLDAS_VIC10_3H.A20000101.0000.020.nc4.SUB.nc4' - payload['SHORTNAME']='GLDAS_VIC10_3H' - payload['SERVICE']='L34RS_LDAS' - payload['VERSION']='1.02' - payload['DATASET_VERSION']='2.0' - payload['VARIABLES']='Qs_acc,Qsb_acc' - - print('- Requesting a subset of GLDAS_VIC10_3H.A2000001.0000.001.grb') - r=requests.get(url, params=payload, auth=cred_obj) - #Downloads data from: - #https://hydro1.gesdisc.eosdis.nasa.gov/daac-bin/OTF/HTTP_services.cgi - # ?FILENAME=/data/GLDAS/GLDAS_VIC10_3H.2.0/2000/001/ - # GLDAS_VIC10_3H.A20000101.0000.020.nc4 - # &FORMAT=bmM0Lw - # &BBOX=-60,-180,90,-180 - # &LABEL=GLDAS_VIC10_3H.A20000101.0000.020.nc4.SUB.nc4 - # &SHORTNAME=GLDAS_VIC10_3H - # &SERVICE=L34RS_LDAS - # &VERSION=1.02 - # &DATASET_VERSION=2.0 - # &VARIABLES=Qs_acc,Qsb_acc' - #requests.get() actually downloads the file into memory and also saves some - #associated download metadata - if r.ok: - print('- The request was successful') - else: - print('ERROR - Status code '+str(r.status_code)) - raise SystemExit(22) - -#------------------------------------------------------------------------------- -#If requesting GLDAS monthly data -#------------------------------------------------------------------------------- -if rrr_lsm_exp=='GLDAS' and rrr_lsm_frq=='M': - print('Checking that service and credentials work for one known file') - - url='https://hydro1.gesdisc.eosdis.nasa.gov/daac-bin/OTF/HTTP_services.cgi' - payload={} - payload['FILENAME']='/data/GLDAS/GLDAS_VIC10_M.2.0/2000/' \ - +'GLDAS_VIC10_M.A200001.020.nc4' - payload['FORMAT']='bmM0Lw' - payload['BBOX']='-60,-180,90,180' - payload['LABEL']='GLDAS_VIC10_M.A200001.020.nc4.SUB.nc4' - payload['SHORTNAME']='GLDAS_VIC10_M' - payload['SERVICE']='L34RS_LDAS' - payload['VERSION']='1.02' - payload['DATASET_VERSION']='2.0' - payload['VARIABLES']='Qs_acc,Qsb_acc' - - print('- Requesting a subset of GLDAS_VIC10_M.A200001.020.nc4') - r=requests.get(url, params=payload, auth=cred_obj) - #Downloads data from: - #https://hydro1.gesdisc.eosdis.nasa.gov/daac-bin/OTF/HTTP_services.cgi - # ?FILENAME=/data/GLDAS/GLDAS_VIC10_M.2.0/2000/ - # GLDAS_VIC10_M.A200001.020.nc4 - # &FORMAT=bmM0Lw - # &BBOX=-60,-180,90,180 - # &LABEL=GLDAS_VIC10_M.A200001.020.nc4.SUB.nc4 - # &SHORTNAME=GLDAS_VIC10_M - # &SERVICE=L34RS_LDAS - # &VERSION=1.02 - # &DATASET_VERSION=2.0 - # &VARIABLES=Qs_acc,Qsb_acc - #requests.get() actually downloads the file into memory and also saves some - #associated download metadata - if r.ok: - print('- The request was successful') - else: - print('ERROR - Status code '+str(r.status_code)) - raise SystemExit(22) - -#******************************************************************************* -#Downloading all files -#******************************************************************************* -print('Downloading all files') - -#------------------------------------------------------------------------------- -#Creating a networking session and assigning associated credentials -#------------------------------------------------------------------------------- -print('- Creating a networking session and assigning associated credentials') - -s=requests.Session() -s.auth=cred_obj - -#------------------------------------------------------------------------------- -#If requesting NLDAS hourly data -#------------------------------------------------------------------------------- -if rrr_lsm_exp=='NLDAS' and rrr_lsm_frq=='H': - - # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #Initializing URL and payload - # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - print('- Initializing URL and payload') - - url='https://hydro1.gesdisc.eosdis.nasa.gov/daac-bin/OTF/HTTP_services.cgi' - payload={} - payload['FILENAME']='/data/NLDAS/NLDAS_VIC0125_H.002/2000/001/' \ - +'NLDAS_VIC0125_H.A20000101.0000.002.grb' - payload['FORMAT']='bmM0Lw' - payload['BBOX']='25,-125,53,-67' - payload['LABEL']='NLDAS_VIC0125_H.A20000101.0000.002.grb.SUB.nc4' - payload['SHORTNAME']='NLDAS_VIC0125_H' - payload['SERVICE']='L34RS_LDAS' - payload['VERSION']='1.02' - payload['DATASET_VERSION']='002' - payload['VARIABLES']='BGRUN,SSRUN' - - # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #Looping over all files - # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - print('- Looping over all files') - - rrr_dat_cur=rrr_dat_beg - for JS_count in range(IS_count): - #- + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - #Determine current datetime and various date strings - #- + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - YS_yr=rrr_dat_cur.strftime('%Y') - YS_mo=rrr_dat_cur.strftime('%m') - YS_da=rrr_dat_cur.strftime('%d') - YS_hr=rrr_dat_cur.strftime('%H') - YS_dy=rrr_dat_cur.strftime('%j') - #- + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - #Generate file name - #- + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - payload['FILENAME']='/data/NLDAS/NLDAS_'+rrr_lsm_mod+'0125_H.002/' \ - +YS_yr+'/'+YS_dy+'/' \ - +'NLDAS_'+rrr_lsm_mod+'0125_H.A'+YS_yr+YS_mo+YS_da\ - +'.'+YS_hr+'00.002.grb' - payload['LABEL'] ='NLDAS_'+rrr_lsm_mod+'0125_H.A'+YS_yr+YS_mo+YS_da\ - +'.'+YS_hr+'00.002.grb.SUB.nc4' - #- + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - #Create directory if it doesn't exist - #- + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - YS_dir='NLDAS_'+rrr_lsm_mod+'0125_H.002/'+YS_yr+'/'+YS_dy+'/' - if rrr_lsm_org=='org_no': YS_dir='/' - if not os.path.isdir(rrr_lsm_dir+YS_dir): - os.makedirs(rrr_lsm_dir+YS_dir) - #- + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - #Place request if file does not already exist, and check it is ok - #- + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - if os.path.isfile(rrr_lsm_dir+YS_dir+payload['LABEL']): - print(' . Skipping '+payload['LABEL']) - else: - print(' . Downloading '+payload['LABEL']) - r=s.get(url, params=payload) - if not r.ok: - print('ERROR - status code '+str(r.status_code)+ \ - 'returned when downloading '+payload['FILENAME']) - raise SystemExit(22) - YS_name=r.headers['content-disposition'] - YS_name=YS_name.replace('attachment; filename=','') - YS_name=YS_name.replace('"','') - #The file name is extracted directly from requests.get() results - open(rrr_lsm_dir+YS_dir+YS_name, 'wb').write(r.content) - #The file is written on local disk - #- + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - #Increment current datetime - #- + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - rrr_dat_cur=rrr_dat_cur+datetime.timedelta(hours=1) - -#------------------------------------------------------------------------------- -#If requesting NLDAS monthly data -#------------------------------------------------------------------------------- -if rrr_lsm_exp=='NLDAS' and rrr_lsm_frq=='M': - - # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #Initializing URL and payload - # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - print('- Initializing URL and payload') - - url='https://hydro1.gesdisc.eosdis.nasa.gov/daac-bin/OTF/HTTP_services.cgi' - payload={} - payload['FILENAME']='/data/NLDAS/NLDAS_VIC0125_M.002/2000/' \ - +'NLDAS_VIC0125_M.A200001.002.grb' - payload['FORMAT']='bmM0Lw' - payload['BBOX']='25,-125,53,-67' - payload['LABEL']='NLDAS_VIC0125_M.A200001.002.grb.SUB.nc4' - payload['SHORTNAME']='NLDAS_VIC0125_M' - payload['SERVICE']='L34RS_LDAS' - payload['VERSION']='1.02' - payload['DATASET_VERSION']='002' - payload['VARIABLES']='BGRUN,SSRUN' - - # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #Looping over all files - # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - print('- Looping over all files') - - rrr_dat_cur=rrr_dat_beg - for JS_count in range(IS_count): - #- + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - #Determine current datetime and various date strings - #- + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - YS_yr=rrr_dat_cur.strftime('%Y') - YS_mo=rrr_dat_cur.strftime('%m') - YS_da=rrr_dat_cur.strftime('%d') - YS_hr=rrr_dat_cur.strftime('%H') - YS_dy=rrr_dat_cur.strftime('%j') - #- + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - #Generate file name - #- + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - payload['FILENAME']='/data/NLDAS/NLDAS_'+rrr_lsm_mod+'0125_M.002/' \ - +YS_yr+'/' \ - +'NLDAS_'+rrr_lsm_mod+'0125_M.A'+YS_yr+''+YS_mo \ - +'.002.grb' - payload['LABEL'] ='NLDAS_'+rrr_lsm_mod+'0125_M.A'+YS_yr+''+YS_mo \ - +'.002.grb.SUB.nc4' - #- + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - #Create directory if it doesn't exist - #- + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - YS_dir='NLDAS_'+rrr_lsm_mod+'0125_M.002/'+YS_yr+'/' - if rrr_lsm_org=='org_no': YS_dir='/' - if not os.path.isdir(rrr_lsm_dir+YS_dir): - os.makedirs(rrr_lsm_dir+YS_dir) - #Update directory name and make sure it exists - #- + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - #Place request if file does not already exist, and check it is ok - #- + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - if os.path.isfile(rrr_lsm_dir+YS_dir+payload['LABEL']): - print(' . Skipping '+payload['LABEL']) - else: - print(' . Downloading '+payload['LABEL']) - r=s.get(url, params=payload) - if not r.ok: - print('ERROR - status code '+str(r.status_code)+ \ - 'returned when downloading '+payload['FILENAME']) - raise SystemExit(22) - YS_name=r.headers['content-disposition'] - YS_name=YS_name.replace('attachment; filename=','') - YS_name=YS_name.replace('"','') - #The file name is extracted directly from requests.get() results - open(rrr_lsm_dir+YS_dir+YS_name, 'wb').write(r.content) - #The file is written on local disk - #- + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - #Increment current datetime - #- + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - rrr_dat_cur=(rrr_dat_cur+datetime.timedelta(days=32)).replace(day=1) - -#------------------------------------------------------------------------------- -#If requesting GLDAS 3-hourly data -#------------------------------------------------------------------------------- -if rrr_lsm_exp=='GLDAS' and rrr_lsm_frq=='3H': - - # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #Initializing URL and payload - # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - print('- Initializing URL and payload') - - url='https://hydro1.gesdisc.eosdis.nasa.gov/daac-bin/OTF/HTTP_services.cgi' - payload={} - payload['FILENAME']='/data/GLDAS/GLDAS_VIC10_3H.2.0/2000/001/' \ - +'GLDAS_VIC10_3H.A20000101.0000.020.nc4' - payload['FORMAT']='bmM0Lw' - payload['BBOX']='-60,-180,90,180' - payload['LABEL']='GLDAS_VIC10_3H.A20000101.0000.020.nc4.SUB.nc4' - payload['SHORTNAME']='GLDAS_VIC10_3H' - payload['SERVICE']='L34RS_LDAS' - payload['VERSION']='1.02' - payload['DATASET_VERSION']='2.0' - payload['VARIABLES']='Qs_acc,Qsb_acc' - - # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #Looping over all files - # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - print('- Looping over all files') - - rrr_dat_cur=rrr_dat_beg - for JS_count in range(IS_count): - #- + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - #Determine current datetime and various date strings - #- + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - YS_yr=rrr_dat_cur.strftime('%Y') - YS_mo=rrr_dat_cur.strftime('%m') - YS_da=rrr_dat_cur.strftime('%d') - YS_hr=rrr_dat_cur.strftime('%H') - YS_dy=rrr_dat_cur.strftime('%j') - #- + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - #Generate file name - #- + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - payload['FILENAME']='/data/GLDAS/GLDAS_'+rrr_lsm_mod+'10_3H.2.0/' \ - +YS_yr+'/'+YS_dy+'/' \ - +'GLDAS_'+rrr_lsm_mod+'10_3H.A'+YS_yr+YS_mo+YS_da \ - +'.'+YS_hr+'00.020.nc4' - payload['LABEL'] ='GLDAS_'+rrr_lsm_mod+'10_3H.A'+YS_yr+YS_mo+YS_da \ - +'.'+YS_hr+'00.020.nc4.SUB.nc4' - #- + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - #Create directory if it doesn't exist - #- + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - YS_dir='GLDAS_'+rrr_lsm_mod+'10_3H.2.0/'+YS_yr+'/'+YS_dy+'/' - if rrr_lsm_org=='org_no': YS_dir='/' - if not os.path.isdir(rrr_lsm_dir+YS_dir): - os.makedirs(rrr_lsm_dir+YS_dir) - #Update directory name and make sure it exists - #- + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - #Place request if file does not already exist, and check it is ok - #- + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - if os.path.isfile(rrr_lsm_dir+YS_dir+payload['LABEL']): - print(' . Skipping '+payload['LABEL']) - else: - print(' . Downloading '+payload['LABEL']) - r=s.get(url, params=payload) - if not r.ok: - print('ERROR - status code '+str(r.status_code)+ \ - 'returned when downloading '+payload['FILENAME']) - raise SystemExit(22) - YS_name=r.headers['content-disposition'] - YS_name=YS_name.replace('attachment; filename=','') - YS_name=YS_name.replace('"','') - #The file name is extracted directly from requests.get() results - open(rrr_lsm_dir+YS_dir+YS_name, 'wb').write(r.content) - #The file is written on local disk - #- + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - #Increment current datetime - #- + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - rrr_dat_cur=rrr_dat_cur+datetime.timedelta(hours=3) - -#------------------------------------------------------------------------------- -#If requesting GLDAS monthly data -#------------------------------------------------------------------------------- -if rrr_lsm_exp=='GLDAS' and rrr_lsm_frq=='M': - - # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #Initializing URL and payload - # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - print('- Initializing URL and payload') - - url='https://hydro1.gesdisc.eosdis.nasa.gov/daac-bin/OTF/HTTP_services.cgi' - payload={} - payload['FILENAME']='/data/GLDAS/GLDAS_VIC10_M.2.0/2000/' \ - +'GLDAS_VIC10_M.A200001.020.nc4' - payload['FORMAT']='bmM0Lw' - payload['BBOX']='-60,-180,90,180' - payload['LABEL']='GLDAS_VIC10_M.A200001.020.nc4.SUB.nc4' - payload['SHORTNAME']='GLDAS_VIC10_M' - payload['SERVICE']='L34RS_LDAS' - payload['VERSION']='1.02' - payload['DATASET_VERSION']='2.0' - payload['VARIABLES']='Qs_acc,Qsb_acc' - - # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #Looping over all files - # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - print('- Looping over all files') - - rrr_dat_cur=rrr_dat_beg - for JS_count in range(IS_count): - #- + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - #Determine current datetime and various date strings - #- + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - YS_yr=rrr_dat_cur.strftime('%Y') - YS_mo=rrr_dat_cur.strftime('%m') - YS_da=rrr_dat_cur.strftime('%d') - YS_hr=rrr_dat_cur.strftime('%H') - YS_dy=rrr_dat_cur.strftime('%j') - #- + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - #Generate file name - #- + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - payload['FILENAME']='/data/GLDAS/GLDAS_'+rrr_lsm_mod+'10_M.2.0/' \ - +YS_yr+'/' \ - +'GLDAS_'+rrr_lsm_mod+'10_M.A'+YS_yr+YS_mo \ - +'.020.nc4' - payload['LABEL'] ='GLDAS_'+rrr_lsm_mod+'10_M.A'+YS_yr+YS_mo \ - +'.020.nc4.SUB.nc4' - #- + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - #Create directory if it doesn't exist - #- + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - YS_dir='GLDAS_'+rrr_lsm_mod+'10_M.2.0/'+YS_yr+'/' - if rrr_lsm_org=='org_no': YS_dir='/' - if not os.path.isdir(rrr_lsm_dir+YS_dir): - os.makedirs(rrr_lsm_dir+YS_dir) - #Update directory name and make sure it exists - #- + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - #Place request if file does not already exist, and check it is ok - #- + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - if os.path.isfile(rrr_lsm_dir+YS_dir+payload['LABEL']): - print(' . Skipping '+payload['LABEL']) - else: - print(' . Downloading '+payload['LABEL']) - r=s.get(url, params=payload) - if not r.ok: - print('ERROR - status code '+str(r.status_code)+ \ - 'returned when downloading '+payload['FILENAME']) - raise SystemExit(22) - YS_name=r.headers['content-disposition'] - YS_name=YS_name.replace('attachment; filename=','') - YS_name=YS_name.replace('"','') - #The file name is extracted directly from requests.get() results - open(rrr_lsm_dir+YS_dir+YS_name, 'wb').write(r.content) - #The file is written on local disk - #- + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - #Increment current datetime - #- + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - rrr_dat_cur=(rrr_dat_cur+datetime.timedelta(days=32)).replace(day=1) - -#------------------------------------------------------------------------------- -#Closing the networking session -#------------------------------------------------------------------------------- -print('- Closing the networking session') - -s.close() - - -#******************************************************************************* -#End -#******************************************************************************* + print("ERROR - Invalid LDAS") + raise SystemExit(22) + + +# ***************************************************************************** +# Earthaccess download function +# ***************************************************************************** +def earthaccess_dwnld(file_count): + retries = 0 # Initialize the number of retries + + earthaccess.login() + + dwnld_files_srch = earthaccess.search_data( + short_name=rrr_lsm_exp + "_" + rrr_lsm_mod + spatial_res + + "_" + rrr_lsm_frq, + cloud_hosted=True, + version=2.0, + bounding_box=boundingBox, + temporal=(rrr_iso_beg, rrr_iso_end), + count=file_count + ) + + while True: # Run the loop indefinitely until condition met + try: + if len(dwnld_files_srch) > 0: + files_dwnld_list = earthaccess.download(dwnld_files_srch, + rrr_lsm_dir) + print("LDAS files downloaded: ", files_dwnld_list) + + # Count the number of properly downloaded files + valid_files = sum(1 for file in files_dwnld_list + if isinstance(file, str) + or issubclass(type(file), str)) + + print("Granules properly downloaded: ", valid_files) + print("Granules to be downloaded: ", file_count) + + if valid_files >= file_count: + # If the number of valid files meets or exceeds the + # required count, exit the loop + break + else: + # Increment the number of retries + retries += 1 + print(f"Retry {retries}...") + else: + print("ERROR - Invalid search parameters for LDAS download") + raise SystemExit(22) + + except requests.exceptions.HTTPError as e: + print(f"HTTPError occurred during download: {e}") + print("Retrying...") + retries += 1 + continue # Retry the download + + except simplejson.errors.JSONDecodeError as e: + print(f"JSON Decode Error occurred during download: {e}") + print("Retrying...") + retries += 1 + continue # Retry the download + + except Exception as e: + print(f"Exception occurred during download: {e}") + print("Retrying...") + retries += 1 + continue # Retry the download + + if valid_files < file_count: + print(f"Failed to download {file_count} files after retries.") + + return len(dwnld_files_srch) + + +# ***************************************************************************** +# Test download for one file +# ***************************************************************************** +print('Checking that service and credentials work for one file') +num_granules = earthaccess_dwnld(1) + + +# ***************************************************************************** +# Bulk download +# ***************************************************************************** +if (num_granules > 0): + print('Downloading all files') + earthaccess_dwnld(IS_count) # 8*31=248, 24x31=744 + + print('Removing surplus variables from netCDF files') + # Iterate through each NetCDF file and delete surplus variables + for nc_file in glob.glob(rrr_lsm_dir + "/*.nc4"): + # keep only Qs_acc and Qsb_acc, remove rest + command = ["ncks", "-v", "Qs_acc,Qsb_acc", nc_file, nc_file, "-O"] + subprocess.run(command, check=True) + + +# ***************************************************************************** +# End +# ***************************************************************************** diff --git a/version.sh b/version.sh index 443f680..b8a5953 100755 --- a/version.sh +++ b/version.sh @@ -18,7 +18,7 @@ if type 'git' > /dev/null; then #git is installed if git rev-parse --git-dir > /dev/null 2>&1; then #this is a git repository - git describe + git describe --tags else #this is not a git repository echo "unknown, NOT a git repository"