Skip to content

refactor: use config constants for retry attempts and time rounding #36

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
SBFRF merged 9 commits into from
Feb 2, 2026
Merged

refactor: use config constants for retry attempts and time rounding #36

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
0c68886
refactor: use config constants for retry attempts and time rounding
SBFRF Feb 2, 2026
4726819
Update tests/test_ncml_indexing_regression.py
SBFRF Feb 2, 2026
91f5582
Update murgtools/getdata/getDataFRF.py
SBFRF Feb 2, 2026
bb60771
Initial plan
Copilot Feb 2, 2026
6a1dadf
Fix all getnc() call sites to unpack 3 return values
Copilot Feb 2, 2026
3ad2a0c
Fix indexRef None check to be explicit
Copilot Feb 2, 2026
4f6fcf4
Merge pull request #41 from SBFRF/copilot/sub-pr-36
SBFRF Feb 2, 2026
ab0355c
Merge branch 'main' into refactor/use-config-constants
SBFRF Feb 2, 2026
80994cc
Fix PR 36 CI test failures: align with PR 37 getnc() return value cha…
SBFRF Feb 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Binary file added frf_wave_and_imagery.png
View file
Open in desktop
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
8 changes: 6 additions & 2 deletions murgtools/config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,8 @@
THREDDS_CRUNCH = 'http://134.164.129.55:8080/thredds/dodsC/cmtb/'

# Survey data URLs
SURVEY_TRANSECTS_URL = 'http://134.164.129.55/thredds/dodsC/FRF/geomorphology/elevationTransects/survey/surveyTransects.ncml'
WAVE_8M_ARRAY_URL = 'http://134.164.129.55/thredds/dodsC/FRF/oceanography/waves/8m-array/2017/FRF-ocean_waves_8m-array_201707.nc'
SURVEY_TRANSECTS_URL = 'http://134.164.129.55/thredds/dodsC/FRF/geomorphology/elevationTransects/survey/surveyTransects.ncml' # noqa: E501
WAVE_8M_ARRAY_URL = 'http://134.164.129.55/thredds/dodsC/FRF/oceanography/waves/8m-array/2017/FRF-ocean_waves_8m-array_201707.nc' # noqa: E501

# =============================================================================
# Imagery URLs
Expand Down Expand Up @@ -62,6 +62,8 @@
DEFAULT_TIMEOUT_SECONDS = 60

# Maximum retry attempts for network operations
# Used in getdata/getDataFRF.py getnc() function when fetching NetCDF files from THREDDS servers
# If a network error occurs, the function will retry up to this many times before giving up
MAX_RETRY_ATTEMPTS = 3

# =============================================================================
Expand All @@ -72,6 +74,8 @@
ARGUS_IMAGE_INTERVAL_MINUTES = 30

# Default time rounding (seconds)
# Used in getdata/getDataFRF.py getnc() function to round timestamps from NetCDF files
# This ensures consistent time intervals when retrieving oceanographic and meteorological data
DEFAULT_TIME_ROUND_SECONDS = 60

# NetCDF time units standard
Expand Down
62 changes: 29 additions & 33 deletions murgtools/getdata/getDataFRF.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ def gettime(allEpoch, epochStart, epochEnd, indexRef=0):
finally:
return idx

def getnc(dataLoc, callingClass, epoch1=0, epoch2=0, dtRound=60, cutrange=100000,**kwargs):
def getnc(dataLoc, callingClass, epoch1=0, epoch2=0, dtRound=None, cutrange=100000,**kwargs):
"""Function grabs the netCDF file interested.

Responsible for drilling down to specific monthly file if applicable to speed things up.
Expand All @@ -59,7 +59,7 @@ def getnc(dataLoc, callingClass, epoch1=0, epoch2=0, dtRound=60, cutrange=100000
dataLoc (str):
THREDDS (str): a key associated with the server location
callingClass (str): which class calls this
dtRound(int): rounding the times returned from the server (Default=60 (s))
dtRound(int): rounding the times returned from the server (Default from config.DEFAULT_TIME_ROUND_SECONDS)

Keyword Args:
start: if given, will parse out to monthly netCDF file (if query is in same month)
Expand All @@ -76,6 +76,10 @@ def getnc(dataLoc, callingClass, epoch1=0, epoch2=0, dtRound=60, cutrange=100000
server = kwargs.get('server', None)
# a list of data sets (just the ncml) that shouldn't drill down to monthly file
doNotDrillList = ['survey', 'integratedBathyTopo']

# Use config default if dtRound not specified
if dtRound is None:
dtRound = config.DEFAULT_TIME_ROUND_SECONDS

# chose which server to select based on IP using centralized config
THREDDSloc, pName = config.get_thredds_server(server=server)
Expand Down Expand Up @@ -116,7 +120,7 @@ def getnc(dataLoc, callingClass, epoch1=0, epoch2=0, dtRound=60, cutrange=100000
ncfileURL = urljoin(THREDDSloc, pName, dataLoc)

# ___________________ go now to open file ___________________________________________
finished, n, maxTries = False, 0, 3 # initializing variables to iterate over
finished, n, maxTries = False, 0, config.MAX_RETRY_ATTEMPTS # initializing variables to iterate over
ncFile, allEpoch = None, None # will return None's when URL doesn't exist
logging.debug(ncfileURL)
indexRef = [0]
Expand Down Expand Up @@ -684,9 +688,9 @@ def getWind(self, gaugenumber=0, collectionlength=10):
valid_gauges = [0, 1, 2, 3, 'derived', 'Derived']
raise InvalidGaugeError(gaugenumber, valid_gauges=valid_gauges)

self.ncfile, self.allEpoch = getnc(dataLoc=self.dataloc, callingClass=self.callingClass,
dtRound=collectionlength * 60, start=self.d1, end=self.d2,
server=self.server)
self.ncfile, self.allEpoch, _ = getnc(dataLoc=self.dataloc, callingClass=self.callingClass,
dtRound=collectionlength * 60, start=self.d1, end=self.d2,
server=self.server)

self.winddataindex = gettime(allEpoch=self.allEpoch, epochStart=self.epochd1,
epochEnd=self.epochd2)
Expand Down Expand Up @@ -849,8 +853,8 @@ def getGaugeWL(self, gaugenumber=5, roundto=1):
self._wlGageURLlookup(gaugenumber)
# parsing out data of interest in time

self.ncfile, self.allEpoch = getnc(dataLoc=self.dataloc, callingClass=self.callingClass,
dtRound=roundto * 60)
self.ncfile, self.allEpoch, _ = getnc(dataLoc=self.dataloc, callingClass=self.callingClass,
dtRound=roundto * 60)

try:
self.wldataindex = gettime(allEpoch=self.allEpoch, epochStart=self.epochd1,
Expand Down Expand Up @@ -939,19 +943,11 @@ def getBathyTransectFromNC(self, profilenumbers=None, method=1, forceReturnAll=F
# acceptableProfileNumbers = [None, ]
self.dataloc = 'geomorphology/elevationTransects/survey/surveyTransects.ncml' # location
# of the gridded surveys
dataReturns = getnc(dataLoc=self.dataloc, callingClass=self.callingClass, server=self.server,
self.ncfile, self.allEpoch, indexRef = getnc(dataLoc=self.dataloc, callingClass=self.callingClass, server=self.server,
dtRound=1 * 60, epoch1=self.epochd1, epoch2=self.epochd2)
if len(dataReturns) == 2:
self.ncfile = dataReturns[0]
self.allEpoch = dataReturns[1]
indexRef=0
elif len(dataReturns) == 3:
self.ncfile = dataReturns[0]
self.allEpoch = dataReturns[1]
indexRef = dataReturns[2]
try:
self.bathydataindex = gettime(allEpoch=self.allEpoch, epochStart=self.epochd1,
epochEnd=self.epochd2, indexRef=indexRef[0])
epochEnd=self.epochd2, indexRef=indexRef[0] if indexRef is not None else 0)
except IOError: # when data are not on CHL thredds
self.bathydataindex = None
# returning None object is convention and must be followed/handled down the line
Expand Down Expand Up @@ -1079,8 +1075,8 @@ def getBathyTransectProfNum(self, method=1):
# acceptableProfileNumbers = [None, ]
self.dataloc = 'geomorphology/elevationTransects/survey/surveyTransects.ncml' # location
# of the gridded surveys
self.ncfile, self.allEpoch = getnc(dataLoc=self.dataloc, callingClass=self.callingClass,
dtRound=1 * 60)
self.ncfile, self.allEpoch, _ = getnc(dataLoc=self.dataloc, callingClass=self.callingClass,
dtRound=1 * 60)

try:
self.bathydataindex = gettime(allEpoch=self.allEpoch, epochStart=self.epochd1,
Expand Down Expand Up @@ -1696,8 +1692,8 @@ def getLidarRunup(self, removeMasked=True):

"""
self.dataloc = 'oceanography/waves/lidarWaveRunup/lidarWaveRunup.ncml'
self.ncfile, self.allEpoch = getnc(dataLoc=self.dataloc, callingClass=self.callingClass,
dtRound=1 * 60)
self.ncfile, self.allEpoch, _ = getnc(dataLoc=self.dataloc, callingClass=self.callingClass,
dtRound=1 * 60)
self.lidarIndex = gettime(allEpoch=self.allEpoch, epochStart=self.epochd1,
epochEnd=self.epochd2)

Expand Down Expand Up @@ -1905,8 +1901,8 @@ def getALT(self, gaugeName=None, removeMasked=True):
else:
raise NotImplementedError('Please use one of the following keys\n'.format(gauge_list))

self.ncfile, self.allEpoch = getnc(dataLoc=self.dataloc, callingClass=self.callingClass,
dtRound=1 * 60)
self.ncfile, self.allEpoch, _ = getnc(dataLoc=self.dataloc, callingClass=self.callingClass,
dtRound=1 * 60)
altdataindex = gettime(allEpoch=self.allEpoch, epochStart=self.epochd1,
epochEnd=self.epochd2)

Expand Down Expand Up @@ -2015,8 +2011,8 @@ def getLidarWaveProf(self, removeMasked=True):

"""
self.dataloc = 'oceanography/waves/lidarHydrodynamics/lidarHydrodynamics.ncml'
self.ncfile, self.allEpoch = getnc(dataLoc=self.dataloc, callingClass=self.callingClass,
dtRound=1 * 60)
self.ncfile, self.allEpoch, _ = getnc(dataLoc=self.dataloc, callingClass=self.callingClass,
dtRound=1 * 60)
self.lidarIndex = gettime(allEpoch=self.allEpoch, epochStart=self.epochd1,
epochEnd=self.epochd2)
if np.size(self.lidarIndex) > 0 and self.lidarIndex is not None:
Expand Down Expand Up @@ -2250,8 +2246,8 @@ def getBathyGridcBathy(self, **kwargs):
fillValue = -999 # assumed fill value from the rest of the files taken as less than or
# equal to
self.dataloc = 'projects/bathyduck/data/cbathy_old/cbathy.ncml'
self.ncfile, self.allEpoch = getnc(dataLoc=self.dataloc, callingClass=self.callingClass,
dtRound=30 * 60)
self.ncfile, self.allEpoch, _ = getnc(dataLoc=self.dataloc, callingClass=self.callingClass,
dtRound=30 * 60)
self.cbidx = gettime(allEpoch=self.allEpoch, epochStart=self.epochd1, epochEnd=self.epochd2)

self.cbtime = nc.num2date(self.allEpoch[self.cbidx], 'seconds since 1970-01-01',
Expand Down Expand Up @@ -2374,8 +2370,8 @@ def getArgus(self, type, **kwargs):
self.dataloc = "projects/bathyduck/data/argus/timex/timex.ncml"

################ go get data index
self.ncfile, self.allEpoch = getnc(dataLoc=self.dataloc, callingClass=self.callingClass,
dtRound=1 * 60)
self.ncfile, self.allEpoch, _ = getnc(dataLoc=self.dataloc, callingClass=self.callingClass,
dtRound=1 * 60)
self.idxArgus = gettime(allEpoch=self.allEpoch, epochStart=self.epochd1,
epochEnd=self.epochd2)

Expand Down Expand Up @@ -2572,8 +2568,8 @@ def getGridCMS(self, method):

"""
self.dataloc = 'grids/CMSwave_v1/CMSwave_v1.ncml'
self.ncfile, self.allEpoch = getnc(dataLoc=self.dataloc, callingClass=self.callingClass,
dtRound=1 * 60)
self.ncfile, self.allEpoch, _ = getnc(dataLoc=self.dataloc, callingClass=self.callingClass,
dtRound=1 * 60)
try:
self.bathydataindex = gettime(allEpoch=self.allEpoch, epochStart=self.epochd1,
epochEnd=self.epochd2) # getting the index of the grid
Expand Down Expand Up @@ -3314,7 +3310,7 @@ def getCSHOREOutput(self, prefix):

"""
dataLoc = 'morphModels/CSHORE/{0}/{0}.ncml'.format(prefix)
ncfile, allEpoch = getnc(dataLoc, self.THREDDS, self.callingClass)
ncfile, allEpoch, _ = getnc(dataLoc, self.THREDDS, self.callingClass)
dataIndex = gettime(allEpoch, epochStart=self.epochd1, epochEnd=self.epochd2)
if dataIndex is None:
print(('There\'s no data in time period ' + self.start.strftime('%Y-%m-%dT%H%M%SZ') +
Expand Down
4 changes: 2 additions & 2 deletions tests/test_getDataFRF.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -461,8 +461,8 @@ def test_getWind_returns_expected_keys(self, mock_num2date, mock_date2num, mock_
mock_ds.title = 'FRF Derived Wind'
mock_ds.geospatial_vertical_max = 19.0

# getnc returns (ncFile, allEpoch) - 2 values for getWind (buggy code, should be 3)
mock_getnc.return_value = (mock_ds, time_values)
# getnc returns (ncFile, allEpoch, indexRef) - 3 values
mock_getnc.return_value = (mock_ds, time_values, None)

# gettime returns indices
mock_gettime.return_value = np.arange(72)
Expand Down
Loading