diff --git a/.gitignore b/.gitignore index 44c81c6..9dcfc55 100755 --- a/.gitignore +++ b/.gitignore @@ -111,3 +111,7 @@ Thumbs.db *.nc *.pickle *.pkl + +# Local development configs +.claude/ +.mcp-debug-tools/ diff --git a/examples/combined_asv_wave_conditions_2023.png b/examples/combined_asv_wave_conditions_2023.png new file mode 100644 index 0000000..e844ba2 Binary files /dev/null and b/examples/combined_asv_wave_conditions_2023.png differ diff --git a/murgtools/__init__.py b/murgtools/__init__.py index 53b56d0..d437578 100644 --- a/murgtools/__init__.py +++ b/murgtools/__init__.py @@ -4,6 +4,7 @@ from .getdata import forecastData, getSatelliteImagery, alt_PlotData from .getdata import getArgusImagery, threadGetArgusImagery from .plotting import conditions_plot, bin_data +from .cache import DataCache, get_cache, enable_cache, disable_cache, clear_cache __all__ = [ "getObs", @@ -18,4 +19,10 @@ "alt_PlotData", "conditions_plot", "bin_data", + # Cache functions + "DataCache", + "get_cache", + "enable_cache", + "disable_cache", + "clear_cache", ] diff --git a/murgtools/cache.py b/murgtools/cache.py new file mode 100644 index 0000000..ab4b7dc --- /dev/null +++ b/murgtools/cache.py @@ -0,0 +1,601 @@ +"""Data caching module for murgtools. + +This module provides disk-based caching for retrieved data to avoid repeated +network requests. Caching is OFF by default and must be explicitly enabled. + +Features: + - Disk-based persistence at configurable location (default: /data/getdata) + - Configurable TTL (default: 6 months / 180 days) + - Thread-safe operations + - Force refresh capability + - Automatic cache invalidation for stale data + +Usage: + from murgtools.cache import DataCache + + # Create cache instance (caching disabled by default) + cache = DataCache() + + # Enable caching + cache = DataCache(enabled=True) + + # Custom configuration + cache = DataCache( + enabled=True, + cache_dir='/custom/path', + ttl_days=90, # 3 months + ) + + # Force refresh even if cache is valid + data = cache.get(key, fetch_func, force_refresh=True) + +Environment Variables: + MURGTOOLS_CACHE_ENABLED: Set to '1' or 'true' to enable caching + MURGTOOLS_CACHE_DIR: Override default cache directory + MURGTOOLS_CACHE_TTL_DAYS: Override default TTL in days +""" + +import hashlib +import json +import logging +import os +import pickle +import threading +import time +from datetime import datetime +from pathlib import Path +from typing import Any, Callable, Dict, Optional, Tuple, Union + +import numpy as np + +from . import config + + +class NumpyJSONEncoder(json.JSONEncoder): + """JSON encoder that handles numpy types and other non-JSON-serializable objects. + + Falls back to string representation for unknown types to ensure cache + operations don't fail unexpectedly. + """ + + def default(self, obj): + if isinstance(obj, np.integer): + return int(obj) + if isinstance(obj, np.floating): + return float(obj) + if isinstance(obj, np.ndarray): + return obj.tolist() + # Handle datetime objects + if hasattr(obj, 'isoformat'): + return obj.isoformat() + # Handle Path objects + if hasattr(obj, '__fspath__'): + return str(obj) + # Fallback to string for unknown types (graceful degradation) + try: + return super().default(obj) + except TypeError: + return str(obj) + +logger = logging.getLogger(__name__) + + +class CacheError(Exception): + """Base exception for cache-related errors.""" + pass + + +class CacheMetadata: + """Metadata for a cached entry.""" + + def __init__(self, created_at: float, data_source: str, time_range: Optional[Tuple] = None, + extra_info: Optional[Dict] = None): + """Initialize cache metadata. + + Args: + created_at: Unix timestamp when cache was created. + data_source: Identifier for the data source (e.g., URL, gauge name). + time_range: Optional tuple of (start_epoch, end_epoch) for time-series data. + extra_info: Optional dictionary with additional metadata. + """ + self.created_at = created_at + self.data_source = data_source + self.time_range = time_range + self.extra_info = extra_info or {} + + def to_dict(self) -> Dict: + """Convert metadata to dictionary for serialization.""" + return { + 'created_at': self.created_at, + 'data_source': self.data_source, + 'time_range': self.time_range, + 'extra_info': self.extra_info, + 'version': '1.0', + } + + @classmethod + def from_dict(cls, data: Dict) -> 'CacheMetadata': + """Create metadata from dictionary.""" + return cls( + created_at=data['created_at'], + data_source=data['data_source'], + time_range=data.get('time_range'), + extra_info=data.get('extra_info', {}), + ) + + def is_expired(self, ttl_days: int) -> bool: + """Check if the cache entry has expired. + + Args: + ttl_days: Time-to-live in days. + + Returns: + True if expired, False otherwise. + """ + if ttl_days <= 0: + return False # TTL of 0 means never expire + + expiry_time = self.created_at + (ttl_days * 24 * 60 * 60) + return time.time() > expiry_time + + def age_days(self) -> float: + """Get the age of this cache entry in days.""" + return (time.time() - self.created_at) / (24 * 60 * 60) + + +class DataCache: + """Thread-safe disk-based data cache. + + Caching is OFF by default. Enable it explicitly via the `enabled` parameter + or by setting the MURGTOOLS_CACHE_ENABLED environment variable. + """ + + def __init__(self, enabled: Optional[bool] = None, cache_dir: Optional[str] = None, + ttl_days: Optional[int] = None): + """Initialize the data cache. + + Args: + enabled: Whether caching is enabled. If None, checks environment + variable MURGTOOLS_CACHE_ENABLED, otherwise defaults to False. + cache_dir: Directory to store cache files. If None, checks environment + variable MURGTOOLS_CACHE_DIR, otherwise uses config.DEFAULT_CACHE_DIR. + ttl_days: Time-to-live in days. If None, checks environment variable + MURGTOOLS_CACHE_TTL_DAYS, otherwise uses config.DEFAULT_CACHE_TTL_DAYS. + Set to 0 for no expiration. + """ + self._lock = threading.RLock() + + # Determine if caching is enabled + if enabled is not None: + self._enabled = enabled + else: + env_enabled = os.environ.get(config.ENV_CACHE_ENABLED, '').lower() + self._enabled = env_enabled in ('1', 'true', 'yes', 'on') + + # Determine cache directory + if cache_dir is not None: + self._cache_dir = Path(cache_dir) + else: + env_dir = os.environ.get(config.ENV_CACHE_DIR) + self._cache_dir = Path(env_dir) if env_dir else Path(config.DEFAULT_CACHE_DIR) + + # Determine TTL (use setter for validation) + if ttl_days is not None: + self.ttl_days = ttl_days # Use setter for validation + else: + env_ttl = os.environ.get(config.ENV_CACHE_TTL_DAYS) + if env_ttl: + try: + self.ttl_days = int(env_ttl) # Use setter for validation + except ValueError: + logger.warning(f"Invalid TTL value '{env_ttl}', using default") + self._ttl_days = config.DEFAULT_CACHE_TTL_DAYS + else: + self._ttl_days = config.DEFAULT_CACHE_TTL_DAYS + + # Create cache directory if enabled and directory doesn't exist + if self._enabled: + self._ensure_cache_dir() + + @property + def enabled(self) -> bool: + """Whether caching is enabled.""" + return self._enabled + + @enabled.setter + def enabled(self, value: bool): + """Enable or disable caching.""" + with self._lock: + self._enabled = value + if value: + self._ensure_cache_dir() + + @property + def cache_dir(self) -> Path: + """The cache directory path.""" + return self._cache_dir + + @cache_dir.setter + def cache_dir(self, value: Union[str, Path]): + """Set the cache directory.""" + with self._lock: + self._cache_dir = Path(value) + if self._enabled: + self._ensure_cache_dir() + + @property + def ttl_days(self) -> int: + """Time-to-live in days.""" + return self._ttl_days + + @ttl_days.setter + def ttl_days(self, value: int): + """Set the TTL in days.""" + if value < 0: + raise ValueError("TTL must be non-negative") + self._ttl_days = value + + def _ensure_cache_dir(self): + """Create cache directory if it doesn't exist.""" + try: + self._cache_dir.mkdir(parents=True, exist_ok=True) + except PermissionError as e: + logger.warning(f"Cannot create cache directory {self._cache_dir}: {e}") + logger.warning("Caching will be disabled") + self._enabled = False + except OSError as e: + logger.warning(f"Error creating cache directory {self._cache_dir}: {e}") + self._enabled = False + + def _generate_cache_key(self, data_source: str, time_range: Optional[Tuple] = None, + extra_params: Optional[Dict] = None) -> str: + """Generate a unique cache key based on request parameters. + + Args: + data_source: Identifier for the data source. + time_range: Optional tuple of (start_epoch, end_epoch). + extra_params: Optional additional parameters that affect the data. + + Returns: + A hex string suitable for use as a filename. + """ + key_parts = [data_source] + + if time_range: + key_parts.append(f"t{time_range[0]}-{time_range[1]}") + + if extra_params: + # Sort keys for deterministic ordering, use custom encoder for numpy types + sorted_params = sorted(extra_params.items()) + key_parts.append(json.dumps(sorted_params, sort_keys=True, cls=NumpyJSONEncoder)) + + key_string = '|'.join(str(p) for p in key_parts) + return hashlib.sha256(key_string.encode()).hexdigest()[:32] + + def _get_cache_paths(self, cache_key: str) -> Tuple[Path, Path]: + """Get paths for data and metadata files. + + Args: + cache_key: The cache key. + + Returns: + Tuple of (data_path, metadata_path). + """ + data_path = self._cache_dir / f"{cache_key}.pkl" + meta_path = self._cache_dir / f"{cache_key}.meta.json" + return data_path, meta_path + + def get(self, data_source: str, fetch_func: Callable[[], Any], + time_range: Optional[Tuple] = None, extra_params: Optional[Dict] = None, + force_refresh: bool = False) -> Any: + """Get data from cache or fetch it. + + Args: + data_source: Identifier for the data source (e.g., URL, gauge name). + fetch_func: Callable that fetches the data if not cached. + time_range: Optional tuple of (start_epoch, end_epoch) for time-series data. + extra_params: Optional additional parameters that affect the data. + force_refresh: If True, bypass cache and fetch fresh data. + + Returns: + The requested data (from cache or freshly fetched). + """ + # If caching is disabled, just fetch + if not self._enabled: + return fetch_func() + + cache_key = self._generate_cache_key(data_source, time_range, extra_params) + + # Check cache first (with lock) + with self._lock: + if not force_refresh: + cached_data = self._read_cache(cache_key) + if cached_data is not None: + return cached_data + + # Fetch fresh data WITHOUT holding the lock (allows concurrent fetches) + data = fetch_func() + + # Store in cache (with lock) + if data is not None: + with self._lock: + self._write_cache(cache_key, data, data_source, time_range, extra_params) + + return data + + def _read_cache(self, cache_key: str) -> Optional[Any]: + """Read data from cache if valid. + + Args: + cache_key: The cache key. + + Returns: + Cached data if valid, None otherwise. + """ + data_path, meta_path = self._get_cache_paths(cache_key) + + # Check if cache files exist + if not data_path.exists() or not meta_path.exists(): + return None + + try: + # Read and validate metadata + with open(meta_path, 'r') as f: + meta_dict = json.load(f) + metadata = CacheMetadata.from_dict(meta_dict) + + # Check if expired + if metadata.is_expired(self._ttl_days): + logger.debug(f"Cache expired for {cache_key} (age: {metadata.age_days():.1f} days)") + self._remove_cache(cache_key) + return None + + # Read data + with open(data_path, 'rb') as f: + data = pickle.load(f) + + logger.debug(f"Cache hit for {cache_key} (age: {metadata.age_days():.1f} days)") + return data + + except (json.JSONDecodeError, pickle.UnpicklingError, KeyError, OSError) as e: + logger.warning(f"Error reading cache {cache_key}: {e}") + self._remove_cache(cache_key) + return None + + def _write_cache(self, cache_key: str, data: Any, data_source: str, + time_range: Optional[Tuple], extra_params: Optional[Dict]): + """Write data to cache. + + Args: + cache_key: The cache key. + data: The data to cache. + data_source: Identifier for the data source. + time_range: Optional time range tuple. + extra_params: Optional extra parameters. + """ + data_path, meta_path = self._get_cache_paths(cache_key) + + try: + # Create metadata + metadata = CacheMetadata( + created_at=time.time(), + data_source=data_source, + time_range=time_range, + extra_info=extra_params, + ) + + # Write metadata first (smaller, validates we can write) + with open(meta_path, 'w') as f: + json.dump(metadata.to_dict(), f, indent=2, cls=NumpyJSONEncoder) + + # Write data + with open(data_path, 'wb') as f: + pickle.dump(data, f, protocol=pickle.HIGHEST_PROTOCOL) + + logger.debug(f"Cached data for {cache_key}") + + except (OSError, pickle.PicklingError) as e: + logger.warning(f"Error writing cache {cache_key}: {e}") + # Clean up partial writes + self._remove_cache(cache_key) + + def _remove_cache(self, cache_key: str): + """Remove cache files for a key. + + Args: + cache_key: The cache key. + """ + data_path, meta_path = self._get_cache_paths(cache_key) + + for path in (data_path, meta_path): + try: + if path.exists(): + path.unlink() + except OSError as e: + logger.warning(f"Error removing cache file {path}: {e}") + + def clear(self, older_than_days: Optional[int] = None): + """Clear cache entries. + + Args: + older_than_days: If specified, only clear entries older than this. + If None, clear all entries. + """ + if not self._cache_dir.exists(): + return + + with self._lock: + cleared = 0 + for meta_path in self._cache_dir.glob("*.meta.json"): + try: + with open(meta_path, 'r') as f: + meta_dict = json.load(f) + metadata = CacheMetadata.from_dict(meta_dict) + + if older_than_days is None or metadata.age_days() > older_than_days: + cache_key = meta_path.stem.replace('.meta', '') + self._remove_cache(cache_key) + cleared += 1 + + except (json.JSONDecodeError, KeyError, OSError) as e: + logger.warning(f"Error reading metadata {meta_path}: {e}") + # Remove corrupted entry + cache_key = meta_path.stem.replace('.meta', '') + self._remove_cache(cache_key) + cleared += 1 + + logger.info(f"Cleared {cleared} cache entries") + + def get_stats(self) -> Dict: + """Get cache statistics. + + Returns: + Dictionary with cache statistics. + """ + if not self._cache_dir.exists(): + return { + 'enabled': self._enabled, + 'cache_dir': str(self._cache_dir), + 'ttl_days': self._ttl_days, + 'entry_count': 0, + 'total_size_mb': 0, + } + + entry_count = 0 + total_size = 0 + oldest_entry = None + newest_entry = None + + for data_path in self._cache_dir.glob("*.pkl"): + entry_count += 1 + total_size += data_path.stat().st_size + + meta_path = data_path.with_suffix('.meta.json') + if meta_path.exists(): + try: + with open(meta_path, 'r') as f: + meta_dict = json.load(f) + created = meta_dict.get('created_at', 0) + if oldest_entry is None or created < oldest_entry: + oldest_entry = created + if newest_entry is None or created > newest_entry: + newest_entry = created + except (json.JSONDecodeError, OSError): + pass + + return { + 'enabled': self._enabled, + 'cache_dir': str(self._cache_dir), + 'ttl_days': self._ttl_days, + 'entry_count': entry_count, + 'total_size_mb': round(total_size / (1024 * 1024), 2), + 'oldest_entry': datetime.fromtimestamp(oldest_entry).isoformat() if oldest_entry else None, + 'newest_entry': datetime.fromtimestamp(newest_entry).isoformat() if newest_entry else None, + } + + def is_cached(self, data_source: str, time_range: Optional[Tuple] = None, + extra_params: Optional[Dict] = None) -> bool: + """Check if data is cached and valid. + + Args: + data_source: Identifier for the data source. + time_range: Optional time range tuple. + extra_params: Optional extra parameters. + + Returns: + True if valid cache exists, False otherwise. + """ + if not self._enabled: + return False + + cache_key = self._generate_cache_key(data_source, time_range, extra_params) + data_path, meta_path = self._get_cache_paths(cache_key) + + if not data_path.exists() or not meta_path.exists(): + return False + + try: + with open(meta_path, 'r') as f: + meta_dict = json.load(f) + metadata = CacheMetadata.from_dict(meta_dict) + return not metadata.is_expired(self._ttl_days) + except (json.JSONDecodeError, KeyError, OSError): + return False + + def invalidate(self, data_source: str, time_range: Optional[Tuple] = None, + extra_params: Optional[Dict] = None): + """Invalidate (remove) a specific cache entry. + + Args: + data_source: Identifier for the data source. + time_range: Optional time range tuple. + extra_params: Optional extra parameters. + """ + cache_key = self._generate_cache_key(data_source, time_range, extra_params) + with self._lock: + self._remove_cache(cache_key) + + +# Global cache instance (disabled by default) +_global_cache: Optional[DataCache] = None +_global_cache_lock = threading.Lock() + + +def get_cache(enabled: Optional[bool] = None, cache_dir: Optional[str] = None, + ttl_days: Optional[int] = None) -> DataCache: + """Get or create the global cache instance. + + Args: + enabled: Whether to enable caching. If None on first call, defaults to False. + cache_dir: Cache directory. If None, uses default or environment variable. + ttl_days: TTL in days. If None, uses default or environment variable. + + Returns: + The global DataCache instance. + """ + global _global_cache + + with _global_cache_lock: + if _global_cache is None: + _global_cache = DataCache(enabled=enabled, cache_dir=cache_dir, ttl_days=ttl_days) + else: + # Update settings if provided + if enabled is not None: + _global_cache.enabled = enabled + if cache_dir is not None: + _global_cache.cache_dir = cache_dir + if ttl_days is not None: + _global_cache.ttl_days = ttl_days + + return _global_cache + + +def enable_cache(cache_dir: Optional[str] = None, ttl_days: Optional[int] = None) -> DataCache: + """Enable the global cache with optional configuration. + + Convenience function to enable caching with one call. + + Args: + cache_dir: Cache directory. If None, uses default. + ttl_days: TTL in days. If None, uses default (180 days). + + Returns: + The enabled DataCache instance. + """ + return get_cache(enabled=True, cache_dir=cache_dir, ttl_days=ttl_days) + + +def disable_cache(): + """Disable the global cache.""" + cache = get_cache() + cache.enabled = False + + +def clear_cache(older_than_days: Optional[int] = None): + """Clear the global cache. + + Args: + older_than_days: If specified, only clear entries older than this. + """ + cache = get_cache() + cache.clear(older_than_days=older_than_days) diff --git a/murgtools/config.py b/murgtools/config.py index cd79afd..c836444 100644 --- a/murgtools/config.py +++ b/murgtools/config.py @@ -6,6 +6,7 @@ """ import socket +import threading # ============================================================================= # THREDDS Server URLs @@ -87,33 +88,113 @@ ARGUS_IMAGE_TYPES = ('timex', 'var', 'snap', 'bright', 'dark') +# ============================================================================= +# Data Cache Configuration +# ============================================================================= +# Caching is OFF by default. Users must explicitly enable it. +# Cache stores retrieved data to disk to avoid repeated network requests. + +# Default cache directory (user can override) +DEFAULT_CACHE_DIR = '/data/getdata' + +# Default cache TTL in days (6 months = ~180 days) +DEFAULT_CACHE_TTL_DAYS = 180 + +# Environment variable names for configuration overrides +ENV_CACHE_ENABLED = 'MURGTOOLS_CACHE_ENABLED' +ENV_CACHE_DIR = 'MURGTOOLS_CACHE_DIR' +ENV_CACHE_TTL_DAYS = 'MURGTOOLS_CACHE_TTL_DAYS' + # ============================================================================= # Helper Functions # ============================================================================= +# ============================================================================= +# Server Detection Cache (thread-safe) +# ============================================================================= +# Cache is computed once at first use to avoid repeated socket operations. +# Both FRF and CHL servers are trusted government endpoints - server selection +# only affects performance (local vs remote), not security. + +# Use RLock (reentrant lock) because get_thredds_server calls _get_cached_ip +# while holding the lock +_cache_lock = threading.RLock() +_cached_ip_address = None +_cached_server_result = None + + +def _get_cached_ip(): + """Get the cached IP address, computing it once if needed (thread-safe). + + Returns: + str: The machine's IP address, or empty string if detection failed. + """ + global _cached_ip_address + + with _cache_lock: + if _cached_ip_address is not None: + return _cached_ip_address + + try: + _cached_ip_address = socket.gethostbyname(socket.gethostname()) + except OSError: + # Covers socket.error, socket.gaierror, socket.herror in Python 3 + _cached_ip_address = '' + + return _cached_ip_address + def get_thredds_server(server=None, ip_address=None): - """Select appropriate THREDDS server based on network location. + """Select appropriate THREDDS server based on network location (thread-safe). + + Results are cached at module level for auto-detection (when server=None + and ip_address=None) to avoid repeated socket operations. Args: server (str, optional): Force server selection. 'FRF' for local, 'CHL' for public. If None, auto-detect based on IP. ip_address (str, optional): IP address to check. If None, - uses current machine's IP. + uses cached machine IP. Returns: tuple: (server_url, server_prefix) where server_prefix is 'FRF' or 'frf' for use in constructing data paths. """ + global _cached_server_result + + # Fast path: return cached result for default auto-detection case + if server is None and ip_address is None: + # Check without lock first + if _cached_server_result is not None: + return _cached_server_result + + with _cache_lock: + # Double-check after acquiring lock + if _cached_server_result is not None: + return _cached_server_result + + # Compute and cache the result + detected_ip = _get_cached_ip() + on_frf_network = detected_ip.startswith(FRF_IP_PREFIXES) + + if on_frf_network: + _cached_server_result = (THREDDS_FRF_LOCAL, 'FRF') + else: + _cached_server_result = (THREDDS_CHL_PUBLIC, 'frf') + + return _cached_server_result + + # Non-cached path: explicit server or ip_address provided if ip_address is None: - try: - ip_address = socket.gethostbyname(socket.gethostname()) - except socket.error: - ip_address = '' + ip_address = _get_cached_ip() - is_frf_network = ip_address.startswith(FRF_IP_PREFIXES) + # Validate ip_address is a string to prevent type confusion + if not isinstance(ip_address, str): + ip_address = '' - if server == 'FRF' or (server is None and is_frf_network): + on_frf_network = ip_address.startswith(FRF_IP_PREFIXES) + + if server == 'FRF' or (server is None and on_frf_network): return THREDDS_FRF_LOCAL, 'FRF' else: return THREDDS_CHL_PUBLIC, 'frf' @@ -124,15 +205,28 @@ def is_frf_network(ip_address=None): Args: ip_address (str, optional): IP address to check. If None, - uses current machine's IP. + uses cached machine IP. Returns: bool: True if on FRF network, False otherwise. """ if ip_address is None: - try: - ip_address = socket.gethostbyname(socket.gethostname()) - except socket.error: - return False + ip_address = _get_cached_ip() + + # Validate ip_address is a string to prevent type confusion + if not isinstance(ip_address, str): + return False return ip_address.startswith(FRF_IP_PREFIXES) + + +def clear_server_cache(): + """Clear the cached server detection results (thread-safe). + + Useful for testing or if network configuration changes during runtime. + """ + global _cached_ip_address, _cached_server_result + + with _cache_lock: + _cached_ip_address = None + _cached_server_result = None diff --git a/murgtools/getdata/getDataFRF.py b/murgtools/getdata/getDataFRF.py index b24b004..002cbe8 100755 --- a/murgtools/getdata/getDataFRF.py +++ b/murgtools/getdata/getDataFRF.py @@ -23,6 +23,7 @@ from murgtools.utils import geoprocess as gp, sblib as sb from murgtools import config +from murgtools.cache import DataCache from murgtools.exceptions import InvalidGaugeError def gettime(allEpoch, epochStart, epochEnd, indexRef=0): @@ -216,8 +217,27 @@ def removeDuplicatesFromDictionary(inputDict): class getObs: """Class focused on retrieving observational data.""" - def __init__(self, d1, d2, **kwargs): - """Data are returned in self.dataindex are inclusive at start, exclusive at end.""" + def __init__(self, d1, d2, use_cache=False, cache_dir=None, cache_ttl_days=None, **kwargs): + """Initialize observation data retriever. + + Data are returned in self.dataindex are inclusive at start, exclusive at end. + + Args: + d1 (datetime): Start date for data retrieval. + d2 (datetime): End date for data retrieval. + use_cache (bool): Enable disk caching of retrieved data. Default False. + When enabled, data is cached to disk and reused on subsequent calls + with the same parameters. Cache expires after cache_ttl_days. + cache_dir (str, optional): Directory for cache files. + Default: /data/getdata (or MURGTOOLS_CACHE_DIR env var). + cache_ttl_days (int, optional): Cache time-to-live in days. + Default: 180 days / 6 months (or MURGTOOLS_CACHE_TTL_DAYS env var). + + Keyword Args: + server (str): Force server selection ('FRF' or 'CHL'). + force_refresh (bool): If True, bypass cache and fetch fresh data. + Only applies when use_cache=True. + """ # this is active wave gauge list for looping through as needed self.waveGaugeList = ['waverider-26m', 'waverider-17m', 'waverider-17m-1d', 'waverider-20m-1d', 'awac-11m', 'awac-jpier-11m', '8m-array', 'awac-6m', 'awac-4.5m', 'adop-3.5m', @@ -243,6 +263,14 @@ def __init__(self, d1, d2, **kwargs): self.chlDataLoc = config.THREDDS_CHL_ALT self.server = kwargs.get('server', None) self._comp_time() + + # Cache configuration + self._use_cache = use_cache + self._force_refresh = kwargs.get('force_refresh', False) + if use_cache: + self._cache = DataCache(enabled=True, cache_dir=cache_dir, ttl_days=cache_ttl_days) + else: + self._cache = None # assert type(self.d2) == DT.datetime, 'd1 need to be in python "Datetime" data types' # assert type(self.d1) == DT.datetime, 'd2 need to be in python "Datetime" data types' @@ -270,7 +298,42 @@ def _roundtime(self, dt=None, roundto=60): rounding = (seconds + roundto / 2) // roundto * roundto return dt + DT.timedelta(0, rounding - seconds, -dt.microsecond) - def getWaveData(self, gaugenumber=0, roundto=30, removeBadDataFlag=4, **kwargs): + def _get_with_cache(self, data_type, fetch_func, extra_params=None, force_refresh=False): + """Retrieve data with optional caching. + + Args: + data_type (str): Type of data being retrieved (e.g., 'wave', 'wind', 'bathy'). + fetch_func (callable): Function that fetches the data from the server. + extra_params (dict, optional): Additional parameters that affect the data + (e.g., gauge number, variable names). Used in cache key generation. + force_refresh (bool): If True, bypass cache and fetch fresh data. + + Returns: + The requested data dictionary. + """ + if not self._use_cache or self._cache is None: + return fetch_func() + + # Build cache key components + data_source = f"getObs.{data_type}" + time_range = (self.epochd1, self.epochd2) + + # Include resolved server URL in extra_params for cache key + # Use FRFdataloc (the actual server being used) rather than self.server + # (which may be None for auto-detect) + cache_params = {'server_url': self.FRFdataloc} + if extra_params: + cache_params.update(extra_params) + + return self._cache.get( + data_source=data_source, + fetch_func=fetch_func, + time_range=time_range, + extra_params=cache_params, + force_refresh=force_refresh or self._force_refresh, + ) + + def getWaveData(self, gaugenumber=0, roundto=30, removeBadDataFlag=4, force_refresh=False, **kwargs): """This function pulls down the data from the thredds server and puts the data into a dictionary. TODO: Set optional date input from function arguments to change self.start self.end @@ -283,6 +346,8 @@ def getWaveData(self, gaugenumber=0, roundto=30, removeBadDataFlag=4, **kwargs): removeBadDataFlag (int): this will remove data with a directional flag of 3/4 signaling questionable or failed directional spectra (default = 4, remove failed (directional) spectral data time periods) valid values: [3, 4, False] False will not remove any data + force_refresh (bool): If True and caching is enabled, bypass cache and fetch fresh data. + Default False. Keyword Args: "returnAB" (bool): if this is True function will return a's and b's for time period (default=False) @@ -317,6 +382,23 @@ def getWaveData(self, gaugenumber=0, roundto=30, removeBadDataFlag=4, **kwargs): """ returnAB = kwargs.get('returnAB', False) spec = kwargs.get('spec', False) + + # Use caching if enabled + cache_params = { + 'gaugenumber': gaugenumber, + 'roundto': roundto, + 'removeBadDataFlag': removeBadDataFlag, + 'returnAB': returnAB, + 'spec': spec, + } + + def fetch_data(): + return self._getWaveData_impl(gaugenumber, roundto, removeBadDataFlag, returnAB, spec) + + return self._get_with_cache('wave', fetch_data, cache_params, force_refresh) + + def _getWaveData_impl(self, gaugenumber, roundto, removeBadDataFlag, returnAB, spec): + """Internal implementation of getWaveData (without caching).""" # Making gauges flexible self._waveGaugeURLlookup(gaugenumber) # parsing out data of interest in time @@ -483,7 +565,7 @@ def getWaveSpec(self, gaugenumber=0, roundto=30, removeBadDataFlag=4, **kwargs): # else: return self.getWaveData(gaugenumber, roundto, removeBadDataFlag, returnAB=returnAB, spec=True) - def getCurrents(self, gaugenumber=5, roundto=1): + def getCurrents(self, gaugenumber=5, roundto=1, force_refresh=False): """This function pulls down the currents data from the Thredds Server. Args: @@ -507,6 +589,8 @@ def getCurrents(self, gaugenumber=5, roundto=1): increments data is rounded to the nearst [roundto] (default 1 min) + force_refresh (bool): If True and caching is enabled, bypass cache and fetch fresh data. + Returns: dict, None if error is encoutered 'name' (str): gauge name @@ -537,6 +621,15 @@ def getCurrents(self, gaugenumber=5, roundto=1): 'meanP' (array): mean pressure """ + cache_params = {'gaugenumber': gaugenumber, 'roundto': roundto} + + def fetch_data(): + return self._getCurrents_impl(gaugenumber, roundto) + + return self._get_with_cache('currents', fetch_data, cache_params, force_refresh) + + def _getCurrents_impl(self, gaugenumber, roundto): + """Internal implementation of getCurrents (without caching).""" valid_gauges = [2, 3, 4, 5, 6, 'awac-11m', 'awac-8m', 'awac-6m', 'awac-4.5m', 'awac-5m', 'adop-3.5m', 'awac-jpier-11m', 'awac-jpier', 'jpier-11m', 'sig769-300', '769-300', @@ -620,7 +713,7 @@ def getCurrents(self, gaugenumber=5, roundto=1): self.curpacket = None return self.curpacket - def getWind(self, gaugenumber=0, collectionlength=10): + def getWind(self, gaugenumber=0, collectionlength=10, force_refresh=False): """This function retrieves the wind data. Collection length is the time over which the wind record exists ie data is collected in 10 minute increments @@ -638,6 +731,8 @@ def getWind(self, gaugenumber=0, collectionlength=10): '732 wind gauge' in [3] + force_refresh (bool): If True and caching is enabled, bypass cache and fetch fresh data. + Returns: dict, will return None if an error is encountered 'name' (str): station name @@ -673,6 +768,15 @@ def getWind(self, gaugenumber=0, collectionlength=10): 'gaugeht' (float): gauge height for uncorrected wind measurements """ + cache_params = {'gaugenumber': gaugenumber, 'collectionlength': collectionlength} + + def fetch_data(): + return self._getWind_impl(gaugenumber, collectionlength) + + return self._get_with_cache('wind', fetch_data, cache_params, force_refresh) + + def _getWind_impl(self, gaugenumber, collectionlength): + """Internal implementation of getWind (without caching).""" # Making gauges flexible # different Gauges if gaugenumber in ['derived', 'Derived', 0]: @@ -762,7 +866,7 @@ def getWind(self, gaugenumber=0, collectionlength=10): print(' ---- ERROR: Problem finding wind !!!') return None - def getWL(self, collectionlength=6): + def getWL(self, collectionlength=6, force_refresh=False): """This function retrieves the water level data from the server. WL data on server is NAVD88 @@ -773,6 +877,7 @@ def getWL(self, collectionlength=6): Args: collectionlength (int): dictates what value to round time to (Default value = 6) + force_refresh (bool): If True and caching is enabled, bypass cache and fetch fresh data. Returns: dictionary with keys @@ -793,6 +898,15 @@ def getWL(self, collectionlength=6): 'predictedWL': predicted tide """ + cache_params = {'collectionlength': collectionlength} + + def fetch_data(): + return self._getWL_impl(collectionlength) + + return self._get_with_cache('waterlevel', fetch_data, cache_params, force_refresh) + + def _getWL_impl(self, collectionlength): + """Internal implementation of getWL (without caching).""" # this is the back end of the url for waterlevel self.dataloc = 'oceanography/waterlevel/eopNoaaTide/eopNoaaTide.ncml' @@ -824,7 +938,7 @@ def getWL(self, collectionlength=6): self.WLpacket = None return self.WLpacket - def getGaugeWL(self, gaugenumber=5, roundto=1): + def getGaugeWL(self, gaugenumber=5, roundto=1, force_refresh=False): """This function pulls down the water level data at a particular gauge from the Server. Args: @@ -832,6 +946,7 @@ def getGaugeWL(self, gaugenumber=5, roundto=1): roundto: the time over which the wind record exists ie data is collected in 10 minute increments data is rounded to the nearst [roundto] (default 1 min) + force_refresh (bool): If True and caching is enabled, bypass cache and fetch fresh data. Returns wlpacket (dict) with keys below @@ -852,6 +967,15 @@ def getGaugeWL(self, gaugenumber=5, roundto=1): 'yFRF': yFRF position of the gage """ + cache_params = {'gaugenumber': gaugenumber, 'roundto': roundto} + + def fetch_data(): + return self._getGaugeWL_impl(gaugenumber, roundto) + + return self._get_with_cache('gaugewl', fetch_data, cache_params, force_refresh) + + def _getGaugeWL_impl(self, gaugenumber, roundto): + """Internal implementation of getGaugeWL (without caching).""" # Making gauges flexible self._wlGageURLlookup(gaugenumber) # parsing out data of interest in time @@ -905,7 +1029,7 @@ def getGaugeWL(self, gaugenumber=5, roundto=1): 'name': str(self.ncfile.title), } return wlpacket - def getBathyTransectFromNC(self, profilenumbers=None, method=1, forceReturnAll=False): + def getBathyTransectFromNC(self, profilenumbers=None, method=1, forceReturnAll=False, force_refresh=False): """This function gets the bathymetric data from the server. Args: @@ -917,6 +1041,8 @@ def getBathyTransectFromNC(self, profilenumbers=None, method=1, forceReturnAll=F forceReturnAll (bool): (Default Value = False) This will force the survey to take and return all indices between start and end, not the single + force_refresh (bool): If True and caching is enabled, bypass cache and fetch fresh data. + Returns: dictionary with keys, will return None if call fails 'xFRF': x coordinate in frf @@ -942,6 +1068,15 @@ def getBathyTransectFromNC(self, profilenumbers=None, method=1, forceReturnAll=F 'Ellipsoid': which ellipsoid is used """ + cache_params = {'profilenumbers': profilenumbers, 'method': method, 'forceReturnAll': forceReturnAll} + + def fetch_data(): + return self._getBathyTransectFromNC_impl(profilenumbers, method, forceReturnAll) + + return self._get_with_cache('bathytransect', fetch_data, cache_params, force_refresh) + + def _getBathyTransectFromNC_impl(self, profilenumbers, method, forceReturnAll): + """Internal implementation of getBathyTransectFromNC (without caching).""" # do check here on profile numbers # acceptableProfileNumbers = [None, ] self.dataloc = 'geomorphology/elevationTransects/survey/surveyTransects.ncml' # location @@ -1654,13 +1789,14 @@ def get_sensor_locations(self, datafile='frf_sensor_locations.pkl', window_days= return sensor_locations - def getLidarRunup(self, removeMasked=True): + def getLidarRunup(self, removeMasked=True, force_refresh=False): """This function will get the wave runup measurements from the lidar mounted in the dune. Args: removeMasked: if data come back as masked, remove from the arrays removeMasked will toggle the removing of data points from the tsTime series based on the flag status (Default value = True) + force_refresh (bool): If True and caching is enabled, bypass cache and fetch fresh data. Returns: dictionary with collected data. keys listed below (for more info see the netCDF file @@ -1693,6 +1829,15 @@ def getLidarRunup(self, removeMasked=True): measurement """ + cache_params = {'removeMasked': removeMasked} + + def fetch_data(): + return self._getLidarRunup_impl(removeMasked) + + return self._get_with_cache('lidarrunup', fetch_data, cache_params, force_refresh) + + def _getLidarRunup_impl(self, removeMasked): + """Internal implementation of getLidarRunup (without caching).""" self.dataloc = 'oceanography/waves/lidarWaveRunup/lidarWaveRunup.ncml' self.ncfile, self.allEpoch, _ = getnc(dataLoc=self.dataloc, callingClass=self.callingClass, dtRound=1 * 60) @@ -1753,14 +1898,15 @@ def getLidarRunup(self, removeMasked=True): out = None return out - def getCTD(self): + def getCTD(self, force_refresh=False): """THIS FUNCTION IS CURRENTLY BROKEN. THE PROBLEM IS THAT self.cshore_ncfile does not have any keys? TODO fix this function This function gets the CTD data from the thredds server - Args: None + Args: + force_refresh (bool): If True and caching is enabled, bypass cache and fetch fresh data. Returns: dict: output dictionary with keys listed below, will return None if error happens @@ -1781,6 +1927,15 @@ def getCTD(self): 'sigmaT': """ + cache_params = {} + + def fetch_data(): + return self._getCTD_impl() + + return self._get_with_cache('ctd', fetch_data, cache_params, force_refresh) + + def _getCTD_impl(self): + """Internal implementation of getCTD (without caching).""" # do check here on profile numbers # acceptableProfileNumbers = [None, ] self.dataloc = 'oceanography/ctd/eop-ctd/eop-ctd.ncml' # location of the gridded surveys @@ -1826,7 +1981,7 @@ def getCTD(self): return ctd_Dict - def getALT(self, gaugeName=None, removeMasked=True): + def getALT(self, gaugeName=None, removeMasked=True, force_refresh=False): """This function gets the Altimeter data from the thredds server. Args: @@ -1837,6 +1992,8 @@ def getALT(self, gaugeName=None, removeMasked=True): removeMasked (bool): remove the data that are masked (Default value = True) + force_refresh (bool): If True and caching is enabled, bypass cache and fetch fresh data. + Returns: a dictionary with below keys with selected data, for more info see netCDF files on server 'name': file title @@ -1864,6 +2021,15 @@ def getALT(self, gaugeName=None, removeMasked=True): 'bottomElev': Kalman filtered elevation """ + cache_params = {'gaugeName': gaugeName, 'removeMasked': removeMasked} + + def fetch_data(): + return self._getALT_impl(gaugeName, removeMasked) + + return self._get_with_cache('altimeter', fetch_data, cache_params, force_refresh) + + def _getALT_impl(self, gaugeName, removeMasked): + """Internal implementation of getALT (without caching).""" # location of the data gauge_list = ['Alt769-150', 'Alt769-200', 'Alt769-250', 'Alt769-300', 'Alt769-350', 'Alt861-150', 'Alt861-200', 'Alt861-250', 'Alt861-300', 'Alt861-350', diff --git a/tests/test_cache.py b/tests/test_cache.py new file mode 100644 index 0000000..7fd6b95 --- /dev/null +++ b/tests/test_cache.py @@ -0,0 +1,606 @@ +"""Tests for murgtools.cache module.""" + +import json +import os +import tempfile +import threading +import time +from pathlib import Path +from unittest.mock import patch + +import pytest + +from murgtools import cache +from murgtools.cache import DataCache, CacheMetadata, get_cache, enable_cache, disable_cache, clear_cache + + +class TestCacheMetadata: + """Tests for CacheMetadata class.""" + + def test_to_dict_and_from_dict(self): + """Test serialization round-trip.""" + meta = CacheMetadata( + created_at=1000000.0, + data_source='test_source', + time_range=(100, 200), + extra_info={'param': 'value'}, + ) + + meta_dict = meta.to_dict() + restored = CacheMetadata.from_dict(meta_dict) + + assert restored.created_at == meta.created_at + assert restored.data_source == meta.data_source + assert restored.time_range == meta.time_range + assert restored.extra_info == meta.extra_info + + def test_is_expired_within_ttl(self): + """Test that recent entries are not expired.""" + meta = CacheMetadata(created_at=time.time(), data_source='test') + assert meta.is_expired(ttl_days=180) is False + + def test_is_expired_after_ttl(self): + """Test that old entries are expired.""" + # Created 200 days ago + created = time.time() - (200 * 24 * 60 * 60) + meta = CacheMetadata(created_at=created, data_source='test') + assert meta.is_expired(ttl_days=180) is True + + def test_is_expired_zero_ttl_never_expires(self): + """Test that TTL of 0 means never expire.""" + # Created 1000 days ago + created = time.time() - (1000 * 24 * 60 * 60) + meta = CacheMetadata(created_at=created, data_source='test') + assert meta.is_expired(ttl_days=0) is False + + def test_age_days(self): + """Test age calculation.""" + # Created 10 days ago + created = time.time() - (10 * 24 * 60 * 60) + meta = CacheMetadata(created_at=created, data_source='test') + assert 9.9 < meta.age_days() < 10.1 + + +class TestDataCache: + """Tests for DataCache class.""" + + def setup_method(self): + """Create a temporary cache directory for each test.""" + self.temp_dir = tempfile.mkdtemp() + # Reset global cache + cache._global_cache = None + + def teardown_method(self): + """Clean up temporary directory.""" + import shutil + shutil.rmtree(self.temp_dir, ignore_errors=True) + # Reset global cache + cache._global_cache = None + + def test_cache_disabled_by_default(self): + """Verify caching is disabled by default.""" + c = DataCache(cache_dir=self.temp_dir) + assert c.enabled is False + + def test_cache_can_be_enabled(self): + """Verify caching can be enabled.""" + c = DataCache(enabled=True, cache_dir=self.temp_dir) + assert c.enabled is True + + def test_cache_disabled_returns_fresh_data(self): + """Verify disabled cache always calls fetch function.""" + c = DataCache(enabled=False, cache_dir=self.temp_dir) + call_count = 0 + + def fetch(): + nonlocal call_count + call_count += 1 + return {'data': call_count} + + result1 = c.get('source', fetch) + result2 = c.get('source', fetch) + + assert call_count == 2 + assert result1['data'] == 1 + assert result2['data'] == 2 + + def test_cache_enabled_returns_cached_data(self): + """Verify enabled cache returns cached data.""" + c = DataCache(enabled=True, cache_dir=self.temp_dir) + call_count = 0 + + def fetch(): + nonlocal call_count + call_count += 1 + return {'data': call_count} + + result1 = c.get('source', fetch) + result2 = c.get('source', fetch) + + assert call_count == 1 + assert result1['data'] == 1 + assert result2['data'] == 1 + + def test_force_refresh_bypasses_cache(self): + """Verify force_refresh fetches fresh data.""" + c = DataCache(enabled=True, cache_dir=self.temp_dir) + call_count = 0 + + def fetch(): + nonlocal call_count + call_count += 1 + return {'data': call_count} + + result1 = c.get('source', fetch) + result2 = c.get('source', fetch, force_refresh=True) + + assert call_count == 2 + assert result1['data'] == 1 + assert result2['data'] == 2 + + def test_different_sources_cached_separately(self): + """Verify different data sources have separate cache entries.""" + c = DataCache(enabled=True, cache_dir=self.temp_dir) + + result1 = c.get('source1', lambda: 'data1') + result2 = c.get('source2', lambda: 'data2') + + assert result1 == 'data1' + assert result2 == 'data2' + + def test_different_time_ranges_cached_separately(self): + """Verify different time ranges have separate cache entries.""" + c = DataCache(enabled=True, cache_dir=self.temp_dir) + + result1 = c.get('source', lambda: 'data1', time_range=(100, 200)) + result2 = c.get('source', lambda: 'data2', time_range=(200, 300)) + + assert result1 == 'data1' + assert result2 == 'data2' + + def test_expired_cache_returns_fresh_data(self): + """Verify expired cache entries are refreshed.""" + c = DataCache(enabled=True, cache_dir=self.temp_dir, ttl_days=1) + + # First fetch + result1 = c.get('source', lambda: 'old_data') + + # Manually make the cache old by modifying metadata + cache_key = c._generate_cache_key('source') + _, meta_path = c._get_cache_paths(cache_key) + with open(meta_path, 'r') as f: + meta_dict = json.load(f) + # Set created_at to 10 days ago + meta_dict['created_at'] = time.time() - (10 * 24 * 60 * 60) + with open(meta_path, 'w') as f: + json.dump(meta_dict, f) + + # Second fetch should get fresh data + result2 = c.get('source', lambda: 'new_data') + + assert result1 == 'old_data' + assert result2 == 'new_data' + + def test_is_cached(self): + """Test is_cached method.""" + c = DataCache(enabled=True, cache_dir=self.temp_dir) + + assert c.is_cached('source') is False + + c.get('source', lambda: 'data') + + assert c.is_cached('source') is True + + def test_invalidate(self): + """Test cache invalidation.""" + c = DataCache(enabled=True, cache_dir=self.temp_dir) + + c.get('source', lambda: 'data1') + assert c.is_cached('source') is True + + c.invalidate('source') + assert c.is_cached('source') is False + + result = c.get('source', lambda: 'data2') + assert result == 'data2' + + def test_clear_all(self): + """Test clearing all cache entries.""" + c = DataCache(enabled=True, cache_dir=self.temp_dir) + + c.get('source1', lambda: 'data1') + c.get('source2', lambda: 'data2') + + assert c.is_cached('source1') is True + assert c.is_cached('source2') is True + + c.clear() + + assert c.is_cached('source1') is False + assert c.is_cached('source2') is False + + def test_clear_older_than(self): + """Test clearing entries older than a threshold.""" + c = DataCache(enabled=True, cache_dir=self.temp_dir) + + # Create two entries + c.get('source1', lambda: 'data1') + c.get('source2', lambda: 'data2') + + # Make source1 old + cache_key = c._generate_cache_key('source1') + _, meta_path = c._get_cache_paths(cache_key) + with open(meta_path, 'r') as f: + meta_dict = json.load(f) + meta_dict['created_at'] = time.time() - (100 * 24 * 60 * 60) # 100 days ago + with open(meta_path, 'w') as f: + json.dump(meta_dict, f) + + # Clear entries older than 50 days + c.clear(older_than_days=50) + + assert c.is_cached('source1') is False # Old, should be cleared + assert c.is_cached('source2') is True # New, should remain + + def test_get_stats(self): + """Test cache statistics.""" + c = DataCache(enabled=True, cache_dir=self.temp_dir) + + stats = c.get_stats() + assert stats['enabled'] is True + assert stats['entry_count'] == 0 + + c.get('source1', lambda: 'data1') + c.get('source2', lambda: 'data2') + + stats = c.get_stats() + assert stats['entry_count'] == 2 + assert stats['total_size_mb'] >= 0 # Small files may round to 0 + + def test_ttl_setting(self): + """Test TTL getter and setter.""" + c = DataCache(enabled=True, cache_dir=self.temp_dir, ttl_days=90) + assert c.ttl_days == 90 + + c.ttl_days = 30 + assert c.ttl_days == 30 + + def test_ttl_negative_raises_error(self): + """Test that negative TTL raises ValueError.""" + c = DataCache(enabled=True, cache_dir=self.temp_dir) + with pytest.raises(ValueError): + c.ttl_days = -1 + + def test_cache_dir_setting(self): + """Test cache_dir getter and setter.""" + c = DataCache(enabled=True, cache_dir=self.temp_dir) + assert c.cache_dir == Path(self.temp_dir) + + new_dir = tempfile.mkdtemp() + try: + c.cache_dir = new_dir + assert c.cache_dir == Path(new_dir) + finally: + import shutil + shutil.rmtree(new_dir, ignore_errors=True) + + def test_thread_safety(self): + """Test thread-safe concurrent access.""" + c = DataCache(enabled=True, cache_dir=self.temp_dir) + results = [] + errors = [] + fetch_count = 0 + fetch_lock = threading.Lock() + + def fetch(): + nonlocal fetch_count + with fetch_lock: + fetch_count += 1 + time.sleep(0.01) # Simulate network delay + return 'data' + + def worker(source): + try: + for _ in range(5): + result = c.get(source, fetch) + results.append(result) + except Exception as e: + errors.append(e) + + threads = [threading.Thread(target=worker, args=(f'source{i % 3}',)) for i in range(10)] + for t in threads: + t.start() + for t in threads: + t.join() + + assert len(errors) == 0, f"Thread errors: {errors}" + assert all(r == 'data' for r in results) + # With 3 unique sources and proper caching, should have at most 3 fetches + # (may have more due to race conditions before cache is populated) + assert fetch_count <= 10 + + +class TestEnvironmentVariables: + """Tests for environment variable configuration.""" + + def setup_method(self): + """Save original environment.""" + self.orig_env = { + k: os.environ.get(k) for k in [ + 'MURGTOOLS_CACHE_ENABLED', + 'MURGTOOLS_CACHE_DIR', + 'MURGTOOLS_CACHE_TTL_DAYS', + ] + } + # Clear environment + for k in self.orig_env: + if k in os.environ: + del os.environ[k] + # Reset global cache + cache._global_cache = None + self.temp_dir = tempfile.mkdtemp() + + def teardown_method(self): + """Restore original environment.""" + for k, v in self.orig_env.items(): + if v is None: + if k in os.environ: + del os.environ[k] + else: + os.environ[k] = v + # Reset global cache + cache._global_cache = None + import shutil + shutil.rmtree(self.temp_dir, ignore_errors=True) + + def test_env_cache_enabled(self): + """Test MURGTOOLS_CACHE_ENABLED environment variable.""" + os.environ['MURGTOOLS_CACHE_ENABLED'] = '1' + os.environ['MURGTOOLS_CACHE_DIR'] = self.temp_dir + + c = DataCache() + assert c.enabled is True + + def test_env_cache_enabled_true_string(self): + """Test MURGTOOLS_CACHE_ENABLED with 'true' string.""" + os.environ['MURGTOOLS_CACHE_ENABLED'] = 'true' + os.environ['MURGTOOLS_CACHE_DIR'] = self.temp_dir + + c = DataCache() + assert c.enabled is True + + def test_env_cache_dir(self): + """Test MURGTOOLS_CACHE_DIR environment variable.""" + custom_dir = tempfile.mkdtemp() + try: + os.environ['MURGTOOLS_CACHE_DIR'] = custom_dir + c = DataCache() + assert c.cache_dir == Path(custom_dir) + finally: + import shutil + shutil.rmtree(custom_dir, ignore_errors=True) + + def test_env_cache_ttl(self): + """Test MURGTOOLS_CACHE_TTL_DAYS environment variable.""" + os.environ['MURGTOOLS_CACHE_TTL_DAYS'] = '90' + os.environ['MURGTOOLS_CACHE_DIR'] = self.temp_dir + + c = DataCache() + assert c.ttl_days == 90 + + def test_env_cache_ttl_invalid(self): + """Test invalid TTL value falls back to default.""" + os.environ['MURGTOOLS_CACHE_TTL_DAYS'] = 'invalid' + os.environ['MURGTOOLS_CACHE_DIR'] = self.temp_dir + + c = DataCache() + assert c.ttl_days == 180 # default + + +class TestGlobalCacheFunctions: + """Tests for global cache functions.""" + + def setup_method(self): + """Reset global cache.""" + cache._global_cache = None + self.temp_dir = tempfile.mkdtemp() + + def teardown_method(self): + """Reset global cache and clean up.""" + cache._global_cache = None + import shutil + shutil.rmtree(self.temp_dir, ignore_errors=True) + + def test_get_cache_returns_same_instance(self): + """Test get_cache returns singleton.""" + c1 = get_cache(cache_dir=self.temp_dir) + c2 = get_cache() + assert c1 is c2 + + def test_enable_cache(self): + """Test enable_cache convenience function.""" + c = enable_cache(cache_dir=self.temp_dir) + assert c.enabled is True + + def test_disable_cache(self): + """Test disable_cache convenience function.""" + enable_cache(cache_dir=self.temp_dir) + disable_cache() + c = get_cache() + assert c.enabled is False + + def test_clear_cache(self): + """Test clear_cache convenience function.""" + c = enable_cache(cache_dir=self.temp_dir) + c.get('source', lambda: 'data') + assert c.is_cached('source') is True + + clear_cache() + assert c.is_cached('source') is False + + +class TestCacheWithComplexData: + """Tests for caching complex data types.""" + + def setup_method(self): + """Create temporary cache directory.""" + self.temp_dir = tempfile.mkdtemp() + cache._global_cache = None + + def teardown_method(self): + """Clean up.""" + import shutil + shutil.rmtree(self.temp_dir, ignore_errors=True) + cache._global_cache = None + + def test_cache_numpy_arrays(self): + """Test caching numpy arrays.""" + import numpy as np + + c = DataCache(enabled=True, cache_dir=self.temp_dir) + + data = {'array': np.array([1.0, 2.0, 3.0]), 'value': 42} + c.get('source', lambda: data) + + # Clear memory and reload from cache + result = c.get('source', lambda: None) + + assert np.array_equal(result['array'], data['array']) + assert result['value'] == 42 + + def test_cache_nested_dict(self): + """Test caching nested dictionaries.""" + c = DataCache(enabled=True, cache_dir=self.temp_dir) + + data = { + 'level1': { + 'level2': { + 'values': [1, 2, 3], + 'name': 'test', + } + } + } + c.get('source', lambda: data) + result = c.get('source', lambda: None) + + assert result == data + + def test_cache_with_extra_params(self): + """Test caching with extra parameters.""" + c = DataCache(enabled=True, cache_dir=self.temp_dir) + + # Different extra_params should create different cache entries + result1 = c.get('source', lambda: 'data1', extra_params={'gaugenumber': 'gauge1'}) + result2 = c.get('source', lambda: 'data2', extra_params={'gaugenumber': 'gauge2'}) + + assert result1 == 'data1' + assert result2 == 'data2' + + +class TestGetObsCacheIntegration: + """Tests for getObs class caching integration.""" + + def setup_method(self): + """Create temporary cache directory.""" + self.temp_dir = tempfile.mkdtemp() + cache._global_cache = None + + def teardown_method(self): + """Clean up.""" + import shutil + shutil.rmtree(self.temp_dir, ignore_errors=True) + cache._global_cache = None + + def test_getobs_cache_disabled_by_default(self): + """Test that getObs has caching disabled by default.""" + import datetime as DT + from murgtools.getdata.getDataFRF import getObs + + d1 = DT.datetime(2024, 1, 1) + d2 = DT.datetime(2024, 1, 2) + + obs = getObs(d1, d2) + assert obs._use_cache is False + assert obs._cache is None + + def test_getobs_cache_can_be_enabled(self): + """Test that getObs caching can be enabled.""" + import datetime as DT + from murgtools.getdata.getDataFRF import getObs + + d1 = DT.datetime(2024, 1, 1) + d2 = DT.datetime(2024, 1, 2) + + obs = getObs(d1, d2, use_cache=True, cache_dir=self.temp_dir) + assert obs._use_cache is True + assert obs._cache is not None + assert obs._cache.enabled is True + + def test_getobs_cache_custom_ttl(self): + """Test that getObs accepts custom TTL.""" + import datetime as DT + from murgtools.getdata.getDataFRF import getObs + + d1 = DT.datetime(2024, 1, 1) + d2 = DT.datetime(2024, 1, 2) + + obs = getObs(d1, d2, use_cache=True, cache_dir=self.temp_dir, cache_ttl_days=90) + assert obs._cache.ttl_days == 90 + + def test_getobs_get_with_cache_helper(self): + """Test the _get_with_cache helper method.""" + import datetime as DT + from murgtools.getdata.getDataFRF import getObs + + d1 = DT.datetime(2024, 1, 1) + d2 = DT.datetime(2024, 1, 2) + + # Test with caching disabled + obs_no_cache = getObs(d1, d2) + call_count = 0 + + def fetch(): + nonlocal call_count + call_count += 1 + return {'data': call_count} + + # Without cache, should call fetch each time + result1 = obs_no_cache._get_with_cache('test', fetch) + result2 = obs_no_cache._get_with_cache('test', fetch) + assert call_count == 2 + + # Test with caching enabled + call_count = 0 + obs_with_cache = getObs(d1, d2, use_cache=True, cache_dir=self.temp_dir) + + result3 = obs_with_cache._get_with_cache('test', fetch) + result4 = obs_with_cache._get_with_cache('test', fetch) + assert call_count == 1 # Only called once due to caching + assert result3 == result4 + + def test_getobs_force_refresh(self): + """Test force_refresh parameter.""" + import datetime as DT + from murgtools.getdata.getDataFRF import getObs + + d1 = DT.datetime(2024, 1, 1) + d2 = DT.datetime(2024, 1, 2) + + obs = getObs(d1, d2, use_cache=True, cache_dir=self.temp_dir) + call_count = 0 + + def fetch(): + nonlocal call_count + call_count += 1 + return {'data': call_count} + + # First call caches + result1 = obs._get_with_cache('test', fetch) + # Second call returns cached + result2 = obs._get_with_cache('test', fetch) + assert call_count == 1 + + # Force refresh should call again + result3 = obs._get_with_cache('test', fetch, force_refresh=True) + assert call_count == 2 + assert result3['data'] == 2 diff --git a/tests/test_config.py b/tests/test_config.py index a6ec5a1..05292b3 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -98,11 +98,146 @@ def test_force_chl_server(self): def test_socket_error_defaults_to_chl(self): """Verify socket errors default to CHL server.""" with patch('socket.gethostbyname', side_effect=OSError('Network error')): + config.clear_server_cache() # Clear cache to force re-detection url, prefix = config.get_thredds_server() assert url == config.THREDDS_CHL_PUBLIC assert prefix == 'frf' +class TestServerCaching: + """Tests for server detection caching functionality.""" + + def setup_method(self): + """Clear the cache before each test.""" + config.clear_server_cache() + + def teardown_method(self): + """Clear the cache after each test.""" + config.clear_server_cache() + + def test_cache_returns_same_result(self): + """Verify cached result is returned on subsequent calls.""" + # First call should compute and cache + result1 = config.get_thredds_server() + # Second call should return cached result + result2 = config.get_thredds_server() + + assert result1 == result2 + + def test_cache_avoids_repeated_socket_calls(self): + """Verify socket operations are only called once with caching.""" + with patch('murgtools.config.socket.gethostbyname', return_value='192.168.1.1') as mock_socket: + config.clear_server_cache() + + # Multiple calls should only trigger one socket operation + config.get_thredds_server() + config.get_thredds_server() + config.get_thredds_server() + + # Socket should only be called once (for IP detection) + assert mock_socket.call_count == 1 + + def test_clear_cache_resets_detection(self): + """Verify clear_server_cache resets the cached state.""" + with patch('murgtools.config.socket.gethostbyname', return_value='192.168.1.1') as mock_socket: + config.clear_server_cache() + + # First call + config.get_thredds_server() + assert mock_socket.call_count == 1 + + # Clear cache + config.clear_server_cache() + + # Second call should trigger socket again + config.get_thredds_server() + assert mock_socket.call_count == 2 + + def test_explicit_ip_bypasses_cache(self): + """Verify explicit ip_address parameter still works correctly.""" + config.clear_server_cache() + + # Cache with external IP + with patch('murgtools.config.socket.gethostbyname', return_value='192.168.1.1'): + url1, prefix1 = config.get_thredds_server() + assert prefix1 == 'frf' # External network + + # Explicit FRF IP should return FRF server regardless of cache + url2, prefix2 = config.get_thredds_server(ip_address='134.164.129.1') + assert prefix2 == 'FRF' + + def test_explicit_server_bypasses_cache(self): + """Verify explicit server parameter still works correctly.""" + config.clear_server_cache() + + # Cache with external IP + with patch('murgtools.config.socket.gethostbyname', return_value='192.168.1.1'): + url1, prefix1 = config.get_thredds_server() + assert prefix1 == 'frf' # External network + + # Explicit FRF server should return FRF regardless of cache + url2, prefix2 = config.get_thredds_server(server='FRF') + assert prefix2 == 'FRF' + + def test_is_frf_network_uses_cached_ip(self): + """Verify is_frf_network uses cached IP address.""" + with patch('murgtools.config.socket.gethostbyname', return_value='134.164.129.1') as mock_socket: + config.clear_server_cache() + + # Multiple calls should only trigger one socket operation + config.is_frf_network() + config.is_frf_network() + config.is_frf_network() + + assert mock_socket.call_count == 1 + + def test_thread_safety_concurrent_access(self): + """Verify cache is thread-safe under concurrent access.""" + import threading + import time + + config.clear_server_cache() + results = [] + errors = [] + + def worker(): + try: + for _ in range(10): + result = config.get_thredds_server() + results.append(result) + except Exception as e: + errors.append(e) + + # Run multiple threads concurrently + threads = [threading.Thread(target=worker) for _ in range(5)] + for t in threads: + t.start() + for t in threads: + t.join() + + # No errors should occur + assert len(errors) == 0, f"Thread errors: {errors}" + + # All results should be identical + assert len(set(results)) == 1, "Inconsistent results across threads" + + def test_invalid_ip_address_type_handled(self): + """Verify non-string ip_address is handled safely.""" + # Should not raise, should return CHL (external) server + url, prefix = config.get_thredds_server(ip_address=12345) + assert prefix == 'frf' + + url, prefix = config.get_thredds_server(ip_address=None) + # None triggers cache lookup, which is valid + assert prefix in ('frf', 'FRF') + + def test_is_frf_network_invalid_ip_returns_false(self): + """Verify is_frf_network returns False for invalid ip_address types.""" + assert config.is_frf_network(ip_address=12345) is False + assert config.is_frf_network(ip_address=['134.164.1.1']) is False + assert config.is_frf_network(ip_address={'ip': '134.164.1.1'}) is False + + class TestIsFrfNetwork: """Tests for is_frf_network helper function."""