diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 30f2b38..3d4027f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,3 +47,10 @@ jobs: - name: Run tests with coverage run: just test-coverage + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v5 + with: + file: ./coverage.xml + fail_ci_if_error: true + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..948295f --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,26 @@ +# Pre-commit hooks for tinygrid +# Install: uv run pre-commit install +# Run manually: uv run pre-commit run --all-files + +repos: + # Use local hooks to respect project configs (ruff.toml, pyrightconfig.json) + - repo: local + hooks: + - id: ruff + name: ruff + entry: uv run ruff check --fix --unsafe-fixes + language: system + types: [python] + + - id: ruff-format + name: ruff-format + entry: uv run ruff format + language: system + types: [python] + + - id: pyright + name: pyright + entry: uv run pyright + language: system + types: [python] + pass_filenames: false diff --git a/README.md b/README.md index dc4a909..5fdfef8 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # Tiny Grid [![CI](https://github.com/kvkenyon/tinygrid/actions/workflows/ci.yml/badge.svg)](https://github.com/kvkenyon/tinygrid/actions) +[![codecov](https://codecov.io/gh/kvkenyon/tinygrid/branch/main/graph/badge.svg)](https://codecov.io/gh/kvkenyon/tinygrid) A Python SDK for accessing electricity grid data from US Independent System Operators (ISOs). @@ -195,19 +196,37 @@ just test # Run tests with coverage just test-coverage -# Lint +# Lint and format just lint - -# Format just format +just lint-fix # Auto-fix lint issues # Type check just type-check -# Run all checks +# Run all checks (lint, format, type-check, test) just check ``` +### Pre-commit Hooks + +Pre-commit hooks automatically run linting and formatting before each commit to catch issues early. + +```bash +# Install hooks (one-time setup after cloning) +just hooks-install + +# Run hooks manually on all files +just hooks-run +``` + +The hooks run: +- **ruff** - Linting with auto-fix +- **ruff-format** - Code formatting +- **pyright** - Type checking + +If a hook fails, fix the issues and re-commit. Ruff auto-fixes will be staged automatically. + ## License MIT diff --git a/examples/ercot_demo.ipynb b/examples/ercot_demo.ipynb index 743e7b0..d1d87ff 100644 --- a/examples/ercot_demo.ipynb +++ b/examples/ercot_demo.ipynb @@ -2,9 +2,18 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": 29, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The autoreload extension is already loaded. To reload it, use:\n", + " %reload_ext autoreload\n" + ] + } + ], "source": [ "%load_ext autoreload\n", "%autoreload 2" @@ -44,7 +53,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 30, "metadata": { "execution": { "iopub.execute_input": "2025-12-28T16:14:56.927060Z", @@ -78,7 +87,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 31, "metadata": { "execution": { "iopub.execute_input": "2025-12-28T16:14:57.854493Z", @@ -122,7 +131,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 32, "metadata": { "execution": { "iopub.execute_input": "2025-12-28T16:14:58.838540Z", @@ -171,7 +180,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 34, "metadata": { "execution": { "iopub.execute_input": "2025-12-28T16:15:00.615946Z", @@ -186,7 +195,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Real-Time SPP: 91,390 records\n" + "Real-Time SPP: 84,930 records\n" ] }, { @@ -221,46 +230,46 @@ " \n", " \n", " 0\n", - " 2024-12-12 23:30:00-06:00\n", - " 2024-12-12 23:45:00-06:00\n", + " 2023-12-12 23:30:00-06:00\n", + " 2023-12-12 23:45:00-06:00\n", " 7RNCHSLR_ALL\n", - " 18.81\n", + " 11.65\n", " REAL_TIME_15_MIN\n", " RN\n", " \n", " \n", " 1\n", - " 2024-12-12 23:30:00-06:00\n", - " 2024-12-12 23:45:00-06:00\n", - " ADL_RN\n", - " 20.58\n", + " 2023-12-12 23:30:00-06:00\n", + " 2023-12-12 23:45:00-06:00\n", + " AEEC\n", + " 12.47\n", " REAL_TIME_15_MIN\n", " RN\n", " \n", " \n", " 2\n", - " 2024-12-12 23:30:00-06:00\n", - " 2024-12-12 23:45:00-06:00\n", - " AEEC\n", - " 8.81\n", + " 2023-12-12 23:30:00-06:00\n", + " 2023-12-12 23:45:00-06:00\n", + " AGUAYO_UNIT1\n", + " 12.30\n", " REAL_TIME_15_MIN\n", " RN\n", " \n", " \n", " 3\n", - " 2024-12-12 23:30:00-06:00\n", - " 2024-12-12 23:45:00-06:00\n", - " AE_RN\n", - " 18.83\n", + " 2023-12-12 23:30:00-06:00\n", + " 2023-12-12 23:45:00-06:00\n", + " AJAXWIND_RN\n", + " 12.37\n", " REAL_TIME_15_MIN\n", " RN\n", " \n", " \n", " 4\n", - " 2024-12-12 23:30:00-06:00\n", - " 2024-12-12 23:45:00-06:00\n", - " AGUAYO_UNIT1\n", - " 8.88\n", + " 2023-12-12 23:30:00-06:00\n", + " 2023-12-12 23:45:00-06:00\n", + " ALGOD_ALL_RN\n", + " 7.64\n", " REAL_TIME_15_MIN\n", " RN\n", " \n", @@ -270,14 +279,14 @@ ], "text/plain": [ " Time End Time Location Price Market Location Type\n", - "0 2024-12-12 23:30:00-06:00 2024-12-12 23:45:00-06:00 7RNCHSLR_ALL 18.81 REAL_TIME_15_MIN RN\n", - "1 2024-12-12 23:30:00-06:00 2024-12-12 23:45:00-06:00 ADL_RN 20.58 REAL_TIME_15_MIN RN\n", - "2 2024-12-12 23:30:00-06:00 2024-12-12 23:45:00-06:00 AEEC 8.81 REAL_TIME_15_MIN RN\n", - "3 2024-12-12 23:30:00-06:00 2024-12-12 23:45:00-06:00 AE_RN 18.83 REAL_TIME_15_MIN RN\n", - "4 2024-12-12 23:30:00-06:00 2024-12-12 23:45:00-06:00 AGUAYO_UNIT1 8.88 REAL_TIME_15_MIN RN" + "0 2023-12-12 23:30:00-06:00 2023-12-12 23:45:00-06:00 7RNCHSLR_ALL 11.65 REAL_TIME_15_MIN RN\n", + "1 2023-12-12 23:30:00-06:00 2023-12-12 23:45:00-06:00 AEEC 12.47 REAL_TIME_15_MIN RN\n", + "2 2023-12-12 23:30:00-06:00 2023-12-12 23:45:00-06:00 AGUAYO_UNIT1 12.30 REAL_TIME_15_MIN RN\n", + "3 2023-12-12 23:30:00-06:00 2023-12-12 23:45:00-06:00 AJAXWIND_RN 12.37 REAL_TIME_15_MIN RN\n", + "4 2023-12-12 23:30:00-06:00 2023-12-12 23:45:00-06:00 ALGOD_ALL_RN 7.64 REAL_TIME_15_MIN RN" ] }, - "execution_count": 5, + "execution_count": 34, "metadata": {}, "output_type": "execute_result" } @@ -285,7 +294,7 @@ "source": [ "# Real-time 15-minute SPP\n", "df = ercot.get_spp(\n", - " start=\"2024-12-12\",\n", + " start=\"2023-12-12\",\n", " market=Market.REAL_TIME_15_MIN,\n", ")\n", "\n", @@ -295,7 +304,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 35, "metadata": { "execution": { "iopub.execute_input": "2025-12-28T16:15:12.766812Z", @@ -310,7 +319,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Load Zone SPP: 1,520 records\n" + "Load Zone SPP: 103,455 records\n" ] }, { @@ -347,61 +356,61 @@ " 0\n", " 2025-12-27 23:30:00-06:00\n", " 2025-12-27 23:45:00-06:00\n", - " LZ_AEN\n", - " 8.10\n", + " 7RNCHSLR_ALL\n", + " 8.51\n", " REAL_TIME_15_MIN\n", - " LZ\n", + " RN\n", " \n", " \n", " 1\n", " 2025-12-27 23:30:00-06:00\n", " 2025-12-27 23:45:00-06:00\n", - " LZ_AEN\n", - " 8.10\n", + " A4_DGR1_RN\n", + " 7.73\n", " REAL_TIME_15_MIN\n", - " LZEW\n", + " RN\n", " \n", " \n", " 2\n", " 2025-12-27 23:30:00-06:00\n", " 2025-12-27 23:45:00-06:00\n", - " LZ_CPS\n", + " A4_DGR2_RN\n", " 7.73\n", " REAL_TIME_15_MIN\n", - " LZEW\n", + " RN\n", " \n", " \n", " 3\n", " 2025-12-27 23:30:00-06:00\n", " 2025-12-27 23:45:00-06:00\n", - " LZ_CPS\n", - " 7.73\n", + " ABINDUST_RN\n", + " 1.21\n", " REAL_TIME_15_MIN\n", - " LZ\n", + " RN\n", " \n", " \n", " 4\n", " 2025-12-27 23:30:00-06:00\n", " 2025-12-27 23:45:00-06:00\n", - " LZ_HOUSTON\n", - " 9.89\n", + " ADL_RN\n", + " 10.00\n", " REAL_TIME_15_MIN\n", - " LZEW\n", + " RN\n", " \n", " \n", "\n", "" ], "text/plain": [ - " Time End Time Location Price Market Location Type\n", - "0 2025-12-27 23:30:00-06:00 2025-12-27 23:45:00-06:00 LZ_AEN 8.10 REAL_TIME_15_MIN LZ\n", - "1 2025-12-27 23:30:00-06:00 2025-12-27 23:45:00-06:00 LZ_AEN 8.10 REAL_TIME_15_MIN LZEW\n", - "2 2025-12-27 23:30:00-06:00 2025-12-27 23:45:00-06:00 LZ_CPS 7.73 REAL_TIME_15_MIN LZEW\n", - "3 2025-12-27 23:30:00-06:00 2025-12-27 23:45:00-06:00 LZ_CPS 7.73 REAL_TIME_15_MIN LZ\n", - "4 2025-12-27 23:30:00-06:00 2025-12-27 23:45:00-06:00 LZ_HOUSTON 9.89 REAL_TIME_15_MIN LZEW" + " Time End Time Location Price Market Location Type\n", + "0 2025-12-27 23:30:00-06:00 2025-12-27 23:45:00-06:00 7RNCHSLR_ALL 8.51 REAL_TIME_15_MIN RN\n", + "1 2025-12-27 23:30:00-06:00 2025-12-27 23:45:00-06:00 A4_DGR1_RN 7.73 REAL_TIME_15_MIN RN\n", + "2 2025-12-27 23:30:00-06:00 2025-12-27 23:45:00-06:00 A4_DGR2_RN 7.73 REAL_TIME_15_MIN RN\n", + "3 2025-12-27 23:30:00-06:00 2025-12-27 23:45:00-06:00 ABINDUST_RN 1.21 REAL_TIME_15_MIN RN\n", + "4 2025-12-27 23:30:00-06:00 2025-12-27 23:45:00-06:00 ADL_RN 10.00 REAL_TIME_15_MIN RN" ] }, - "execution_count": 6, + "execution_count": 35, "metadata": {}, "output_type": "execute_result" } @@ -2646,7 +2655,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 36, "metadata": { "execution": { "iopub.execute_input": "2025-12-27T23:03:14.961791Z", @@ -2661,7 +2670,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Week of DAM SPP: 1,152 records\n" + "Week of DAM SPP: 136,944 records\n" ] }, { @@ -2697,40 +2706,40 @@ " 0\n", " 2024-12-26 00:00:00-06:00\n", " 2024-12-26 01:00:00-06:00\n", - " LZ_AEN\n", - " 16.64\n", + " 7RNCHSLR_ALL\n", + " 16.72\n", " DAY_AHEAD_HOURLY\n", " \n", " \n", " 1\n", " 2024-12-26 00:00:00-06:00\n", " 2024-12-26 01:00:00-06:00\n", - " LZ_CPS\n", - " 16.76\n", + " ADL_RN\n", + " 16.72\n", " DAY_AHEAD_HOURLY\n", " \n", " \n", " 2\n", " 2024-12-26 00:00:00-06:00\n", " 2024-12-26 01:00:00-06:00\n", - " LZ_HOUSTON\n", - " 16.72\n", + " AEEC\n", + " 17.89\n", " DAY_AHEAD_HOURLY\n", " \n", " \n", " 3\n", " 2024-12-26 00:00:00-06:00\n", " 2024-12-26 01:00:00-06:00\n", - " LZ_LCRA\n", - " 16.78\n", + " AE_RN\n", + " 16.72\n", " DAY_AHEAD_HOURLY\n", " \n", " \n", " 4\n", " 2024-12-26 00:00:00-06:00\n", " 2024-12-26 01:00:00-06:00\n", - " LZ_NORTH\n", - " 16.75\n", + " AGUAYO_UNIT1\n", + " 16.20\n", " DAY_AHEAD_HOURLY\n", " \n", " \n", @@ -2738,15 +2747,15 @@ "" ], "text/plain": [ - " Time End Time Location Price Market\n", - "0 2024-12-26 00:00:00-06:00 2024-12-26 01:00:00-06:00 LZ_AEN 16.64 DAY_AHEAD_HOURLY\n", - "1 2024-12-26 00:00:00-06:00 2024-12-26 01:00:00-06:00 LZ_CPS 16.76 DAY_AHEAD_HOURLY\n", - "2 2024-12-26 00:00:00-06:00 2024-12-26 01:00:00-06:00 LZ_HOUSTON 16.72 DAY_AHEAD_HOURLY\n", - "3 2024-12-26 00:00:00-06:00 2024-12-26 01:00:00-06:00 LZ_LCRA 16.78 DAY_AHEAD_HOURLY\n", - "4 2024-12-26 00:00:00-06:00 2024-12-26 01:00:00-06:00 LZ_NORTH 16.75 DAY_AHEAD_HOURLY" + " Time End Time Location Price Market\n", + "0 2024-12-26 00:00:00-06:00 2024-12-26 01:00:00-06:00 7RNCHSLR_ALL 16.72 DAY_AHEAD_HOURLY\n", + "1 2024-12-26 00:00:00-06:00 2024-12-26 01:00:00-06:00 ADL_RN 16.72 DAY_AHEAD_HOURLY\n", + "2 2024-12-26 00:00:00-06:00 2024-12-26 01:00:00-06:00 AEEC 17.89 DAY_AHEAD_HOURLY\n", + "3 2024-12-26 00:00:00-06:00 2024-12-26 01:00:00-06:00 AE_RN 16.72 DAY_AHEAD_HOURLY\n", + "4 2024-12-26 00:00:00-06:00 2024-12-26 01:00:00-06:00 AGUAYO_UNIT1 16.20 DAY_AHEAD_HOURLY" ] }, - "execution_count": 4, + "execution_count": 36, "metadata": {}, "output_type": "execute_result" } diff --git a/justfile b/justfile index 58a6f56..6315611 100644 --- a/justfile +++ b/justfile @@ -18,7 +18,7 @@ test-func func: # Run tests with coverage report test-coverage: - uv run pytest --cov=tinygrid + uv run pytest --cov=tinygrid --cov-report=xml --cov-report=term-missing # Lint code lint: @@ -44,6 +44,14 @@ type-check: pre-commit: lint-fix format type-check @echo "Pre-commit checks completed!" +# Install pre-commit hooks (run once after cloning) +hooks-install: + uv run pre-commit install + +# Run pre-commit hooks on all files +hooks-run: + uv run pre-commit run --all-files + # Run all checks: lint, format, type check, and tests check: lint format type-check test @echo "All checks passed!" diff --git a/pyproject.toml b/pyproject.toml index 65b810b..2f81d87 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,7 @@ dev = [ "pyright>=1.1.0", "pandas>=2.3.3", "respx>=0.21.0", + "pre-commit>=3.6.0", ] [tool.pytest.ini_options] diff --git a/tests/test_base.py b/tests/test_base.py index 7e57d82..b18897a 100644 --- a/tests/test_base.py +++ b/tests/test_base.py @@ -106,3 +106,13 @@ def test_handle_error_preserves_grid_error(self): client._handle_error(original_error, endpoint="/test") assert exc_info.value.status_code == 500 + + def test_handle_error_timeout_error(self): + """Test that TimeoutError is converted to GridTimeoutError.""" + from tinygrid.errors import GridTimeoutError + + client = ConcreteISOClient(base_url="https://api.test.com", timeout=30.0) + timeout_error = TimeoutError("Connection timed out") + + with pytest.raises(GridTimeoutError, match="Request timed out"): + client._handle_error(timeout_error, endpoint="/test-endpoint") diff --git a/tests/test_constants.py b/tests/test_constants.py new file mode 100644 index 0000000..ce8576b --- /dev/null +++ b/tests/test_constants.py @@ -0,0 +1,238 @@ +"""Tests for ERCOT constants and enums.""" + +from tinygrid.constants.ercot import ( + COLUMN_MAPPINGS, + EMIL_IDS, + ENDPOINT_MAPPINGS, + ERCOT_TIMEZONE, + HISTORICAL_THRESHOLD_DAYS, + LIVE_API_RETENTION, + LOAD_ZONES, + PUBLIC_API_BASE_URL, + TRADING_HUBS, + LocationType, + Market, + SettlementPointType, +) + + +class TestMarketEnum: + """Test Market enum.""" + + def test_market_enum_values(self): + """Test that Market enum has expected values.""" + assert Market.REAL_TIME_SCED == "REAL_TIME_SCED" + assert Market.REAL_TIME_15_MIN == "REAL_TIME_15_MIN" + assert Market.DAY_AHEAD_HOURLY == "DAY_AHEAD_HOURLY" + + def test_market_enum_is_string(self): + """Test that Market enum values are strings.""" + assert isinstance(Market.REAL_TIME_SCED, str) + assert isinstance(Market.REAL_TIME_15_MIN, str) + assert isinstance(Market.DAY_AHEAD_HOURLY, str) + + def test_market_enum_str_representation(self): + """Test string representation of Market enum.""" + assert str(Market.REAL_TIME_SCED) == "REAL_TIME_SCED" + + +class TestLocationTypeEnum: + """Test LocationType enum.""" + + def test_location_type_values(self): + """Test that LocationType enum has expected values.""" + assert LocationType.LOAD_ZONE == "Load Zone" + assert LocationType.TRADING_HUB == "Trading Hub" + assert LocationType.RESOURCE_NODE == "Resource Node" + assert LocationType.ELECTRICAL_BUS == "Electrical Bus" + + def test_location_type_is_string(self): + """Test that LocationType enum values are strings.""" + assert isinstance(LocationType.LOAD_ZONE, str) + assert isinstance(LocationType.TRADING_HUB, str) + + def test_location_type_str_representation(self): + """Test string representation of LocationType enum.""" + assert str(LocationType.LOAD_ZONE) == "Load Zone" + + +class TestSettlementPointTypeEnum: + """Test SettlementPointType enum.""" + + def test_settlement_point_type_values(self): + """Test that SettlementPointType enum has expected values.""" + assert SettlementPointType.LZ == "LZ" + assert SettlementPointType.HB == "HB" + assert SettlementPointType.RN == "RN" + + def test_settlement_point_type_is_string(self): + """Test that SettlementPointType values are strings.""" + assert isinstance(SettlementPointType.LZ, str) + assert isinstance(SettlementPointType.HB, str) + + +class TestConstants: + """Test ERCOT constants.""" + + def test_ercot_timezone(self): + """Test ERCOT timezone constant.""" + assert ERCOT_TIMEZONE == "US/Central" + + def test_historical_threshold_days(self): + """Test historical threshold constant.""" + assert HISTORICAL_THRESHOLD_DAYS == 90 + assert isinstance(HISTORICAL_THRESHOLD_DAYS, int) + + def test_public_api_base_url(self): + """Test public API base URL.""" + assert PUBLIC_API_BASE_URL == "https://api.ercot.com/api/public-reports" + assert PUBLIC_API_BASE_URL.startswith("https://") + + def test_load_zones_list(self): + """Test LOAD_ZONES list.""" + assert isinstance(LOAD_ZONES, list) + assert len(LOAD_ZONES) == 8 + assert "LZ_HOUSTON" in LOAD_ZONES + assert "LZ_NORTH" in LOAD_ZONES + assert "LZ_SOUTH" in LOAD_ZONES + assert "LZ_WEST" in LOAD_ZONES + + def test_trading_hubs_list(self): + """Test TRADING_HUBS list.""" + assert isinstance(TRADING_HUBS, list) + assert len(TRADING_HUBS) == 7 + assert "HB_HOUSTON" in TRADING_HUBS + assert "HB_NORTH" in TRADING_HUBS + assert "HB_SOUTH" in TRADING_HUBS + assert "HB_WEST" in TRADING_HUBS + + +class TestEndpointMappings: + """Test ENDPOINT_MAPPINGS dictionary.""" + + def test_endpoint_mappings_has_spp(self): + """Test SPP endpoint mappings.""" + assert "spp" in ENDPOINT_MAPPINGS + assert Market.REAL_TIME_15_MIN in ENDPOINT_MAPPINGS["spp"] + assert Market.DAY_AHEAD_HOURLY in ENDPOINT_MAPPINGS["spp"] + + def test_endpoint_mappings_has_lmp(self): + """Test LMP endpoint mappings.""" + assert "lmp" in ENDPOINT_MAPPINGS + assert Market.REAL_TIME_SCED in ENDPOINT_MAPPINGS["lmp"] + assert Market.DAY_AHEAD_HOURLY in ENDPOINT_MAPPINGS["lmp"] + + def test_endpoint_mappings_has_as_prices(self): + """Test AS prices endpoint mapping.""" + assert "as_prices" in ENDPOINT_MAPPINGS + assert isinstance(ENDPOINT_MAPPINGS["as_prices"], str) + + def test_endpoint_mappings_has_as_plan(self): + """Test AS plan endpoint mapping.""" + assert "as_plan" in ENDPOINT_MAPPINGS + assert isinstance(ENDPOINT_MAPPINGS["as_plan"], str) + + def test_endpoint_mappings_has_wind_solar(self): + """Test wind and solar endpoint mappings.""" + assert "wind_forecast" in ENDPOINT_MAPPINGS + assert "wind_forecast_geo" in ENDPOINT_MAPPINGS + assert "solar_forecast" in ENDPOINT_MAPPINGS + assert "solar_forecast_geo" in ENDPOINT_MAPPINGS + + +class TestLiveAPIRetention: + """Test LIVE_API_RETENTION dictionary.""" + + def test_live_api_retention_has_expected_keys(self): + """Test that LIVE_API_RETENTION has expected categories.""" + assert "real_time" in LIVE_API_RETENTION + assert "day_ahead" in LIVE_API_RETENTION + assert "forecast" in LIVE_API_RETENTION + assert "load" in LIVE_API_RETENTION + assert "default" in LIVE_API_RETENTION + + def test_live_api_retention_values(self): + """Test LIVE_API_RETENTION values.""" + assert LIVE_API_RETENTION["real_time"] == 1 + assert LIVE_API_RETENTION["day_ahead"] == 2 + assert LIVE_API_RETENTION["forecast"] == 3 + assert LIVE_API_RETENTION["load"] == 3 + assert LIVE_API_RETENTION["default"] == 1 + + def test_live_api_retention_all_positive(self): + """Test that all retention values are positive.""" + for value in LIVE_API_RETENTION.values(): + assert value > 0 + + +class TestEmilIDs: + """Test EMIL_IDs dictionary.""" + + def test_emil_ids_has_disclosure_reports(self): + """Test EMIL IDs for disclosure reports.""" + assert "as_reports_dam" in EMIL_IDS + assert "as_reports_sced" in EMIL_IDS + assert "dam_disclosure" in EMIL_IDS + assert "sced_disclosure" in EMIL_IDS + + def test_emil_ids_has_real_time_endpoints(self): + """Test EMIL IDs for real-time endpoints.""" + assert "np6-905-cd" in EMIL_IDS + assert "np6-788-cd" in EMIL_IDS + assert "np6-787-cd" in EMIL_IDS + + def test_emil_ids_has_dam_endpoints(self): + """Test EMIL IDs for DAM endpoints.""" + assert "np4-190-cd" in EMIL_IDS + assert "np4-183-cd" in EMIL_IDS + assert "np4-188-cd" in EMIL_IDS + assert "np4-33-cd" in EMIL_IDS + + def test_emil_ids_has_forecast_endpoints(self): + """Test EMIL IDs for forecast endpoints.""" + assert "np4-732-cd" in EMIL_IDS + assert "np4-742-cd" in EMIL_IDS + assert "np4-737-cd" in EMIL_IDS + assert "np4-745-cd" in EMIL_IDS + + def test_emil_ids_values_are_strings(self): + """Test that all EMIL ID values are strings.""" + for value in EMIL_IDS.values(): + assert isinstance(value, str) + + +class TestColumnMappings: + """Test COLUMN_MAPPINGS dictionary.""" + + def test_column_mappings_has_location_columns(self): + """Test column mappings for location fields.""" + assert "ElectricalBus" in COLUMN_MAPPINGS + assert "SettlementPoint" in COLUMN_MAPPINGS + assert "SettlementPointName" in COLUMN_MAPPINGS + assert COLUMN_MAPPINGS["SettlementPoint"] == "Location" + + def test_column_mappings_has_price_columns(self): + """Test column mappings for price fields.""" + assert "SettlementPointPrice" in COLUMN_MAPPINGS + assert "LMP" in COLUMN_MAPPINGS + assert "ShadowPrice" in COLUMN_MAPPINGS + assert COLUMN_MAPPINGS["LMP"] == "Price" + + def test_column_mappings_has_time_columns(self): + """Test column mappings for time fields.""" + assert "SCEDTimestamp" in COLUMN_MAPPINGS + assert "DeliveryDate" in COLUMN_MAPPINGS + assert "DeliveryHour" in COLUMN_MAPPINGS + assert "PostedDatetime" in COLUMN_MAPPINGS + assert COLUMN_MAPPINGS["SCEDTimestamp"] == "Timestamp" + + def test_column_mappings_has_flag_columns(self): + """Test column mappings for flag fields.""" + assert "DSTFlag" in COLUMN_MAPPINGS + assert "RepeatedHourFlag" in COLUMN_MAPPINGS + assert COLUMN_MAPPINGS["DSTFlag"] == "DST" + + def test_column_mappings_values_are_strings(self): + """Test that all column mapping values are strings.""" + for value in COLUMN_MAPPINGS.values(): + assert isinstance(value, str) diff --git a/tests/test_dates_utils.py b/tests/test_dates_utils.py new file mode 100644 index 0000000..0730f7c --- /dev/null +++ b/tests/test_dates_utils.py @@ -0,0 +1,9 @@ +import pandas as pd + +from tinygrid.utils.dates import parse_date + + +def test_parse_date_defaults_when_none(): + ts = parse_date(None) + assert isinstance(ts, pd.Timestamp) + assert ts.tz is not None diff --git a/tests/test_decorators.py b/tests/test_decorators.py new file mode 100644 index 0000000..4c9db08 --- /dev/null +++ b/tests/test_decorators.py @@ -0,0 +1,309 @@ +"""Tests for decorators module.""" + +import logging + +import pandas as pd + +from tinygrid.utils.decorators import support_date_range + + +class MockClient: + """Mock client for testing decorators.""" + + def __init__(self): + self.call_count = 0 + self.calls = [] + + @support_date_range(freq=None) + def no_chunking( + self, + start: str | pd.Timestamp = "today", + end: str | pd.Timestamp | None = None, + **kwargs, + ) -> pd.DataFrame: + """Method with no chunking.""" + self.call_count += 1 + self.calls.append((start, end, kwargs)) + return pd.DataFrame({"value": [1, 2, 3]}) + + @support_date_range(freq="7D") + def with_chunking_7days( + self, + start: str | pd.Timestamp = "today", + end: str | pd.Timestamp | None = None, + **kwargs, + ) -> pd.DataFrame: + """Method with 7-day chunking.""" + self.call_count += 1 + self.calls.append((start, end, kwargs)) + return pd.DataFrame({"date": [start], "value": [self.call_count]}) + + @support_date_range(freq="1D") + def with_chunking_1day( + self, + start: str | pd.Timestamp = "today", + end: str | pd.Timestamp | None = None, + **kwargs, + ) -> pd.DataFrame: + """Method with 1-day chunking.""" + self.call_count += 1 + self.calls.append((start, end, kwargs)) + return pd.DataFrame({"date": [start], "value": [self.call_count]}) + + @support_date_range(freq="7D") + def with_empty_results( + self, + start: str | pd.Timestamp = "today", + end: str | pd.Timestamp | None = None, + **kwargs, + ) -> pd.DataFrame: + """Method that returns empty DataFrames.""" + self.call_count += 1 + self.calls.append((start, end, kwargs)) + return pd.DataFrame() + + @support_date_range(freq="7D") + def with_errors( + self, + start: str | pd.Timestamp = "today", + end: str | pd.Timestamp | None = None, + **kwargs, + ) -> pd.DataFrame: + """Method that raises errors for certain chunks.""" + self.call_count += 1 + self.calls.append((start, end, kwargs)) + if self.call_count == 2: + raise RuntimeError("Simulated error for chunk 2") + return pd.DataFrame({"date": [start], "value": [self.call_count]}) + + +class TestSupportDateRangeNoChunking: + """Test support_date_range decorator without chunking.""" + + def test_no_chunking_single_call(self): + """Test that no chunking mode calls function once.""" + client = MockClient() + + result = client.no_chunking(start="2024-01-01", end="2024-01-31") + + assert client.call_count == 1 + assert len(result) == 3 + assert list(result["value"]) == [1, 2, 3] + + def test_no_chunking_default_end(self): + """Test no chunking with default end parameter.""" + client = MockClient() + + result = client.no_chunking(start="2024-01-01") + + assert client.call_count == 1 + assert len(result) == 3 + + def test_no_chunking_preserves_kwargs(self): + """Test that kwargs are passed through correctly.""" + client = MockClient() + + result = client.no_chunking( + start="2024-01-01", end="2024-01-31", extra_param="value" + ) + + assert client.call_count == 1 + assert client.calls[0][2]["extra_param"] == "value" + + +class TestSupportDateRangeWithChunking: + """Test support_date_range decorator with chunking.""" + + def test_chunking_small_range_no_split(self): + """Test that small date ranges don't get chunked.""" + client = MockClient() + + # 3 days should not be chunked with 7D freq + result = client.with_chunking_7days(start="2024-01-01", end="2024-01-03") + + assert client.call_count == 1 + + def test_chunking_large_range_splits(self): + """Test that large date ranges get chunked.""" + client = MockClient() + + # 20 days with 7D freq should create 3 chunks + result = client.with_chunking_7days(start="2024-01-01", end="2024-01-21") + + assert client.call_count == 3 + assert len(result) == 3 # One row per chunk + + def test_chunking_1day_freq(self): + """Test chunking with 1-day frequency.""" + client = MockClient() + + # 3 days with 1D freq should create 3 chunks + result = client.with_chunking_1day(start="2024-01-01", end="2024-01-04") + + assert client.call_count == 3 + assert len(result) == 3 + + def test_chunking_concatenates_results(self): + """Test that chunked results are concatenated correctly.""" + client = MockClient() + + result = client.with_chunking_7days(start="2024-01-01", end="2024-01-21") + + # Should have 3 chunks with sequential values + assert len(result) == 3 + assert list(result["value"]) == [1, 2, 3] + + def test_chunking_preserves_kwargs(self): + """Test that kwargs are passed to each chunk.""" + client = MockClient() + + result = client.with_chunking_7days( + start="2024-01-01", end="2024-01-21", extra_param="test_value" + ) + + assert client.call_count == 3 + # Check that each chunk received the kwargs + for call in client.calls: + assert call[2]["extra_param"] == "test_value" + + def test_chunking_empty_results(self): + """Test handling of empty DataFrames in chunks.""" + client = MockClient() + + result = client.with_empty_results(start="2024-01-01", end="2024-01-21") + + # Should call multiple times but return empty DataFrame + assert client.call_count == 3 + assert len(result) == 0 + assert isinstance(result, pd.DataFrame) + + +class TestSupportDateRangeErrorHandling: + """Test error handling in support_date_range decorator.""" + + def test_chunking_continues_on_error(self, caplog): + """Test that errors in one chunk don't stop other chunks.""" + client = MockClient() + + with caplog.at_level(logging.WARNING): + result = client.with_errors(start="2024-01-01", end="2024-01-21") + + # Should have attempted 3 chunks + assert client.call_count == 3 + # Should only have 2 results (chunk 2 failed) + assert len(result) == 2 + assert list(result["value"]) == [1, 3] + # Should have logged the error + assert "Failed to fetch chunk" in caplog.text + + def test_chunking_all_errors_returns_empty(self): + """Test that if all chunks error, returns empty DataFrame.""" + + class AlwaysErrorClient: + @support_date_range(freq="7D") + def always_errors( + self, + start: str | pd.Timestamp = "today", + end: str | pd.Timestamp | None = None, + **kwargs, + ) -> pd.DataFrame: + raise RuntimeError("Always fails") + + client = AlwaysErrorClient() + result = client.always_errors(start="2024-01-01", end="2024-01-21") + + assert len(result) == 0 + assert isinstance(result, pd.DataFrame) + + +class TestSupportDateRangeDateParsing: + """Test date parsing in support_date_range decorator.""" + + def test_parses_string_dates(self): + """Test that string dates are parsed correctly.""" + client = MockClient() + + result = client.no_chunking(start="2024-01-01", end="2024-01-31") + + # Check that dates were converted to Timestamps + assert isinstance(client.calls[0][0], pd.Timestamp) + assert isinstance(client.calls[0][1], pd.Timestamp) + + def test_parses_timestamp_dates(self): + """Test that Timestamp dates are handled correctly.""" + client = MockClient() + start_ts = pd.Timestamp("2024-01-01") + end_ts = pd.Timestamp("2024-01-31") + + result = client.no_chunking(start=start_ts, end=end_ts) + + assert client.call_count == 1 + + def test_parses_today_keyword(self): + """Test that 'today' keyword is parsed.""" + client = MockClient() + + result = client.no_chunking(start="today") + + assert client.call_count == 1 + assert isinstance(client.calls[0][0], pd.Timestamp) + + def test_default_end_is_parsed(self): + """Test that default end parameter is handled correctly.""" + client = MockClient() + + result = client.no_chunking(start="2024-01-01", end=None) + + assert client.call_count == 1 + # End should be defaulted to start + 1 day + assert isinstance(client.calls[0][1], pd.Timestamp) + + +class TestSupportDateRangeEdgeCases: + """Test edge cases for support_date_range decorator.""" + + def test_exact_chunk_boundary(self): + """Test date range that's exactly on chunk boundary.""" + client = MockClient() + + # Exactly 7 days with 7D freq should create 1 chunk (<=) + result = client.with_chunking_7days(start="2024-01-01", end="2024-01-08") + + assert client.call_count == 1 + + def test_one_day_over_boundary(self): + """Test date range that's one day over chunk boundary.""" + client = MockClient() + + # 8 days with 7D freq should create 2 chunks + result = client.with_chunking_7days(start="2024-01-01", end="2024-01-09") + + assert client.call_count == 2 + + def test_preserves_function_metadata(self): + """Test that decorator preserves function metadata.""" + client = MockClient() + + # Check that functools.wraps preserved the metadata + assert client.no_chunking.__name__ == "no_chunking" + assert "Method with no chunking" in client.no_chunking.__doc__ # pyright: ignore[reportOperatorIssue] + + def test_with_args(self): + """Test that positional args are passed through.""" + + class ClientWithArgs: + @support_date_range(freq=None) + def method_with_args( + self, + start: str | pd.Timestamp = "today", + end: str | pd.Timestamp | None = None, + *args, + **kwargs, + ) -> pd.DataFrame: + return pd.DataFrame({"args": list(args)}) + + client = ClientWithArgs() + result = client.method_with_args("2024-01-01", "2024-01-31", "extra_arg") + + # Extra positional arg should be passed through + assert "extra_arg" in result["args"].tolist() diff --git a/tests/test_ercot_auth.py b/tests/test_ercot_auth.py new file mode 100644 index 0000000..cbde16a --- /dev/null +++ b/tests/test_ercot_auth.py @@ -0,0 +1,615 @@ +"""Tests for ERCOT authentication module.""" + +import time + +import httpx +import pytest +import respx + +from tinygrid.auth import ERCOTAuth, ERCOTAuthConfig +from tinygrid.errors import GridAuthenticationError + + +class TestERCOTAuthConfig: + """Test ERCOTAuthConfig configuration.""" + + def test_auth_config_defaults(self): + """Test that ERCOTAuthConfig has correct default values.""" + config = ERCOTAuthConfig( + username="test@example.com", + password="password123", + subscription_key="key123", + ) + + assert config.username == "test@example.com" + assert config.password == "password123" + assert config.subscription_key == "key123" + assert ( + config.auth_url + == "https://ercotb2c.b2clogin.com/ercotb2c.onmicrosoft.com/B2C_1_PUBAPI-ROPC-FLOW/oauth2/v2.0/token" + ) + assert config.client_id == "fec253ea-0d06-4272-a5e6-b478baeecd70" + assert config.token_cache_ttl == 3300 + + def test_auth_config_custom_values(self): + """Test ERCOTAuthConfig with custom values.""" + config = ERCOTAuthConfig( + username="custom@example.com", + password="custompass", + subscription_key="customkey", + auth_url="https://custom.auth.url", + client_id="custom-client-id", + token_cache_ttl=1800, + ) + + assert config.username == "custom@example.com" + assert config.password == "custompass" + assert config.subscription_key == "customkey" + assert config.auth_url == "https://custom.auth.url" + assert config.client_id == "custom-client-id" + assert config.token_cache_ttl == 1800 + + +class TestERCOTAuthTokenCaching: + """Test ERCOT authentication token caching logic.""" + + def test_is_token_valid_no_token(self): + """Test that _is_token_valid returns False when no token is cached.""" + config = ERCOTAuthConfig( + username="test@example.com", + password="password123", + subscription_key="key123", + ) + auth = ERCOTAuth(config) + + assert auth._is_token_valid() is False + + def test_is_token_valid_expired_token(self): + """Test that _is_token_valid returns False when token is expired.""" + config = ERCOTAuthConfig( + username="test@example.com", + password="password123", + subscription_key="key123", + ) + auth = ERCOTAuth(config) + auth._cached_token = "expired-token" + auth._token_expires_at = time.time() - 100 # Expired 100 seconds ago + + assert auth._is_token_valid() is False + + def test_is_token_valid_current_token(self): + """Test that _is_token_valid returns True when token is still valid.""" + config = ERCOTAuthConfig( + username="test@example.com", + password="password123", + subscription_key="key123", + ) + auth = ERCOTAuth(config) + auth._cached_token = "valid-token" + auth._token_expires_at = time.time() + 1000 # Expires in 1000 seconds + + assert auth._is_token_valid() is True + + def test_clear_token_cache(self): + """Test that clear_token_cache clears the cached token.""" + config = ERCOTAuthConfig( + username="test@example.com", + password="password123", + subscription_key="key123", + ) + auth = ERCOTAuth(config) + auth._cached_token = "valid-token" + auth._token_expires_at = time.time() + 1000 + + auth.clear_token_cache() + + assert auth._cached_token is None + assert auth._token_expires_at is None + + def test_get_subscription_key(self): + """Test that get_subscription_key returns the subscription key.""" + config = ERCOTAuthConfig( + username="test@example.com", + password="password123", + subscription_key="my-sub-key", + ) + auth = ERCOTAuth(config) + + assert auth.get_subscription_key() == "my-sub-key" + + +class TestERCOTAuthSyncTokenFetch: + """Test ERCOT authentication synchronous token fetching.""" + + @respx.mock + def test_fetch_token_sync_success_access_token(self): + """Test successful token fetch with access_token in response.""" + config = ERCOTAuthConfig( + username="test@example.com", + password="password123", + subscription_key="key123", + ) + auth = ERCOTAuth(config) + + # Mock the authentication endpoint + respx.post(url__regex=r".*ercotb2c\.b2clogin\.com.*").mock( + return_value=httpx.Response(200, json={"access_token": "test-access-token"}) + ) + + token = auth._fetch_token_sync() + + assert token == "test-access-token" + + @respx.mock + def test_fetch_token_sync_success_id_token(self): + """Test successful token fetch with id_token in response.""" + config = ERCOTAuthConfig( + username="test@example.com", + password="password123", + subscription_key="key123", + ) + auth = ERCOTAuth(config) + + # Mock the authentication endpoint + respx.post(url__regex=r".*ercotb2c\.b2clogin\.com.*").mock( + return_value=httpx.Response(200, json={"id_token": "test-id-token"}) + ) + + token = auth._fetch_token_sync() + + assert token == "test-id-token" + + @respx.mock + def test_fetch_token_sync_http_error(self): + """Test token fetch with HTTP error response.""" + config = ERCOTAuthConfig( + username="test@example.com", + password="password123", + subscription_key="key123", + ) + auth = ERCOTAuth(config) + + # Mock a 401 unauthorized response + respx.post(url__regex=r".*ercotb2c\.b2clogin\.com.*").mock( + return_value=httpx.Response( + 401, + json={ + "error": "invalid_credentials", + "error_description": "Invalid username or password", + }, + ) + ) + + with pytest.raises(GridAuthenticationError) as exc_info: + auth._fetch_token_sync() + + assert exc_info.value.status_code == 401 + assert "ERCOT authentication failed with status 401" in str(exc_info.value) + + @respx.mock + def test_fetch_token_sync_invalid_json(self): + """Test token fetch with invalid JSON response.""" + config = ERCOTAuthConfig( + username="test@example.com", + password="password123", + subscription_key="key123", + ) + auth = ERCOTAuth(config) + + # Mock a response with invalid JSON + respx.post(url__regex=r".*ercotb2c\.b2clogin\.com.*").mock( + return_value=httpx.Response(200, text="invalid json response") + ) + + with pytest.raises(GridAuthenticationError) as exc_info: + auth._fetch_token_sync() + + assert "Failed to parse authentication response as JSON" in str(exc_info.value) + + @respx.mock + def test_fetch_token_sync_no_token_in_response(self): + """Test token fetch when response has no access_token or id_token.""" + config = ERCOTAuthConfig( + username="test@example.com", + password="password123", + subscription_key="key123", + ) + auth = ERCOTAuth(config) + + # Mock a response without token + respx.post(url__regex=r".*ercotb2c\.b2clogin\.com.*").mock( + return_value=httpx.Response(200, json={"some_field": "some_value"}) + ) + + with pytest.raises(GridAuthenticationError) as exc_info: + auth._fetch_token_sync() + + assert "No access token in ERCOT authentication response" in str(exc_info.value) + + @respx.mock + def test_fetch_token_sync_timeout(self): + """Test token fetch with timeout.""" + config = ERCOTAuthConfig( + username="test@example.com", + password="password123", + subscription_key="key123", + ) + auth = ERCOTAuth(config) + + # Mock a timeout + respx.post(url__regex=r".*ercotb2c\.b2clogin\.com.*").mock( + side_effect=httpx.TimeoutException("Timeout") + ) + + with pytest.raises(GridAuthenticationError) as exc_info: + auth._fetch_token_sync() + + assert "ERCOT authentication request timed out" in str(exc_info.value) + + @respx.mock + def test_fetch_token_sync_request_error(self): + """Test token fetch with request error.""" + config = ERCOTAuthConfig( + username="test@example.com", + password="password123", + subscription_key="key123", + ) + auth = ERCOTAuth(config) + + # Mock a request error + respx.post(url__regex=r".*ercotb2c\.b2clogin\.com.*").mock( + side_effect=httpx.ConnectError("Connection failed") + ) + + with pytest.raises(GridAuthenticationError) as exc_info: + auth._fetch_token_sync() + + assert "ERCOT authentication request failed" in str(exc_info.value) + + @respx.mock + def test_fetch_token_sync_unexpected_error(self): + """Test token fetch with unexpected error.""" + config = ERCOTAuthConfig( + username="test@example.com", + password="password123", + subscription_key="key123", + ) + auth = ERCOTAuth(config) + + # Mock an unexpected error + respx.post(url__regex=r".*ercotb2c\.b2clogin\.com.*").mock( + side_effect=RuntimeError("Unexpected error") + ) + + with pytest.raises(GridAuthenticationError) as exc_info: + auth._fetch_token_sync() + + assert "Unexpected error during ERCOT authentication" in str(exc_info.value) + + +class TestERCOTAuthAsyncTokenFetch: + """Test ERCOT authentication asynchronous token fetching.""" + + @respx.mock + @pytest.mark.asyncio + async def test_fetch_token_async_success_access_token(self): + """Test successful async token fetch with access_token in response.""" + config = ERCOTAuthConfig( + username="test@example.com", + password="password123", + subscription_key="key123", + ) + auth = ERCOTAuth(config) + + # Mock the authentication endpoint + respx.post(url__regex=r".*ercotb2c\.b2clogin\.com.*").mock( + return_value=httpx.Response( + 200, json={"access_token": "test-access-token-async"} + ) + ) + + token = await auth._fetch_token_async() + + assert token == "test-access-token-async" + + @respx.mock + @pytest.mark.asyncio + async def test_fetch_token_async_success_id_token(self): + """Test successful async token fetch with id_token in response.""" + config = ERCOTAuthConfig( + username="test@example.com", + password="password123", + subscription_key="key123", + ) + auth = ERCOTAuth(config) + + # Mock the authentication endpoint + respx.post(url__regex=r".*ercotb2c\.b2clogin\.com.*").mock( + return_value=httpx.Response(200, json={"id_token": "test-id-token-async"}) + ) + + token = await auth._fetch_token_async() + + assert token == "test-id-token-async" + + @respx.mock + @pytest.mark.asyncio + async def test_fetch_token_async_http_error(self): + """Test async token fetch with HTTP error response.""" + config = ERCOTAuthConfig( + username="test@example.com", + password="password123", + subscription_key="key123", + ) + auth = ERCOTAuth(config) + + # Mock a 401 unauthorized response + respx.post(url__regex=r".*ercotb2c\.b2clogin\.com.*").mock( + return_value=httpx.Response( + 401, + json={ + "error": "invalid_credentials", + "error_description": "Invalid username or password", + }, + ) + ) + + with pytest.raises(GridAuthenticationError) as exc_info: + await auth._fetch_token_async() + + assert exc_info.value.status_code == 401 + assert "ERCOT authentication failed with status 401" in str(exc_info.value) + + @respx.mock + @pytest.mark.asyncio + async def test_fetch_token_async_invalid_json(self): + """Test async token fetch with invalid JSON response.""" + config = ERCOTAuthConfig( + username="test@example.com", + password="password123", + subscription_key="key123", + ) + auth = ERCOTAuth(config) + + # Mock a response with invalid JSON + respx.post(url__regex=r".*ercotb2c\.b2clogin\.com.*").mock( + return_value=httpx.Response(200, text="invalid json response") + ) + + with pytest.raises(GridAuthenticationError) as exc_info: + await auth._fetch_token_async() + + assert "Failed to parse authentication response as JSON" in str(exc_info.value) + + @respx.mock + @pytest.mark.asyncio + async def test_fetch_token_async_no_token_in_response(self): + """Test async token fetch when response has no access_token or id_token.""" + config = ERCOTAuthConfig( + username="test@example.com", + password="password123", + subscription_key="key123", + ) + auth = ERCOTAuth(config) + + # Mock a response without token + respx.post(url__regex=r".*ercotb2c\.b2clogin\.com.*").mock( + return_value=httpx.Response(200, json={"some_field": "some_value"}) + ) + + with pytest.raises(GridAuthenticationError) as exc_info: + await auth._fetch_token_async() + + assert "No access token in ERCOT authentication response" in str(exc_info.value) + + @respx.mock + @pytest.mark.asyncio + async def test_fetch_token_async_timeout(self): + """Test async token fetch with timeout.""" + config = ERCOTAuthConfig( + username="test@example.com", + password="password123", + subscription_key="key123", + ) + auth = ERCOTAuth(config) + + # Mock a timeout + respx.post(url__regex=r".*ercotb2c\.b2clogin\.com.*").mock( + side_effect=httpx.TimeoutException("Timeout") + ) + + with pytest.raises(GridAuthenticationError) as exc_info: + await auth._fetch_token_async() + + assert "ERCOT authentication request timed out" in str(exc_info.value) + + @respx.mock + @pytest.mark.asyncio + async def test_fetch_token_async_request_error(self): + """Test async token fetch with request error.""" + config = ERCOTAuthConfig( + username="test@example.com", + password="password123", + subscription_key="key123", + ) + auth = ERCOTAuth(config) + + # Mock a request error + respx.post(url__regex=r".*ercotb2c\.b2clogin\.com.*").mock( + side_effect=httpx.ConnectError("Connection failed") + ) + + with pytest.raises(GridAuthenticationError) as exc_info: + await auth._fetch_token_async() + + assert "ERCOT authentication request failed" in str(exc_info.value) + + @respx.mock + @pytest.mark.asyncio + async def test_fetch_token_async_unexpected_error(self): + """Test async token fetch with unexpected error.""" + config = ERCOTAuthConfig( + username="test@example.com", + password="password123", + subscription_key="key123", + ) + auth = ERCOTAuth(config) + + # Mock an unexpected error + respx.post(url__regex=r".*ercotb2c\.b2clogin\.com.*").mock( + side_effect=RuntimeError("Unexpected error") + ) + + with pytest.raises(GridAuthenticationError) as exc_info: + await auth._fetch_token_async() + + assert "Unexpected error during ERCOT authentication" in str(exc_info.value) + + +class TestERCOTAuthGetToken: + """Test ERCOT authentication get_token methods.""" + + @respx.mock + def test_get_token_fetches_new_token(self): + """Test that get_token fetches a new token when cache is empty.""" + config = ERCOTAuthConfig( + username="test@example.com", + password="password123", + subscription_key="key123", + ) + auth = ERCOTAuth(config) + + # Mock the authentication endpoint + respx.post(url__regex=r".*ercotb2c\.b2clogin\.com.*").mock( + return_value=httpx.Response(200, json={"access_token": "new-token"}) + ) + + token = auth.get_token() + + assert token == "new-token" + assert auth._cached_token == "new-token" + assert auth._token_expires_at is not None + + @respx.mock + def test_get_token_uses_cached_token(self): + """Test that get_token uses cached token when valid.""" + config = ERCOTAuthConfig( + username="test@example.com", + password="password123", + subscription_key="key123", + ) + auth = ERCOTAuth(config) + + # Set a valid cached token + auth._cached_token = "cached-token" + auth._token_expires_at = time.time() + 1000 + + # Mock should not be called + respx.post(url__regex=r".*ercotb2c\.b2clogin\.com.*").mock( + return_value=httpx.Response( + 200, json={"access_token": "should-not-be-called"} + ) + ) + + token = auth.get_token() + + assert token == "cached-token" + + @respx.mock + def test_get_token_refreshes_expired_token(self): + """Test that get_token fetches a new token when cached token is expired.""" + config = ERCOTAuthConfig( + username="test@example.com", + password="password123", + subscription_key="key123", + ) + auth = ERCOTAuth(config) + + # Set an expired cached token + auth._cached_token = "expired-token" + auth._token_expires_at = time.time() - 100 + + # Mock the authentication endpoint + respx.post(url__regex=r".*ercotb2c\.b2clogin\.com.*").mock( + return_value=httpx.Response(200, json={"access_token": "refreshed-token"}) + ) + + token = auth.get_token() + + assert token == "refreshed-token" + assert auth._cached_token == "refreshed-token" + + @respx.mock + @pytest.mark.asyncio + async def test_get_token_async_fetches_new_token(self): + """Test that get_token_async fetches a new token when cache is empty.""" + config = ERCOTAuthConfig( + username="test@example.com", + password="password123", + subscription_key="key123", + ) + auth = ERCOTAuth(config) + + # Mock the authentication endpoint + respx.post(url__regex=r".*ercotb2c\.b2clogin\.com.*").mock( + return_value=httpx.Response(200, json={"access_token": "new-async-token"}) + ) + + token = await auth.get_token_async() + + assert token == "new-async-token" + assert auth._cached_token == "new-async-token" + assert auth._token_expires_at is not None + + @respx.mock + @pytest.mark.asyncio + async def test_get_token_async_uses_cached_token(self): + """Test that get_token_async uses cached token when valid.""" + config = ERCOTAuthConfig( + username="test@example.com", + password="password123", + subscription_key="key123", + ) + auth = ERCOTAuth(config) + + # Set a valid cached token + auth._cached_token = "cached-async-token" + auth._token_expires_at = time.time() + 1000 + + # Mock should not be called + respx.post(url__regex=r".*ercotb2c\.b2clogin\.com.*").mock( + return_value=httpx.Response( + 200, json={"access_token": "should-not-be-called"} + ) + ) + + token = await auth.get_token_async() + + assert token == "cached-async-token" + + @respx.mock + @pytest.mark.asyncio + async def test_get_token_async_refreshes_expired_token(self): + """Test that get_token_async fetches a new token when cached token is expired.""" + config = ERCOTAuthConfig( + username="test@example.com", + password="password123", + subscription_key="key123", + ) + auth = ERCOTAuth(config) + + # Set an expired cached token + auth._cached_token = "expired-async-token" + auth._token_expires_at = time.time() - 100 + + # Mock the authentication endpoint + respx.post(url__regex=r".*ercotb2c\.b2clogin\.com.*").mock( + return_value=httpx.Response( + 200, json={"access_token": "refreshed-async-token"} + ) + ) + + token = await auth.get_token_async() + + assert token == "refreshed-async-token" + assert auth._cached_token == "refreshed-async-token" diff --git a/tests/test_ercot_endpoint_coverage.py b/tests/test_ercot_endpoint_coverage.py new file mode 100644 index 0000000..57f7a11 --- /dev/null +++ b/tests/test_ercot_endpoint_coverage.py @@ -0,0 +1,140 @@ +"""Parametrized tests to cover ERCOT endpoint wrapper methods.""" + +from unittest.mock import patch + +import pandas as pd +import pytest + +from tinygrid import ERCOT + +# List of simple endpoint wrapper methods that just call _call_endpoint +SIMPLE_ENDPOINT_METHODS = [ + # Aggregated endpoints + "get_aggregated_dsr_loads", + "get_aggregated_generation_summary", + "get_aggregated_generation_summary_houston", + "get_aggregated_generation_summary_north", + "get_aggregated_generation_summary_south", + "get_aggregated_generation_summary_west", + "get_aggregated_load_summary", + "get_aggregated_load_summary_houston", + "get_aggregated_load_summary_north", + "get_aggregated_load_summary_south", + "get_aggregated_load_summary_west", + "get_aggregated_outage_schedule", + "get_aggregated_outage_schedule_houston", + "get_aggregated_outage_schedule_north", + "get_aggregated_outage_schedule_south", + "get_aggregated_outage_schedule_west", + # AS Offers + "get_aggregated_as_offers_ecrsm", + "get_aggregated_as_offers_ecrss", + "get_aggregated_as_offers_offns", + "get_aggregated_as_offers_onns", + "get_aggregated_as_offers_regdn", + "get_aggregated_as_offers_regup", + "get_aggregated_as_offers_rrsffr", + "get_aggregated_as_offers_rrspfr", + "get_aggregated_as_offers_rrsufr", + # SCED Disclosure + "get_sced_dsr_load_data", + "get_sced_gen_res_data", + "get_sced_smne_gen_res", + "get_load_res_data_in_sced", + "get_hdl_ldl_manual_override", + "get_sced_qse_self_arranged_as", + # SASM + "get_sasm_gen_res_as_offers", + "get_sasm_gen_res_as_offer_awards", + "get_sasm_load_res_as_offers", + "get_sasm_load_res_as_offer_awards", + # Price Corrections + "get_rtm_price_corrections_eblmp", + "get_rtm_price_corrections_spp", + "get_rtm_price_corrections_shadow", + "get_rtm_price_corrections_soglmp", + "get_rtm_price_corrections_sogprice", + "get_rtm_price_corrections_splmp", + "get_dam_price_corrections_eblmp", + "get_dam_price_corrections_mcpc", + "get_dam_price_corrections_spp", + # Cleared AS + "get_cleared_dam_as_ecrsm", + "get_cleared_dam_as_ecrss", + "get_cleared_dam_as_nspin", + "get_cleared_dam_as_regdn", + "get_cleared_dam_as_regup", + "get_cleared_dam_as_rrsffr", + "get_cleared_dam_as_rrspfr", + "get_cleared_dam_as_rrsufr", + # Self-arranged AS + "get_self_arranged_as_ecrsm", + "get_self_arranged_as_ecrss", + "get_self_arranged_as_nspin", + "get_self_arranged_as_nspnm", + "get_self_arranged_as_regdn", + "get_self_arranged_as_regup", + "get_self_arranged_as_rrsffr", + "get_self_arranged_as_rrspfr", + "get_self_arranged_as_rrsufr", + # DAM Disclosure + "get_dam_energy_bid_awards", + "get_dam_energy_bids", + "get_dam_energy_only_offer_awards", + "get_dam_energy_only_offers", + "get_dam_gen_res_as_offers", + "get_dam_gen_res_data", + "get_dam_load_res_as_offers", + "get_dam_load_res_data", + "get_dam_ptp_obl_bid_awards", + "get_dam_ptp_obl_bids", + "get_dam_ptp_obl_opt", + "get_dam_ptp_obl_opt_awards", + "get_dam_qse_self_as", + # Load and forecasting + "get_actual_system_load_by_weather_zone", + "get_actual_system_load_by_forecast_zone", + "get_wpp_hourly_average_actual_forecast", + "get_wpp_actual_5min_avg_values", + "get_wpp_hourly_actual_forecast_geo", + "get_wpp_actual_5min_avg_values_geo", + "get_spp_hourly_average_actual_forecast", + "get_spp_actual_5min_avg_values", + "get_spp_hourly_actual_forecast_geo", + "get_spp_actual_5min_avg_values_geo", + # Pricing + "get_dam_settlement_point_prices", + "get_dam_shadow_prices", + "get_dam_clear_price_for_cap", + "get_dam_system_lambda", + "get_sced_system_lambda", + "get_lmp_electrical_bus", + "get_lmp_node_zone_hub", + "get_spp_node_zone_hub", + "get_rtd_lmp_node_zone_hub", + "get_shadow_prices_bound_transmission_constraint", + # Other + "get_hourly_res_outage_cap", + "get_total_as_service_offers", + "get_load_distribution_factors", +] + + +class TestSimpleEndpointWrappers: + """Test simple endpoint wrapper methods using parametrization.""" + + @pytest.mark.parametrize("method_name", SIMPLE_ENDPOINT_METHODS) + @patch.object(ERCOT, "_call_endpoint") + def test_endpoint_wrapper(self, mock_call_endpoint, method_name): + """Test that endpoint wrapper methods call _call_endpoint and return DataFrame.""" + # Setup mock + mock_call_endpoint.return_value = pd.DataFrame({"test": [1, 2, 3]}) + + # Create client and call method + ercot = ERCOT() + method = getattr(ercot, method_name) + result = method() + + # Verify + assert isinstance(result, pd.DataFrame) + mock_call_endpoint.assert_called_once() diff --git a/tests/test_ercot_helpers.py b/tests/test_ercot_helpers.py new file mode 100644 index 0000000..6c9b55a --- /dev/null +++ b/tests/test_ercot_helpers.py @@ -0,0 +1,488 @@ +from unittest.mock import MagicMock + +import pandas as pd +import pytest +from pyercot.errors import UnexpectedStatus + +from tinygrid.constants.ercot import LOAD_ZONES, TRADING_HUBS, LocationType, Market +from tinygrid.ercot import ERCOT +from tinygrid.errors import GridAPIError, GridAuthenticationError, GridTimeoutError + + +def make_ercot(**kwargs: object) -> ERCOT: + """Create an ERCOT instance with sensible defaults for unit tests.""" + return ERCOT(auth=kwargs.get("auth")) # type: ignore[arg-type] + + +def test_get_client_wraps_unexpected_auth_error() -> None: + auth = MagicMock() + auth.get_token.side_effect = ValueError("boom") + auth.get_subscription_key.return_value = "sub" + + client = make_ercot(auth=auth) + + with pytest.raises(GridAuthenticationError): + client._get_client() + + auth.get_subscription_key.assert_not_called() + + +def test_get_client_creates_authenticated_client( + monkeypatch: pytest.MonkeyPatch, +) -> None: + created: dict[str, object] = {} + + class DummyAuthenticatedClient: + def __init__( + self, base_url, token, timeout, verify_ssl, raise_on_unexpected_status + ): # type: ignore[no-untyped-def] + created["token"] = token + created["base_url"] = base_url + created["headers"] = {} + self.token = token + + def with_headers(self, headers): + created["headers"] = headers + return self + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + auth = MagicMock() + auth.get_token.return_value = "token123" + auth.get_subscription_key.return_value = "subkey" + + monkeypatch.setattr("tinygrid.ercot.AuthenticatedClient", DummyAuthenticatedClient) + + client = make_ercot(auth=auth) + result = client._get_client() + + assert isinstance(result, DummyAuthenticatedClient) + assert created["headers"] == {"Ocp-Apim-Subscription-Key": "subkey"} + + +def test_get_client_refreshes_token_and_closes_old( + monkeypatch: pytest.MonkeyPatch, +) -> None: + closed = {"value": False} + + class DummyAuthenticatedClient: + def __init__( + self, + base_url=None, + token=None, + timeout=None, + verify_ssl=None, + raise_on_unexpected_status=None, + ): # type: ignore[no-untyped-def] + self.token = token + + def with_headers(self, headers): + return self + + def __exit__(self, exc_type, exc, tb): + closed["value"] = True + return False + + auth = MagicMock() + auth.get_token.return_value = "new" + auth.get_subscription_key.return_value = "sub" + + monkeypatch.setattr("tinygrid.ercot.AuthenticatedClient", DummyAuthenticatedClient) + + client = make_ercot(auth=auth) + client._client = DummyAuthenticatedClient(token="old") + + result = client._get_client() + + assert isinstance(result, DummyAuthenticatedClient) + assert closed["value"] is True + assert result.token == "new" + + +def test_handle_api_error_wraps_exceptions() -> None: + client = make_ercot() + unexpected = UnexpectedStatus(status_code=500, content=b"err") + + with pytest.raises(GridAPIError): + client._handle_api_error(unexpected, endpoint="/test") + + with pytest.raises(GridTimeoutError): + client._handle_api_error(TimeoutError(), endpoint="/test") + + +def test_flatten_dict_handles_nested_lists_and_nulls() -> None: + client = make_ercot() + data = { + "meta": {"id": 1, "name": "x"}, + "values": [{"foo": "bar"}, {"foo": "baz"}], + "tags": ["a", "b", "c"], + "_links": {"self": "skip"}, + "none_val": None, + } + + flattened = client._flatten_dict_for_dataframe(data) + + assert flattened["meta.id"] == 1 + assert flattened["values_count"] == 2 + assert "values" in flattened and "skip" not in str(flattened) + assert flattened["tags"] == "a, b, c" + assert "none_val" in flattened and flattened["none_val"] is None + + +def test_products_to_dataframe_handles_additional_properties() -> None: + client = make_ercot() + response = { + "additional_properties": { + "_embedded": { + "products": [ + { + "emilId": "np1", + "name": "test", + "details": {"nested": 1}, + } + ] + } + } + } + + df = client._products_to_dataframe(response) + + assert not df.empty + assert next(iter(df["emilId"])) == "np1" + assert df.filter(like="details.nested").iloc[0, 0] == 1 + + +def test_product_history_to_dataframe_expands_archives() -> None: + client = make_ercot() + response = { + "emilId": "np1", + "archives": [ + {"version": 1, "size": 10}, + {"version": 2, "size": 20}, + ], + } + + df = client._product_history_to_dataframe(response) + + assert len(df) == 2 + assert set(df["version"].tolist()) == {1, 2} + assert df["emilId"].iloc[0] == "np1" + + +def test_filter_by_location_excludes_zones_for_resource_nodes() -> None: + client = make_ercot() + df = pd.DataFrame( + { + "Settlement Point": ["HB_HOUSTON", "CUSTOM1", "LZ_NORTH", "CUSTOM2"], + "val": [1, 2, 3, 4], + } + ) + + filtered = client._filter_by_location( + df, + location_type=LocationType.RESOURCE_NODE, + location_column="Settlement Point", + ) + + assert set(filtered["Settlement Point"]) == {"CUSTOM1", "CUSTOM2"} + assert all( + pt not in LOAD_ZONES + TRADING_HUBS for pt in filtered["Settlement Point"] + ) + + +def test_filter_by_location_matches_allowed_types() -> None: + client = make_ercot() + df = pd.DataFrame( + { + "Settlement Point": ["HB_HOUSTON", "LZ_SOUTH", "CUSTOM"], + "val": [1, 2, 3], + } + ) + + filtered = client._filter_by_location( + df, + location_type=[LocationType.TRADING_HUB, LocationType.LOAD_ZONE], + location_column="Settlement Point", + ) + + assert set(filtered["Settlement Point"]) == {"HB_HOUSTON", "LZ_SOUTH"} + + +def test_filter_by_date_handles_alternate_column_names() -> None: + client = make_ercot() + df = pd.DataFrame({"DeliveryDate": ["2024-01-01", "2024-01-03"]}) + + start = pd.Timestamp("2024-01-01") + end = pd.Timestamp("2024-01-02") + + filtered = client._filter_by_date(df, start, end, date_column="Delivery Date") + + assert len(filtered) == 1 + assert filtered.iloc[0]["DeliveryDate"] == "2024-01-01" + + +def test_add_time_columns_from_interval_fields() -> None: + client = make_ercot() + df = pd.DataFrame({"Date": ["2024-01-01"], "Hour": [1], "Interval": [2]}) + + result = client._add_time_columns(df.copy()) + + assert "Time" in result and "End Time" in result + assert result["Time"].dt.tz is not None + assert (result["End Time"] - result["Time"]).dt.total_seconds().iloc[0] == 900 + + +def test_add_time_columns_from_hour_ending_strings() -> None: + client = make_ercot() + df = pd.DataFrame({"Date": ["2024-01-01"], "Hour Ending": ["01:00"]}) + + result = client._add_time_columns(df.copy()) + + assert "Time" in result and "End Time" in result + assert result["Time"].dt.hour.iloc[0] == 0 + assert result["End Time"].dt.hour.iloc[0] == 1 + + +def test_get_shadow_prices_routes_to_archive_for_day_ahead( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class HistoricalERCOT(ERCOT): + def _needs_historical(self, start: pd.Timestamp, market: str) -> bool: # type: ignore[override] + return True + + def _filter_by_date(self, df: pd.DataFrame, *args, **kwargs) -> pd.DataFrame: # type: ignore[override] + return df + + def _standardize_columns(self, df: pd.DataFrame) -> pd.DataFrame: # type: ignore[override] + return df + + client = HistoricalERCOT() + calls: dict[str, object] = {} + + archive = MagicMock() + archive.fetch_historical.return_value = pd.DataFrame( + {"Delivery Date": ["2024-01-01"]} + ) + monkeypatch.setattr(client, "_get_archive", lambda: archive) + monkeypatch.setattr( + client, + "get_dam_shadow_prices", + lambda **kwargs: calls.setdefault("live", kwargs) or pd.DataFrame(), + ) + + df = client.get_shadow_prices( + start="2024-01-01", end="2024-01-02", market=Market.DAY_AHEAD_HOURLY + ) + + assert not df.empty + archive.fetch_historical.assert_called_once() + assert "live" not in calls + + +def test_get_load_uses_historical_weather_zone(monkeypatch: pytest.MonkeyPatch) -> None: + class HistoricalERCOT(ERCOT): + def _needs_historical(self, start: pd.Timestamp, market: str) -> bool: # type: ignore[override] + return True + + def _filter_by_date(self, df: pd.DataFrame, *args, **kwargs) -> pd.DataFrame: # type: ignore[override] + return df + + def _standardize_columns(self, df: pd.DataFrame) -> pd.DataFrame: # type: ignore[override] + return df + + client = HistoricalERCOT() + + archive = MagicMock() + archive.fetch_historical.return_value = pd.DataFrame( + {"Oper Day": ["2024-01-01"], "value": [1]} + ) + monkeypatch.setattr(client, "_get_archive", lambda: archive) + monkeypatch.setattr( + client, + "get_actual_system_load_by_weather_zone", + lambda **kwargs: pd.DataFrame(), + ) + + df = client.get_load(start="2024-01-01", end="2024-01-02", by="weather_zone") + + assert df["value"].iloc[0] == 1 + archive.fetch_historical.assert_called_once() + + +def test_get_shadow_prices_real_time_live_path(monkeypatch: pytest.MonkeyPatch) -> None: + class LiveERCOT(ERCOT): + def _needs_historical(self, start: pd.Timestamp, market: str) -> bool: # type: ignore[override] + return False + + def _filter_by_date(self, df: pd.DataFrame, *args, **kwargs) -> pd.DataFrame: # type: ignore[override] + return df + + def _standardize_columns(self, df: pd.DataFrame) -> pd.DataFrame: # type: ignore[override] + return df + + client = LiveERCOT() + monkeypatch.setattr( + client, + "get_shadow_prices_bound_transmission_constraint", + lambda **kwargs: pd.DataFrame({"Delivery Date": ["2024-01-01"]}), + ) + + df = client.get_shadow_prices( + start="2024-01-01", end="2024-01-02", market=Market.REAL_TIME_SCED + ) + + assert not df.empty + + +def test_get_wind_forecast_mixes_historical_and_live( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class MixedERCOT(ERCOT): + def _needs_historical(self, start: pd.Timestamp, market: str) -> bool: # type: ignore[override] + return market == "forecast" + + def _filter_by_date(self, df: pd.DataFrame, *args, **kwargs) -> pd.DataFrame: # type: ignore[override] + return df + + def _standardize_columns(self, df: pd.DataFrame) -> pd.DataFrame: # type: ignore[override] + return df + + client = MixedERCOT() + + archive = MagicMock() + archive.fetch_historical.return_value = pd.DataFrame( + {"Posted Datetime": ["2024-01-01"]} + ) + monkeypatch.setattr(client, "_get_archive", lambda: archive) + monkeypatch.setattr( + client, + "get_wpp_hourly_actual_forecast_geo", + lambda **kwargs: pd.DataFrame({"Posted Datetime": ["2024-01-02"]}), + ) + + df_region = client.get_wind_forecast( + start="2024-01-01", end="2024-01-02", by_region=True + ) + df_global = client.get_wind_forecast( + start="2024-01-01", end="2024-01-02", by_region=False + ) + + assert not df_region.empty and not df_global.empty + assert archive.fetch_historical.call_count == 2 + + +def test_get_solar_forecast_live_region(monkeypatch: pytest.MonkeyPatch) -> None: + class LiveERCOT(ERCOT): + def _needs_historical(self, start: pd.Timestamp, market: str) -> bool: # type: ignore[override] + return False + + def _filter_by_date(self, df: pd.DataFrame, *args, **kwargs) -> pd.DataFrame: # type: ignore[override] + return df + + def _standardize_columns(self, df: pd.DataFrame) -> pd.DataFrame: # type: ignore[override] + return df + + client = LiveERCOT() + monkeypatch.setattr( + client, + "get_spp_hourly_actual_forecast_geo", + lambda **kwargs: pd.DataFrame({"Posted Datetime": ["2024-01-01"]}), + ) + + df = client.get_solar_forecast(start="2024-01-01", end="2024-01-02", by_region=True) + + assert not df.empty + + +def test_get_solar_forecast_historical(monkeypatch: pytest.MonkeyPatch) -> None: + class HistoricalERCOT(ERCOT): + def _needs_historical(self, start: pd.Timestamp, market: str) -> bool: # type: ignore[override] + return True + + def _filter_by_date(self, df: pd.DataFrame, *args, **kwargs) -> pd.DataFrame: # type: ignore[override] + return df + + def _standardize_columns(self, df: pd.DataFrame) -> pd.DataFrame: # type: ignore[override] + return df + + client = HistoricalERCOT() + archive = MagicMock() + archive.fetch_historical.return_value = pd.DataFrame( + {"Posted Datetime": ["2024-01-01"]} + ) + monkeypatch.setattr(client, "_get_archive", lambda: archive) + + df = client.get_solar_forecast( + start="2024-01-01", end="2024-01-02", by_region=False + ) + + assert not df.empty + archive.fetch_historical.assert_called_once() + + +def test_get_60_day_dam_disclosure_uses_archive( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class DisclosureERCOT(ERCOT): + def _standardize_columns(self, df: pd.DataFrame) -> pd.DataFrame: # type: ignore[override] + return df + + client = DisclosureERCOT() + + archive = MagicMock() + archive.fetch_historical.return_value = pd.DataFrame( + {"DeliveryDate": ["2024-01-01"]} + ) + monkeypatch.setattr(client, "_get_archive", lambda: archive) + + for method_name in [ + "get_dam_gen_res_as_offers", + "get_dam_load_res_data", + "get_dam_load_res_as_offers", + "get_dam_energy_only_offers", + "get_dam_energy_only_offer_awards", + "get_dam_energy_bids", + "get_dam_energy_bid_awards", + "get_dam_ptp_obl_bids", + "get_dam_ptp_obl_bid_awards", + "get_dam_ptp_obl_opt", + "get_dam_ptp_obl_opt_awards", + ]: + monkeypatch.setattr( + client, method_name, lambda **kwargs: pd.DataFrame({"dummy": [1]}) + ) + + reports = client.get_60_day_dam_disclosure("today") + + assert "dam_gen_resource" in reports + archive.fetch_historical.assert_called_once() + + +def test_get_60_day_sced_disclosure(monkeypatch: pytest.MonkeyPatch) -> None: + class DisclosureERCOT(ERCOT): + def _standardize_columns(self, df: pd.DataFrame) -> pd.DataFrame: # type: ignore[override] + return df + + client = DisclosureERCOT() + + archive = MagicMock() + archive.fetch_historical.return_value = pd.DataFrame( + {"DeliveryDate": ["2024-01-01"]} + ) + monkeypatch.setattr(client, "_get_archive", lambda: archive) + + monkeypatch.setattr( + client, "get_sced_gen_res_data", lambda **kwargs: pd.DataFrame({"a": [1]}) + ) + monkeypatch.setattr( + client, "get_load_res_data_in_sced", lambda **kwargs: pd.DataFrame({"b": [2]}) + ) + + reports = client.get_60_day_sced_disclosure("today") + + assert "sced_smne" in reports + archive.fetch_historical.assert_called_once() diff --git a/tests/test_historical_archive.py b/tests/test_historical_archive.py new file mode 100644 index 0000000..a8c36c3 --- /dev/null +++ b/tests/test_historical_archive.py @@ -0,0 +1,238 @@ +import io +from zipfile import ZipFile + +import httpx +import pandas as pd +import pytest + +from tinygrid.errors import GridAPIError +from tinygrid.historical.ercot import ArchiveLink, ERCOTArchive + + +class DummyClient: + def __init__(self, auth=None) -> None: + self.auth = auth + + +def make_zip_bytes(csv: str) -> io.BytesIO: + """Create a zip file in memory containing a CSV payload.""" + buf = io.BytesIO() + with ZipFile(buf, "w") as zf: + zf.writestr("data.csv", csv) + buf.seek(0) + return buf + + +def test_fetch_historical_returns_empty_when_no_links() -> None: + class NoLinksArchive(ERCOTArchive): + def get_archive_links(self, emil_id, start, end): # type: ignore[override] + return [] + + archive = NoLinksArchive(client=DummyClient()) + + df = archive.fetch_historical( + "/np6-905-cd/spp_node_zone_hub", + pd.Timestamp("2024-01-01"), + pd.Timestamp("2024-01-02"), + ) + + assert df.empty + + +def test_fetch_historical_adds_post_datetime() -> None: + class DownloadArchive(ERCOTArchive): + def get_archive_links(self, emil_id, start, end): # type: ignore[override] + return [ + ArchiveLink( + doc_id="123", + url="https://example.com/doc?id=123", + post_datetime="2024-01-01T00:00:00Z", + ) + ] + + def bulk_download(self, doc_ids, emil_id): # type: ignore[override] + csv_bytes = make_zip_bytes("col1,col2\n1,2\n") + return [(csv_bytes, "123.zip")] + + archive = DownloadArchive(client=DummyClient()) + + df = archive.fetch_historical( + "/np6-905-cd/spp_node_zone_hub", + pd.Timestamp("2024-01-01"), + pd.Timestamp("2024-01-02"), + add_post_datetime=True, + ) + + assert not df.empty + assert df["postDatetime"].iloc[0] == "2024-01-01T00:00:00Z" + + +def test_fetch_historical_parallel_handles_partial_failures() -> None: + class ParallelArchive(ERCOTArchive): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.links = [ + ArchiveLink( + doc_id="ok", url="http://example.com/ok", post_datetime="2024-01-01" + ), + ArchiveLink( + doc_id="fail", + url="http://example.com/fail", + post_datetime="2024-01-02", + ), + ] + + def get_archive_links(self, emil_id, start, end): # type: ignore[override] + return self.links + + def _download_single(self, link: ArchiveLink) -> pd.DataFrame: # type: ignore[override] + if link.doc_id == "fail": + raise ValueError("boom") + return pd.DataFrame({"DeliveryDate": ["2024-01-01"], "val": [1]}) + + archive = ParallelArchive(client=DummyClient(), max_concurrent=2) + + df = archive.fetch_historical_parallel( + "/np6-905-cd/spp_node_zone_hub", + pd.Timestamp("2024-01-01"), + pd.Timestamp("2024-01-02"), + add_post_datetime=True, + ) + + assert len(df) == 1 + assert df["postDatetime"].iloc[0] == "2024-01-01" + + +def test_fetch_historical_parallel_returns_empty_when_no_links() -> None: + class NoLinksArchive(ERCOTArchive): + def get_archive_links(self, emil_id, start, end): # type: ignore[override] + return [] + + archive = NoLinksArchive(client=DummyClient()) + df = archive.fetch_historical_parallel( + "/np6-905-cd/spp_node_zone_hub", + pd.Timestamp("2024-01-01"), + pd.Timestamp("2024-01-02"), + ) + + assert df.empty + + +def test_fetch_historical_handles_parse_failure() -> None: + class BadDownloadArchive(ERCOTArchive): + def get_archive_links(self, emil_id, start, end): # type: ignore[override] + return [ + ArchiveLink( + doc_id="bad", + url="https://example.com/bad", + post_datetime="2024-01-01", + ), + ] + + def bulk_download(self, doc_ids, emil_id): # type: ignore[override] + # Return invalid zip content to trigger parse failure + return [(io.BytesIO(b"not-a-zip"), "bad.zip")] + + archive = BadDownloadArchive(client=DummyClient()) + + df = archive.fetch_historical( + "/np6-905-cd/spp_node_zone_hub", + pd.Timestamp("2024-01-01"), + pd.Timestamp("2024-01-02"), + add_post_datetime=True, + ) + + assert df.empty + + +def test_fetch_historical_parallel_all_failures_returns_empty() -> None: + class AllFailArchive(ERCOTArchive): + def get_archive_links(self, emil_id, start, end): # type: ignore[override] + return [ + ArchiveLink( + doc_id="bad", + url="http://example.com/bad", + post_datetime="2024-01-01", + ), + ] + + def _download_single(self, link: ArchiveLink) -> pd.DataFrame: # type: ignore[override] + raise RuntimeError("fail") + + archive = AllFailArchive(client=DummyClient(), max_concurrent=1) + + df = archive.fetch_historical_parallel( + "/np6-905-cd/spp_node_zone_hub", + pd.Timestamp("2024-01-01"), + pd.Timestamp("2024-01-02"), + ) + + assert df.empty + + +def test_download_single_reads_zip(monkeypatch: pytest.MonkeyPatch) -> None: + class DownloadArchive(ERCOTArchive): + def _make_request(self, url, parse_json=True): # type: ignore[override] + return make_zip_bytes("col1\n1\n").getvalue() + + archive = DownloadArchive(client=DummyClient()) + link = ArchiveLink( + doc_id="1", url="http://example.com/1", post_datetime="2024-01-01" + ) + + df = archive._download_single(link) + + assert df.iloc[0]["col1"] == 1 + + +def test_make_request_handles_timeout(monkeypatch: pytest.MonkeyPatch) -> None: + class TimeoutClient: + def __init__(self, *args, **kwargs): + pass + + def __enter__(self): + raise httpx.TimeoutException("timeout") + + def __exit__(self, exc_type, exc, tb): + return False + + archive = ERCOTArchive(client=DummyClient()) + monkeypatch.setattr(httpx, "Client", TimeoutClient) + + with pytest.raises(GridAPIError): + archive._make_request("http://example.com") + + +def test_make_request_handles_request_error(monkeypatch: pytest.MonkeyPatch) -> None: + class ErrorClient: + def __init__(self, *args, **kwargs): + self + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def get(self, *args, **kwargs): + raise httpx.RequestError("fail", request=None) + + archive = ERCOTArchive(client=DummyClient()) + monkeypatch.setattr(httpx, "Client", ErrorClient) + + with pytest.raises(GridAPIError): + archive._make_request("http://example.com") + + +def test_get_auth_headers_uses_client_auth(monkeypatch: pytest.MonkeyPatch) -> None: + auth = type( + "Auth", + (), + {"get_token": lambda self: "token", "get_subscription_key": lambda self: "sub"}, + )() + archive = ERCOTArchive(client=DummyClient(auth=auth)) + + headers = archive._get_auth_headers() + + assert headers["Authorization"] == "Bearer token" + assert headers["Ocp-Apim-Subscription-Key"] == "sub" diff --git a/tests/test_tz.py b/tests/test_tz.py new file mode 100644 index 0000000..e87f3cb --- /dev/null +++ b/tests/test_tz.py @@ -0,0 +1,434 @@ +"""Tests for timezone utilities.""" + +import pandas as pd +import pytest +import pytz + +from tinygrid.constants.ercot import ERCOT_TIMEZONE +from tinygrid.utils.tz import ( + _localize_single, + dst_flag_to_ambiguous, + get_utc_offset, + is_dst_transition_date, + localize_with_dst, + resolve_ambiguous_dst, +) + + +class TestResolveAmbiguousDST: + """Test resolve_ambiguous_dst function.""" + + def test_resolve_ambiguous_dst_with_dst_flags(self): + """Test resolving ambiguous DST timestamps with DSTFlag.""" + # During fall back, 1:30 AM occurs twice + timestamps = pd.Series( + [ + "2023-11-05 01:30:00", + "2023-11-05 01:30:00", + ] + ) + dst_flags = pd.Series([True, False]) # First is DST, second is Standard + + result = resolve_ambiguous_dst(timestamps, dst_flags) + + assert result.dt.tz is not None + assert len(result) == 2 + # Both timestamps should be timezone-aware + assert result[0].tz is not None + assert result[1].tz is not None + + def test_resolve_ambiguous_dst_without_dst_flags(self): + """Test resolving ambiguous DST timestamps without DSTFlag (defaults to DST).""" + timestamps = pd.Series(["2023-11-05 01:30:00"]) + + result = resolve_ambiguous_dst(timestamps) + + assert result.dt.tz is not None + assert len(result) == 1 + + def test_resolve_ambiguous_dst_with_null_dst_flags(self): + """Test resolving with null DSTFlag values (should default to True).""" + timestamps = pd.Series(["2023-11-05 01:30:00", "2023-11-05 02:30:00"]) + dst_flags = pd.Series([None, True]) + + result = resolve_ambiguous_dst(timestamps, dst_flags) + + assert result.dt.tz is not None + assert len(result) == 2 + + def test_resolve_ambiguous_dst_non_ambiguous_times(self): + """Test resolving non-ambiguous timestamps.""" + timestamps = pd.Series(["2023-06-15 12:00:00", "2023-06-15 13:00:00"]) + + result = resolve_ambiguous_dst(timestamps) + + assert result.dt.tz is not None + assert len(result) == 2 + + def test_resolve_ambiguous_dst_custom_timezone(self): + """Test resolving with a custom timezone.""" + timestamps = pd.Series(["2023-11-05 01:30:00"]) + + result = resolve_ambiguous_dst(timestamps, tz="US/Eastern") + + assert result.dt.tz is not None + assert str(result[0].tz) == "US/Eastern" + + +class TestLocalizeWithDST: + """Test localize_with_dst function.""" + + def test_localize_naive_timestamp(self): + """Test localizing a naive timestamp.""" + dt = pd.Timestamp("2023-06-15 12:00:00") + + result = localize_with_dst(dt) + + assert result.tz is not None + assert str(result.tz) == ERCOT_TIMEZONE + + def test_localize_string_timestamp(self): + """Test localizing a datetime string.""" + dt = "2023-06-15 12:00:00" + + result = localize_with_dst(dt) + + assert result.tz is not None + assert str(result.tz) == ERCOT_TIMEZONE + + def test_localize_already_aware_timestamp(self): + """Test localizing an already timezone-aware timestamp (should convert).""" + dt = pd.Timestamp("2023-06-15 12:00:00", tz="UTC") + + result = localize_with_dst(dt) + + assert result.tz is not None + assert str(result.tz) == ERCOT_TIMEZONE + + def test_localize_ambiguous_time_dst(self): + """Test localizing ambiguous time with DST=True.""" + # During fall back, 1:30 AM occurs twice + dt = pd.Timestamp("2023-11-05 01:30:00") + + result = localize_with_dst(dt, ambiguous=True) + + assert result.tz is not None + + def test_localize_ambiguous_time_standard(self): + """Test localizing ambiguous time with DST=False.""" + # During fall back, 1:30 AM occurs twice + dt = pd.Timestamp("2023-11-05 01:30:00") + + result = localize_with_dst(dt, ambiguous=False) + + assert result.tz is not None + + def test_localize_custom_timezone(self): + """Test localizing to a custom timezone.""" + dt = pd.Timestamp("2023-06-15 12:00:00") + + result = localize_with_dst(dt, tz="US/Eastern") + + assert result.tz is not None + assert str(result.tz) == "US/Eastern" + + +class TestDSTFlagToAmbiguous: + """Test dst_flag_to_ambiguous function.""" + + def test_dst_flag_true_values(self): + """Test converting True DST flags.""" + dst_flag = pd.Series([True, True, True]) + + result = dst_flag_to_ambiguous(dst_flag) + + assert all(result == [True, True, True]) + + def test_dst_flag_false_values(self): + """Test converting False DST flags.""" + dst_flag = pd.Series([False, False, False]) + + result = dst_flag_to_ambiguous(dst_flag) + + assert all(result == [False, False, False]) + + def test_dst_flag_mixed_values(self): + """Test converting mixed DST flags.""" + dst_flag = pd.Series([True, False, True]) + + result = dst_flag_to_ambiguous(dst_flag) + + assert result[0] # pyright: ignore[reportGeneralTypeIssues] + assert not result[1] # pyright: ignore[reportGeneralTypeIssues] + assert result[2] # pyright: ignore[reportGeneralTypeIssues] + + def test_dst_flag_with_nulls(self): + """Test converting DST flags with null values (should default to True).""" + dst_flag = pd.Series([True, None, False]) + + result = dst_flag_to_ambiguous(dst_flag) + + assert result[0] # pyright: ignore[reportGeneralTypeIssues] + assert result[1] # pyright: ignore[reportGeneralTypeIssues] # Null should become True + assert not result[2] # pyright: ignore[reportGeneralTypeIssues] + + def test_dst_flag_all_nulls(self): + """Test converting all null DST flags.""" + dst_flag = pd.Series([None, None, None]) + + result = dst_flag_to_ambiguous(dst_flag) + + assert all(result == [True, True, True]) + + +class TestIsDSTTransitionDate: + """Test is_dst_transition_date function.""" + + def test_dst_transition_spring_forward(self): + """Test detecting spring forward DST transition.""" + # March 2023 spring forward (2nd Sunday of March) + date = pd.Timestamp("2023-03-12", tz=ERCOT_TIMEZONE) + + result = is_dst_transition_date(date) + + # This date should be a DST transition date + assert isinstance(result, bool) + + def test_dst_transition_fall_back(self): + """Test detecting fall back DST transition.""" + # November 2023 fall back (1st Sunday of November) + date = pd.Timestamp("2023-11-05", tz=ERCOT_TIMEZONE) + + result = is_dst_transition_date(date) + + # This date should be a DST transition date + assert isinstance(result, bool) + + def test_non_dst_transition_date(self): + """Test detecting non-DST transition date.""" + date = pd.Timestamp("2023-06-15", tz=ERCOT_TIMEZONE) + + result = is_dst_transition_date(date) + + # Regular date should not be a DST transition + assert result is False + + def test_dst_transition_custom_timezone(self): + """Test detecting DST transition in custom timezone.""" + date = pd.Timestamp("2023-03-12", tz="US/Eastern") + + result = is_dst_transition_date(date, tz="US/Eastern") + + assert isinstance(result, bool) + + +class TestGetUTCOffset: + """Test get_utc_offset function.""" + + def test_get_utc_offset_cdt(self): + """Test getting UTC offset for CDT (Central Daylight Time).""" + # Summer date in Central Daylight Time (UTC-5) + dt = pd.Timestamp("2023-06-15 12:00:00", tz=ERCOT_TIMEZONE) + + offset = get_utc_offset(dt) + + assert offset == -5 # CDT is UTC-5 + + def test_get_utc_offset_cst(self): + """Test getting UTC offset for CST (Central Standard Time).""" + # Winter date in Central Standard Time (UTC-6) + dt = pd.Timestamp("2023-01-15 12:00:00", tz=ERCOT_TIMEZONE) + + offset = get_utc_offset(dt) + + assert offset == -6 # CST is UTC-6 + + def test_get_utc_offset_utc(self): + """Test getting UTC offset for UTC timezone.""" + dt = pd.Timestamp("2023-06-15 12:00:00", tz="UTC") + + offset = get_utc_offset(dt) + + assert offset == 0 # UTC is offset 0 + + def test_get_utc_offset_eastern(self): + """Test getting UTC offset for Eastern timezone.""" + # Summer date in Eastern Daylight Time (UTC-4) + dt = pd.Timestamp("2023-06-15 12:00:00", tz="US/Eastern") + + offset = get_utc_offset(dt) + + assert offset == -4 # EDT is UTC-4 + + def test_get_utc_offset_naive_timestamp_raises(self): + """Test that naive timestamp raises ValueError.""" + dt = pd.Timestamp("2023-06-15 12:00:00") + + with pytest.raises(ValueError, match="Timestamp must be timezone-aware"): + get_utc_offset(dt) + + +class TestEdgeCases: + """Test edge cases and error handling.""" + + def test_resolve_ambiguous_dst_empty_series(self): + """Test resolving empty series.""" + timestamps = pd.Series([], dtype="datetime64[ns]") + + result = resolve_ambiguous_dst(timestamps) + + assert len(result) == 0 + assert result.dt.tz is not None + + def test_localize_with_dst_already_in_target_tz(self): + """Test localizing timestamp already in target timezone.""" + dt = pd.Timestamp("2023-06-15 12:00:00", tz=ERCOT_TIMEZONE) + + result = localize_with_dst(dt, tz=ERCOT_TIMEZONE) + + assert result.tz is not None + assert str(result.tz) == ERCOT_TIMEZONE + + def test_dst_flag_to_ambiguous_empty_series(self): + """Test converting empty DST flag series.""" + dst_flag = pd.Series([], dtype=bool) + + result = dst_flag_to_ambiguous(dst_flag) + + assert len(result) == 0 + + def test_localize_with_dst_nonexistent_shift_forward(self): + """Test localizing nonexistent time with shift_forward.""" + # During spring forward, 2:30 AM doesn't exist + dt = pd.Timestamp("2023-03-12 02:30:00") + + result = localize_with_dst(dt, tz=ERCOT_TIMEZONE, nonexistent="shift_forward") + + assert result.tz is not None + # Should be shifted forward to next valid time + + def test_localize_with_dst_nonexistent_shift_backward(self): + """Test localizing nonexistent time with shift_backward.""" + # During spring forward, 2:30 AM doesn't exist + dt = pd.Timestamp("2023-03-12 02:30:00") + + result = localize_with_dst(dt, tz=ERCOT_TIMEZONE, nonexistent="shift_backward") + + assert result.tz is not None + # Should be shifted backward to previous valid time + assert result.hour == 1 + + +class TestAdditionalDSTPaths: + def test_resolve_ambiguous_dst_fallback(self, monkeypatch: pytest.MonkeyPatch): + ts = pd.Series([pd.Timestamp("2021-11-07 01:30")]) + + from pandas.core.indexes.accessors import DatetimeProperties + + def raise_ambiguous(self, tz=None, ambiguous=None): # type: ignore[override] + raise pytz.exceptions.AmbiguousTimeError() + + monkeypatch.setattr( + DatetimeProperties, "tz_localize", raise_ambiguous, raising=False + ) + + result = resolve_ambiguous_dst(ts) + + assert not result.isna().all() + + def test_localize_with_dst_nonexistent_explicit_backward(self): + ts = "2024-03-10 02:15" + result = localize_with_dst(ts, nonexistent="shift_backward") + assert result.hour == 1 + + def test_localize_with_dst_nonexistent_shift_forward_fallback( + self, monkeypatch: pytest.MonkeyPatch + ): + orig = pd.Timestamp.tz_localize + calls = {"count": 0} + + def fake(self, tz=None, ambiguous=True, nonexistent="raise"): # type: ignore[override] + calls["count"] += 1 + if calls["count"] == 1: + raise pytz.exceptions.NonExistentTimeError() + return orig(self, tz=tz, ambiguous=ambiguous, nonexistent=nonexistent) + + monkeypatch.setattr(pd.Timestamp, "tz_localize", fake, raising=False) + + ts = pd.Timestamp("2024-03-10 02:15") + result = localize_with_dst(ts, nonexistent="shift_forward") + assert result.tz is not None + + def test_localize_with_dst_nonexistent_shift_backward_fallback( + self, monkeypatch: pytest.MonkeyPatch + ): + orig = pd.Timestamp.tz_localize + calls = {"count": 0} + + def fake(self, tz=None, ambiguous=True, nonexistent="raise"): # type: ignore[override] + calls["count"] += 1 + if calls["count"] == 1: + raise pytz.exceptions.NonExistentTimeError() + return orig(self, tz=tz, ambiguous=ambiguous, nonexistent=nonexistent) + + monkeypatch.setattr(pd.Timestamp, "tz_localize", fake, raising=False) + + ts = pd.Timestamp("2024-03-10 02:15") + result = localize_with_dst(ts, nonexistent="shift_backward") + assert result.tz is not None + + def test_localize_with_dst_ambiguous_fallback( + self, monkeypatch: pytest.MonkeyPatch + ): + orig = pd.Timestamp.tz_localize + calls = {"count": 0} + + def fake(self, tz=None, ambiguous=True, nonexistent="raise"): # type: ignore[override] + calls["count"] += 1 + if calls["count"] == 1: + raise pytz.exceptions.AmbiguousTimeError() + return orig(self, tz=tz, ambiguous=ambiguous, nonexistent=nonexistent) + + monkeypatch.setattr(pd.Timestamp, "tz_localize", fake, raising=False) + + ts = pd.Timestamp("2021-11-07 01:30") + result = localize_with_dst(ts) + assert result.tz is not None + + def test_localize_with_dst_nonexistent_invalid_mode(self): + ts = "2024-03-10 02:15" + with pytest.raises(ValueError): + localize_with_dst(ts, nonexistent="raise") + + def test_localize_with_dst_nonexistent_raise_branch( + self, monkeypatch: pytest.MonkeyPatch + ): + ts = pd.Timestamp("2024-03-10 02:15") + + def raise_nonexistent(self, tz=None, ambiguous=True, nonexistent="raise"): # type: ignore[override] + raise pytz.exceptions.NonExistentTimeError() + + monkeypatch.setattr( + pd.Timestamp, "tz_localize", raise_nonexistent, raising=False + ) + + with pytest.raises(ValueError): + localize_with_dst(ts, nonexistent="raise") + + def test_localize_single_returns_nat(self): + assert pd.isna(_localize_single(pd.NaT, ERCOT_TIMEZONE)) + + def test_localize_single_success(self): + ts = pd.Timestamp("2024-01-01 00:00:00") + result = _localize_single(ts, ERCOT_TIMEZONE) + assert result.tz is not None + + def test_get_utc_offset_with_missing_offset(self): + class Dummy: + tz = "UTC" + + def utcoffset(self): + return None + + with pytest.raises(ValueError): + get_utc_offset(Dummy()) # type: ignore[arg-type] diff --git a/tinygrid/utils/tz.py b/tinygrid/utils/tz.py index 7f50c30..bfe8c3b 100644 --- a/tinygrid/utils/tz.py +++ b/tinygrid/utils/tz.py @@ -35,7 +35,13 @@ def resolve_ambiguous_dst( dt_series = pd.to_datetime(timestamps) # Use DSTFlag to resolve ambiguous times (DST=True, Standard=False) - ambiguous = dst_flags.fillna(True).astype(bool) if dst_flags is not None else True + if dst_flags is None: + ambiguous = True + else: + # Normalize to pandas nullable boolean to avoid downcast warnings, then + # fill missing values as True (DST). + normalized_flags = dst_flags.astype("boolean").fillna(True) + ambiguous = normalized_flags.astype(bool) try: localized = dt_series.dt.tz_localize(tz, ambiguous=ambiguous) @@ -133,7 +139,8 @@ def dst_flag_to_ambiguous(dst_flag: pd.Series) -> pd.Series: Returns: Boolean series for use with tz_localize(ambiguous=...) """ - return dst_flag.fillna(True).astype(bool) + normalized_flags = dst_flag.astype("boolean").fillna(True) + return normalized_flags.astype(bool) def is_dst_transition_date(date: pd.Timestamp, tz: str = ERCOT_TIMEZONE) -> bool: diff --git a/uv.lock b/uv.lock index f1d3be4..1d66b6c 100644 --- a/uv.lock +++ b/uv.lock @@ -261,6 +261,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ae/3a/dbeec9d1ee0844c679f6bb5d6ad4e9f198b1224f4e7a32825f47f6192b0c/cffi-2.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0a1527a803f0a659de1af2e1fd700213caba79377e27e4693648c2923da066f9", size = 184195, upload-time = "2025-09-08T23:23:43.004Z" }, ] +[[package]] +name = "cfgv" +version = "3.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4e/b5/721b8799b04bf9afe054a3899c6cf4e880fcf8563cc71c15610242490a0c/cfgv-3.5.0.tar.gz", hash = "sha256:d5b1034354820651caa73ede66a6294d6e95c1b00acc5e9b098e917404669132", size = 7334, upload-time = "2025-11-19T20:55:51.612Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/db/3c/33bac158f8ab7f89b2e59426d5fe2e4f63f7ed25df84c036890172b412b5/cfgv-3.5.0-py2.py3-none-any.whl", hash = "sha256:a8dc6b26ad22ff227d2634a65cb388215ce6cc96bbcc5cfde7641ae87e8dacc0", size = 7445, upload-time = "2025-11-19T20:55:50.744Z" }, +] + [[package]] name = "charset-normalizer" version = "3.4.4" @@ -532,6 +541,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61", size = 25604, upload-time = "2021-03-08T10:59:24.45Z" }, ] +[[package]] +name = "distlib" +version = "0.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/96/8e/709914eb2b5749865801041647dc7f4e6d00b549cfe88b65ca192995f07c/distlib-0.4.0.tar.gz", hash = "sha256:feec40075be03a04501a973d81f633735b4b69f98b05450592310c0f401a4e0d", size = 614605, upload-time = "2025-07-17T16:52:00.465Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/33/6b/e0547afaf41bf2c42e52430072fa5658766e3d65bd4b03a563d1b6336f57/distlib-0.4.0-py2.py3-none-any.whl", hash = "sha256:9659f7d87e46584a30b5780e43ac7a2143098441670ff0a49d5f9034c54a6c16", size = 469047, upload-time = "2025-07-17T16:51:58.613Z" }, +] + [[package]] name = "exceptiongroup" version = "1.3.1" @@ -562,6 +580,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/a8/20d0723294217e47de6d9e2e40fd4a9d2f7c4b6ef974babd482a59743694/fastjsonschema-2.21.2-py3-none-any.whl", hash = "sha256:1c797122d0a86c5cace2e54bf4e819c36223b552017172f32c5c024a6b77e463", size = 24024, upload-time = "2025-08-14T18:49:34.776Z" }, ] +[[package]] +name = "filelock" +version = "3.20.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a7/23/ce7a1126827cedeb958fc043d61745754464eb56c5937c35bbf2b8e26f34/filelock-3.20.1.tar.gz", hash = "sha256:b8360948b351b80f420878d8516519a2204b07aefcdcfd24912a5d33127f188c", size = 19476, upload-time = "2025-12-15T23:54:28.027Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e3/7f/a1a97644e39e7316d850784c642093c99df1290a460df4ede27659056834/filelock-3.20.1-py3-none-any.whl", hash = "sha256:15d9e9a67306188a44baa72f569d2bfd803076269365fdea0934385da4dc361a", size = 16666, upload-time = "2025-12-15T23:54:26.874Z" }, +] + [[package]] name = "fqdn" version = "1.5.1" @@ -608,6 +635,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] +[[package]] +name = "identify" +version = "2.6.15" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ff/e7/685de97986c916a6d93b3876139e00eef26ad5bbbd61925d670ae8013449/identify-2.6.15.tar.gz", hash = "sha256:e4f4864b96c6557ef2a1e1c951771838f4edc9df3a72ec7118b338801b11c7bf", size = 99311, upload-time = "2025-10-02T17:43:40.631Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0f/1c/e5fd8f973d4f375adb21565739498e2e9a1e54c858a97b9a8ccfdc81da9b/identify-2.6.15-py2.py3-none-any.whl", hash = "sha256:1181ef7608e00704db228516541eb83a88a9f94433a8c80bb9b5bd54b1d81757", size = 99183, upload-time = "2025-10-02T17:43:39.137Z" }, +] + [[package]] name = "idna" version = "3.11" @@ -1572,6 +1608,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] +[[package]] +name = "pre-commit" +version = "4.5.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cfgv" }, + { name = "identify" }, + { name = "nodeenv" }, + { name = "pyyaml" }, + { name = "virtualenv" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/40/f1/6d86a29246dfd2e9b6237f0b5823717f60cad94d47ddc26afa916d21f525/pre_commit-4.5.1.tar.gz", hash = "sha256:eb545fcff725875197837263e977ea257a402056661f09dae08e4b149b030a61", size = 198232, upload-time = "2025-12-16T21:14:33.552Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/19/fd3ef348460c80af7bb4669ea7926651d1f95c23ff2df18b9d24bab4f3fa/pre_commit-4.5.1-py2.py3-none-any.whl", hash = "sha256:3b3afd891e97337708c1674210f8eba659b52a38ea5f822ff142d10786221f77", size = 226437, upload-time = "2025-12-16T21:14:32.409Z" }, +] + [[package]] name = "prometheus-client" version = "0.23.1" @@ -2318,6 +2370,7 @@ dependencies = [ [package.optional-dependencies] dev = [ { name = "pandas" }, + { name = "pre-commit" }, { name = "pyright" }, { name = "pytest" }, { name = "pytest-asyncio" }, @@ -2339,6 +2392,7 @@ dev = [ requires-dist = [ { name = "pandas", specifier = ">=2.0.0" }, { name = "pandas", marker = "extra == 'dev'", specifier = ">=2.3.3" }, + { name = "pre-commit", marker = "extra == 'dev'", specifier = ">=3.6.0" }, { name = "pyercot", editable = "pyercot" }, { name = "pyright", marker = "extra == 'dev'", specifier = ">=1.1.0" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=7.4.0" }, @@ -2473,6 +2527,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6d/b9/4095b668ea3678bf6a0af005527f39de12fb026516fb3df17495a733b7f8/urllib3-2.6.2-py3-none-any.whl", hash = "sha256:ec21cddfe7724fc7cb4ba4bea7aa8e2ef36f607a4bab81aa6ce42a13dc3f03dd", size = 131182, upload-time = "2025-12-11T15:56:38.584Z" }, ] +[[package]] +name = "virtualenv" +version = "20.35.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "distlib" }, + { name = "filelock" }, + { name = "platformdirs" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/20/28/e6f1a6f655d620846bd9df527390ecc26b3805a0c5989048c210e22c5ca9/virtualenv-20.35.4.tar.gz", hash = "sha256:643d3914d73d3eeb0c552cbb12d7e82adf0e504dbf86a3182f8771a153a1971c", size = 6028799, upload-time = "2025-10-29T06:57:40.511Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/0c/c05523fa3181fdf0c9c52a6ba91a23fbf3246cc095f26f6516f9c60e6771/virtualenv-20.35.4-py3-none-any.whl", hash = "sha256:c21c9cede36c9753eeade68ba7d523529f228a403463376cf821eaae2b650f1b", size = 6005095, upload-time = "2025-10-29T06:57:37.598Z" }, +] + [[package]] name = "wcwidth" version = "0.2.14"