diff --git a/src/cli.py b/src/cli.py index bac175c..c28bb47 100644 --- a/src/cli.py +++ b/src/cli.py @@ -2,6 +2,7 @@ import argparse import logging +import os import sys from collections import Counter from datetime import datetime @@ -118,7 +119,7 @@ def parse_args(): "--failed-file", type=str, default="failed_events.json", - help="Path to save failed tournaments JSON if any events fail", + help="Path to save failed tournaments JSON if any events fail (a datestamped copy is also saved)", ) parser.add_argument( "--delay", @@ -143,7 +144,9 @@ def parse_args(): def main(): args = parse_args() - setup_logging(args.verbose, args.log_file) + run_timestamp = datetime.now().strftime("%Y%m%d-%H%M%S") + log_file = args.log_file or f"scraper_{run_timestamp}.log" + setup_logging(args.verbose, log_file) start_date = date_parser.parse(args.start_date).date() if args.start_date else None end_date = date_parser.parse(args.end_date).date() if args.end_date else None @@ -187,11 +190,28 @@ def main(): failed_events = stats.get("failed_events", []) if failed_events: - save_failed_events(failed_events, args.failed_file) - print(f"\nSaved {len(failed_events)} failed event(s) to: {args.failed_file}") - print( - f"To retry these events, run: python main.py --retry-failed {args.failed_file}" + failed_dir = os.path.dirname(args.failed_file) + timestamped_file = ( + os.path.join(failed_dir, f"failed_events_{run_timestamp}.json") + if failed_dir + else f"failed_events_{run_timestamp}.json" ) + save_failed_events(failed_events, timestamped_file) + if args.failed_file and args.failed_file != timestamped_file: + save_failed_events(failed_events, args.failed_file) + print( + f"\nSaved {len(failed_events)} failed event(s) to: {args.failed_file} and {timestamped_file}" + ) + print( + f"To retry these events, run: python main.py --retry-failed {args.failed_file}" + ) + else: + print( + f"\nSaved {len(failed_events)} failed event(s) to: {timestamped_file}" + ) + print( + f"To retry these events, run: python main.py --retry-failed {timestamped_file}" + ) reasons = Counter(t.failure_reason or "Unknown reason" for t in failed_events) print("\nFailure breakdown by reason:") @@ -211,6 +231,12 @@ def main(): elif args.retry_failed: print("\nAll retried tournaments succeeded!") + if stats.get("created", 0) == 0 and stats.get("updated", 0) == 0: + print( + "No tournaments processed or updated successfully. Exiting with non-zero status.", + ) + sys.exit(1) + if __name__ == "__main__": main() diff --git a/src/client.py b/src/client.py index e9db0c4..c9a485b 100644 --- a/src/client.py +++ b/src/client.py @@ -20,6 +20,7 @@ from .config import MTGO_LIST_URL from .config import MTGO_ROOT_URL +from .config import SKIP_FORMATS from .config import VALID_FORMATS from .config import get_user_agent from .models import CacheItem @@ -154,7 +155,9 @@ def fetch_calendar(self, start_date: date, end_date: date) -> List[Tournament]: continue title = h3_tag.text.strip() - if title.startswith("Limited"): + if any( + title.lower().startswith(skip.lower()) for skip in SKIP_FORMATS + ): continue event_url = urljoin(MTGO_ROOT_URL, a_tag.get("href", "")) date_str = time_tag.get("datetime", "") diff --git a/src/config.py b/src/config.py index 64170c3..d00c388 100644 --- a/src/config.py +++ b/src/config.py @@ -15,6 +15,14 @@ "Premodern", ] +# Skip any formats where the name starts with this. +# eg. skip "contraption-league-2026-06-0810735" +# but not "premodern-challenge-32---contraption-2025-12-3012828126" +SKIP_FORMATS = [ + "Contraption", + "Limited", +] + SCRYFALL_BULK_URL = "https://api.scryfall.com/bulk-data" SCRYFALL_CACHE_TTL_HOURS = 24 diff --git a/src/scraper.py b/src/scraper.py index d0151b4..56c7966 100644 --- a/src/scraper.py +++ b/src/scraper.py @@ -22,6 +22,7 @@ from .client import tournament_from_url from .config import DEFAULT_LOOKBACK_DAYS from .config import DEFAULT_REQUEST_DELAY +from .config import SKIP_FORMATS from .models import Tournament from .scryfall import ScryfallNormalizer @@ -137,8 +138,10 @@ def _sync_tournament( if not t.date: t.date = today - if t.name and t.name.startswith("Limited"): - logger.info("Skipping Limited event: %s", t.name) + if t.name and any( + t.name.lower().startswith(skip.lower()) for skip in SKIP_FORMATS + ): + logger.info("Skipping ignored format event: %s", t.name) return "skipped" safe_filename = sanitize_filename(t.json_file or "unknown.json") @@ -183,8 +186,9 @@ def _sync_tournament( if not isinstance(reason, str): reason = "Failed to fetch event data" t.failure_reason = reason + # Intentional space here to make copying URLs from logs easier logger.warning( - "Failed to fetch event data for %s: %s", t.uri, t.failure_reason + "Failed to fetch event data for %s (%s)", t.uri, t.failure_reason ) return "failed" diff --git a/tests/test_cli.py b/tests/test_cli.py index 2fec732..676e140 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,8 +1,13 @@ import io import logging import sys +from unittest.mock import MagicMock +from unittest.mock import patch + +import pytest from src.cli import LevelFilter +from src.cli import main from src.cli import setup_logging @@ -51,3 +56,113 @@ def test_setup_logging_verbose(tmp_path, monkeypatch): assert log_file.exists() log_file_content = log_file.read_text(encoding="utf-8") assert "This is a debug message" in log_file_content + + +def test_main_exits_nonzero_when_no_decks_created_or_updated(monkeypatch): + monkeypatch.setattr("sys.argv", ["main.py"]) + fake_stats = { + "total_found": 1, + "created": 0, + "updated": 0, + "skipped": 1, + "failed": 0, + "failed_events": [], + } + with patch("src.cli.MTGOSyncEngine") as mock_engine_cls: + mock_engine = MagicMock() + mock_engine.sync.return_value = fake_stats + mock_engine_cls.return_value = mock_engine + + with pytest.raises(SystemExit) as exc_info: + main() + assert exc_info.value.code == 1 + + +def test_main_succeeds_when_decks_created(monkeypatch): + monkeypatch.setattr("sys.argv", ["main.py"]) + fake_stats = { + "total_found": 1, + "created": 1, + "updated": 0, + "skipped": 0, + "failed": 0, + "failed_events": [], + } + with patch("src.cli.MTGOSyncEngine") as mock_engine_cls: + mock_engine = MagicMock() + mock_engine.sync.return_value = fake_stats + mock_engine_cls.return_value = mock_engine + + main() + + +def test_main_succeeds_when_decks_updated(monkeypatch): + monkeypatch.setattr("sys.argv", ["main.py"]) + fake_stats = { + "total_found": 1, + "created": 0, + "updated": 1, + "skipped": 0, + "failed": 0, + "failed_events": [], + } + with patch("src.cli.MTGOSyncEngine") as mock_engine_cls: + mock_engine = MagicMock() + mock_engine.sync.return_value = fake_stats + mock_engine_cls.return_value = mock_engine + + main() + + +def test_main_saves_datestamped_failed_events(tmp_path, monkeypatch): + failed_file = tmp_path / "failed_events.json" + log_file = tmp_path / "test.log" + monkeypatch.setattr( + "sys.argv", + [ + "main.py", + "--failed-file", + str(failed_file), + "--log-file", + str(log_file), + ], + ) + mock_tournament = MagicMock() + mock_tournament.to_failed_dict.return_value = { + "name": "Failed Event", + "uri": "https://www.mtgo.com/failed", + "failure_reason": "HTTP 500", + } + mock_tournament.name = "Failed Event" + mock_tournament.uri = "https://www.mtgo.com/failed" + mock_tournament.date = None + mock_tournament.failure_reason = "HTTP 500" + + fake_stats = { + "total_found": 1, + "created": 0, + "updated": 0, + "skipped": 0, + "failed": 1, + "failed_events": [mock_tournament], + } + + with patch("src.cli.MTGOSyncEngine") as mock_engine_cls: + mock_engine = MagicMock() + mock_engine.sync.return_value = fake_stats + mock_engine_cls.return_value = mock_engine + + with pytest.raises(SystemExit) as exc_info: + main() + assert exc_info.value.code == 1 + + assert failed_file.exists() + timestamped_files = [ + f + for f in tmp_path.glob("failed_events_*.json") + if f.name != "failed_events.json" + ] + assert len(timestamped_files) == 1 + assert timestamped_files[0].read_text(encoding="utf-8") == failed_file.read_text( + encoding="utf-8" + ) diff --git a/tests/test_client.py b/tests/test_client.py index 972c86d..395c2e1 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -138,6 +138,16 @@ def test_tournament_from_url(): t_premodern = tournament_from_url(premodern_url) assert t_premodern.formats == "Premodern" + contraption_url = "https://www.mtgo.com/decklist/contraption-league-2026-06-0810735" + t_contraption = tournament_from_url(contraption_url) + assert t_contraption.name == "Contraption League" + assert t_contraption.formats is None + + premodern_contraption_url = "https://www.mtgo.com/decklist/premodern-challenge-32---contraption-2025-12-3012828126" + t_premodern_contraption = tournament_from_url(premodern_contraption_url) + assert t_premodern_contraption.name == "Premodern Challenge 32 Contraption" + assert t_premodern_contraption.formats == "Premodern" + def test_fetch_event_data_404_no_retry(): client = MTGOClient(max_retries=3) @@ -159,7 +169,7 @@ def test_parse_event_no_decks_sets_last_error(): assert client.last_error == "Tournament has no decks (event likely did not fire)" -def test_fetch_calendar_skips_limited_events(): +def test_fetch_calendar_skips_configured_skip_formats(): client = MTGOClient() mock_resp = MagicMock() mock_resp.status_code = 200 @@ -178,6 +188,12 @@ def test_fetch_calendar_skips_limited_events(): +
  • + +

    Contraption League

    + +     +

  • Premodern Challenge 32

    diff --git a/tests/test_incremental.py b/tests/test_incremental.py index 4dbe70d..149b55d 100644 --- a/tests/test_incremental.py +++ b/tests/test_incremental.py @@ -229,6 +229,65 @@ def test_sync_skips_limited_event(tmp_path): assert mock_client.fetch_event_data.call_count == 0 +def test_sync_skips_contraption_event(tmp_path): + mock_client = MagicMock() + contraption_tournament = Tournament( + date=date(2026, 6, 8), + name="Contraption League", + uri="https://www.mtgo.com/decklist/contraption-league-2026-06-0810735", + formats=None, + json_file="contraption-league-2026-06-0810735.json", + ) + # Also test case-insensitive lowercase slug name + contraption_slug_tournament = Tournament( + date=date(2026, 6, 8), + name="contraption-league-2026-06-0810735", + uri="https://www.mtgo.com/decklist/contraption-league-2026-06-0810735", + formats=None, + json_file="contraption-league-2026-06-0810735.json", + ) + engine = MTGOSyncEngine(cache_root=str(tmp_path), client=mock_client) + stats = engine.sync( + tournaments=[contraption_tournament, contraption_slug_tournament] + ) + + assert stats["skipped"] == 2 + assert stats["created"] == 0 + assert stats["failed"] == 0 + assert mock_client.fetch_event_data.call_count == 0 + + +def test_sync_does_not_skip_event_with_contraption_in_middle(tmp_path): + mock_client = MagicMock() + mock_client.fetch_event_data.return_value = { + "decklists": [{"player": "P1"}], + "player_count": {"players": "16"}, + } + mock_item = MagicMock() + mock_item.to_dict.return_value = { + "Tournament": { + "Name": "Premodern Challenge 32 Contraption", + "PlayerCount": 16, + }, + "Decks": [{"player": "P1"}], + } + mock_client.parse_event.return_value = mock_item + + tournament = Tournament( + date=date(2025, 12, 30), + name="Premodern Challenge 32 Contraption", + uri="https://www.mtgo.com/decklist/premodern-challenge-32---contraption-2025-12-3012828126", + formats="Premodern", + json_file="premodern-challenge-32---contraption-2025-12-3012828126.json", + ) + engine = MTGOSyncEngine(cache_root=str(tmp_path), client=mock_client) + stats = engine.sync(tournaments=[tournament]) + + assert stats["skipped"] == 0 + assert stats["created"] == 1 + assert mock_client.fetch_event_data.call_count == 1 + + def test_cli_parse_retry_args(monkeypatch): from src.cli import parse_args