Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 32 additions & 6 deletions src/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import argparse
import logging
import os
import sys
from collections import Counter
from datetime import datetime
Expand Down Expand Up @@ -118,7 +119,7 @@ def parse_args():
"--failed-file",
type=str,
default="failed_events.json",
help="Path to save failed tournaments JSON if any events fail",
help="Path to save failed tournaments JSON if any events fail (a datestamped copy is also saved)",
)
parser.add_argument(
"--delay",
Expand All @@ -143,7 +144,9 @@ def parse_args():

def main():
args = parse_args()
setup_logging(args.verbose, args.log_file)
run_timestamp = datetime.now().strftime("%Y%m%d-%H%M%S")
log_file = args.log_file or f"scraper_{run_timestamp}.log"
setup_logging(args.verbose, log_file)

start_date = date_parser.parse(args.start_date).date() if args.start_date else None
end_date = date_parser.parse(args.end_date).date() if args.end_date else None
Expand Down Expand Up @@ -187,11 +190,28 @@ def main():

failed_events = stats.get("failed_events", [])
if failed_events:
save_failed_events(failed_events, args.failed_file)
print(f"\nSaved {len(failed_events)} failed event(s) to: {args.failed_file}")
print(
f"To retry these events, run: python main.py --retry-failed {args.failed_file}"
failed_dir = os.path.dirname(args.failed_file)
timestamped_file = (
os.path.join(failed_dir, f"failed_events_{run_timestamp}.json")
if failed_dir
else f"failed_events_{run_timestamp}.json"
)
save_failed_events(failed_events, timestamped_file)
if args.failed_file and args.failed_file != timestamped_file:
save_failed_events(failed_events, args.failed_file)
print(
f"\nSaved {len(failed_events)} failed event(s) to: {args.failed_file} and {timestamped_file}"
)
print(
f"To retry these events, run: python main.py --retry-failed {args.failed_file}"
)
else:
print(
f"\nSaved {len(failed_events)} failed event(s) to: {timestamped_file}"
)
print(
f"To retry these events, run: python main.py --retry-failed {timestamped_file}"
)

reasons = Counter(t.failure_reason or "Unknown reason" for t in failed_events)
print("\nFailure breakdown by reason:")
Expand All @@ -211,6 +231,12 @@ def main():
elif args.retry_failed:
print("\nAll retried tournaments succeeded!")

if stats.get("created", 0) == 0 and stats.get("updated", 0) == 0:
print(
"No tournaments processed or updated successfully. Exiting with non-zero status.",
)
sys.exit(1)


if __name__ == "__main__":
main()
5 changes: 4 additions & 1 deletion src/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

from .config import MTGO_LIST_URL
from .config import MTGO_ROOT_URL
from .config import SKIP_FORMATS
from .config import VALID_FORMATS
from .config import get_user_agent
from .models import CacheItem
Expand Down Expand Up @@ -154,7 +155,9 @@ def fetch_calendar(self, start_date: date, end_date: date) -> List[Tournament]:
continue

title = h3_tag.text.strip()
if title.startswith("Limited"):
if any(
title.lower().startswith(skip.lower()) for skip in SKIP_FORMATS
):
continue
event_url = urljoin(MTGO_ROOT_URL, a_tag.get("href", ""))
date_str = time_tag.get("datetime", "")
Expand Down
8 changes: 8 additions & 0 deletions src/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,14 @@
"Premodern",
]

# Skip any formats where the name starts with this.
# eg. skip "contraption-league-2026-06-0810735"
# but not "premodern-challenge-32---contraption-2025-12-3012828126"
SKIP_FORMATS = [
"Contraption",
"Limited",
]

SCRYFALL_BULK_URL = "https://api.scryfall.com/bulk-data"
SCRYFALL_CACHE_TTL_HOURS = 24

Expand Down
10 changes: 7 additions & 3 deletions src/scraper.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
from .client import tournament_from_url
from .config import DEFAULT_LOOKBACK_DAYS
from .config import DEFAULT_REQUEST_DELAY
from .config import SKIP_FORMATS
from .models import Tournament
from .scryfall import ScryfallNormalizer

Expand Down Expand Up @@ -137,8 +138,10 @@ def _sync_tournament(
if not t.date:
t.date = today

if t.name and t.name.startswith("Limited"):
logger.info("Skipping Limited event: %s", t.name)
if t.name and any(
t.name.lower().startswith(skip.lower()) for skip in SKIP_FORMATS
):
logger.info("Skipping ignored format event: %s", t.name)
return "skipped"

safe_filename = sanitize_filename(t.json_file or "unknown.json")
Expand Down Expand Up @@ -183,8 +186,9 @@ def _sync_tournament(
if not isinstance(reason, str):
reason = "Failed to fetch event data"
t.failure_reason = reason
# Intentional space here to make copying URLs from logs easier
logger.warning(
"Failed to fetch event data for %s: %s", t.uri, t.failure_reason
"Failed to fetch event data for %s (%s)", t.uri, t.failure_reason
)
return "failed"

Expand Down
115 changes: 115 additions & 0 deletions tests/test_cli.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,13 @@
import io
import logging
import sys
from unittest.mock import MagicMock
from unittest.mock import patch

import pytest

from src.cli import LevelFilter
from src.cli import main
from src.cli import setup_logging


Expand Down Expand Up @@ -51,3 +56,113 @@ def test_setup_logging_verbose(tmp_path, monkeypatch):
assert log_file.exists()
log_file_content = log_file.read_text(encoding="utf-8")
assert "This is a debug message" in log_file_content


def test_main_exits_nonzero_when_no_decks_created_or_updated(monkeypatch):
monkeypatch.setattr("sys.argv", ["main.py"])
fake_stats = {
"total_found": 1,
"created": 0,
"updated": 0,
"skipped": 1,
"failed": 0,
"failed_events": [],
}
with patch("src.cli.MTGOSyncEngine") as mock_engine_cls:
mock_engine = MagicMock()
mock_engine.sync.return_value = fake_stats
mock_engine_cls.return_value = mock_engine

with pytest.raises(SystemExit) as exc_info:
main()
assert exc_info.value.code == 1


def test_main_succeeds_when_decks_created(monkeypatch):
monkeypatch.setattr("sys.argv", ["main.py"])
fake_stats = {
"total_found": 1,
"created": 1,
"updated": 0,
"skipped": 0,
"failed": 0,
"failed_events": [],
}
with patch("src.cli.MTGOSyncEngine") as mock_engine_cls:
mock_engine = MagicMock()
mock_engine.sync.return_value = fake_stats
mock_engine_cls.return_value = mock_engine

main()


def test_main_succeeds_when_decks_updated(monkeypatch):
monkeypatch.setattr("sys.argv", ["main.py"])
fake_stats = {
"total_found": 1,
"created": 0,
"updated": 1,
"skipped": 0,
"failed": 0,
"failed_events": [],
}
with patch("src.cli.MTGOSyncEngine") as mock_engine_cls:
mock_engine = MagicMock()
mock_engine.sync.return_value = fake_stats
mock_engine_cls.return_value = mock_engine

main()


def test_main_saves_datestamped_failed_events(tmp_path, monkeypatch):
failed_file = tmp_path / "failed_events.json"
log_file = tmp_path / "test.log"
monkeypatch.setattr(
"sys.argv",
[
"main.py",
"--failed-file",
str(failed_file),
"--log-file",
str(log_file),
],
)
mock_tournament = MagicMock()
mock_tournament.to_failed_dict.return_value = {
"name": "Failed Event",
"uri": "https://www.mtgo.com/failed",
"failure_reason": "HTTP 500",
}
mock_tournament.name = "Failed Event"
mock_tournament.uri = "https://www.mtgo.com/failed"
mock_tournament.date = None
mock_tournament.failure_reason = "HTTP 500"

fake_stats = {
"total_found": 1,
"created": 0,
"updated": 0,
"skipped": 0,
"failed": 1,
"failed_events": [mock_tournament],
}

with patch("src.cli.MTGOSyncEngine") as mock_engine_cls:
mock_engine = MagicMock()
mock_engine.sync.return_value = fake_stats
mock_engine_cls.return_value = mock_engine

with pytest.raises(SystemExit) as exc_info:
main()
assert exc_info.value.code == 1

assert failed_file.exists()
timestamped_files = [
f
for f in tmp_path.glob("failed_events_*.json")
if f.name != "failed_events.json"
]
assert len(timestamped_files) == 1
assert timestamped_files[0].read_text(encoding="utf-8") == failed_file.read_text(
encoding="utf-8"
)
18 changes: 17 additions & 1 deletion tests/test_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,16 @@ def test_tournament_from_url():
t_premodern = tournament_from_url(premodern_url)
assert t_premodern.formats == "Premodern"

contraption_url = "https://www.mtgo.com/decklist/contraption-league-2026-06-0810735"
t_contraption = tournament_from_url(contraption_url)
assert t_contraption.name == "Contraption League"
assert t_contraption.formats is None

premodern_contraption_url = "https://www.mtgo.com/decklist/premodern-challenge-32---contraption-2025-12-3012828126"
t_premodern_contraption = tournament_from_url(premodern_contraption_url)
assert t_premodern_contraption.name == "Premodern Challenge 32 Contraption"
assert t_premodern_contraption.formats == "Premodern"


def test_fetch_event_data_404_no_retry():
client = MTGOClient(max_retries=3)
Expand All @@ -159,7 +169,7 @@ def test_parse_event_no_decks_sets_last_error():
assert client.last_error == "Tournament has no decks (event likely did not fire)"


def test_fetch_calendar_skips_limited_events():
def test_fetch_calendar_skips_configured_skip_formats():
client = MTGOClient()
mock_resp = MagicMock()
mock_resp.status_code = 200
Expand All @@ -178,6 +188,12 @@ def test_fetch_calendar_skips_limited_events():
<time datetime="2026-09-10T14:05:00.000Z"></time>
</a>
</li>
<li class="decklists-item">
<a href="/decklist/contraption-league-2026-06-0810735">
<div><h3>Contraption League</h3></div>
<time datetime="2026-06-08T10:00:00.000Z"></time>
</a>
</li>
<li class="decklists-item">
<a href="/decklist/premodern-challenge-32-2026-09-1012854063">
<div><h3>Premodern Challenge 32</h3></div>
Expand Down
59 changes: 59 additions & 0 deletions tests/test_incremental.py
Original file line number Diff line number Diff line change
Expand Up @@ -229,6 +229,65 @@ def test_sync_skips_limited_event(tmp_path):
assert mock_client.fetch_event_data.call_count == 0


def test_sync_skips_contraption_event(tmp_path):
mock_client = MagicMock()
contraption_tournament = Tournament(
date=date(2026, 6, 8),
name="Contraption League",
uri="https://www.mtgo.com/decklist/contraption-league-2026-06-0810735",
formats=None,
json_file="contraption-league-2026-06-0810735.json",
)
# Also test case-insensitive lowercase slug name
contraption_slug_tournament = Tournament(
date=date(2026, 6, 8),
name="contraption-league-2026-06-0810735",
uri="https://www.mtgo.com/decklist/contraption-league-2026-06-0810735",
formats=None,
json_file="contraption-league-2026-06-0810735.json",
)
engine = MTGOSyncEngine(cache_root=str(tmp_path), client=mock_client)
stats = engine.sync(
tournaments=[contraption_tournament, contraption_slug_tournament]
)

assert stats["skipped"] == 2
assert stats["created"] == 0
assert stats["failed"] == 0
assert mock_client.fetch_event_data.call_count == 0


def test_sync_does_not_skip_event_with_contraption_in_middle(tmp_path):
mock_client = MagicMock()
mock_client.fetch_event_data.return_value = {
"decklists": [{"player": "P1"}],
"player_count": {"players": "16"},
}
mock_item = MagicMock()
mock_item.to_dict.return_value = {
"Tournament": {
"Name": "Premodern Challenge 32 Contraption",
"PlayerCount": 16,
},
"Decks": [{"player": "P1"}],
}
mock_client.parse_event.return_value = mock_item

tournament = Tournament(
date=date(2025, 12, 30),
name="Premodern Challenge 32 Contraption",
uri="https://www.mtgo.com/decklist/premodern-challenge-32---contraption-2025-12-3012828126",
formats="Premodern",
json_file="premodern-challenge-32---contraption-2025-12-3012828126.json",
)
engine = MTGOSyncEngine(cache_root=str(tmp_path), client=mock_client)
stats = engine.sync(tournaments=[tournament])

assert stats["skipped"] == 0
assert stats["created"] == 1
assert mock_client.fetch_event_data.call_count == 1


def test_cli_parse_retry_args(monkeypatch):
from src.cli import parse_args

Expand Down
Loading