Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
File renamed without changes.
27 changes: 20 additions & 7 deletions .github/workflows/batch-assign.yml
Original file line number Diff line number Diff line change
@@ -1,12 +1,15 @@
name: Batch Assign Unknown Entities

env:
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}

on:
schedule:
- cron: "0 20 * * 1" # Monday – odp
- cron: "0 20 * * 2" # Tuesday – mandated
- cron: "0 20 * * 3" # Wednesday – odp
- cron: "0 20 * * 4" # Thursday – single-source
- cron: "0 20 * * 5" # Friday – odp
- cron: "0 19 * * 1" # Monday – odp
- cron: "0 19 * * 2" # Tuesday – mandated
- cron: "0 19 * * 3" # Wednesday – odp
- cron: "0 19 * * 4" # Thursday – single-source
- cron: "0 19 * * 5" # Friday – odp

repository_dispatch:
types: [auto-batch-assign]
Expand Down Expand Up @@ -153,7 +156,7 @@ jobs:

if [ "${BATCH_SIZE:-0}" -gt 0 ]; then
# Save the script before git checkout may overwrite it with an older version
cp bin/batch_assign_entities.py /tmp/batch_assign_entities.py
cp .github/scripts/batch_assign_entities.py /tmp/batch_assign_entities.py

# Switch to config-manager-update so pipeline files accumulate correctly across batches
if git show-ref --verify --quiet refs/heads/config-manager-update; then
Expand All @@ -164,10 +167,12 @@ jobs:

CURRENT_BATCH="${START_BATCH:-1}"
while true; do
set +e
python3 /tmp/batch_assign_entities.py "${ARGS[@]}" \
--batch-size "$BATCH_SIZE" \
--start-batch "$CURRENT_BATCH"
EXIT=$?
set -e
if [ $EXIT -eq 2 ]; then
echo "All batches complete."
break
Expand All @@ -178,7 +183,7 @@ jobs:
CURRENT_BATCH=$(( CURRENT_BATCH + 1 ))
done
else
python3 bin/batch_assign_entities.py "${ARGS[@]}"
python3 .github/scripts/batch_assign_entities.py "${ARGS[@]}"
fi

- name: Upload artifact
Expand All @@ -191,3 +196,11 @@ jobs:
issue_summary.csv
invalid_uri_issues.csv
issue_summary_full.csv

- name: Notify slack failure
if: failure() && github.ref == 'refs/heads/main'
uses: digital-land/github-action-slack-notify-build@main
with:
channel: planning-data-alerts
status: FAILED
color: danger
2 changes: 1 addition & 1 deletion tests/integration/test_batch_assign_entities.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import pandas as pd
import pytest

sys.path.insert(0, str(Path(__file__).parent.parent.parent / "bin"))
sys.path.insert(0, str(Path(__file__).parent.parent.parent / ".github/scripts"))

from batch_assign_entities import (
_collect_validation_rows,
Expand Down
5 changes: 4 additions & 1 deletion tests/integration/test_process_csv.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent.parent.parent / ".github/scripts"))
import requests
import pandas as pd
import pytest
import bin.batch_assign_entities as batch_assign_entities
import batch_assign_entities

@pytest.fixture
def setup_test_path(monkeypatch, tmp_path):
Expand Down
Loading