diff --git a/.github/workflows/qq-jobs-sync.yml b/.github/workflows/qq-jobs-sync.yml index ab287ab..9970500 100644 --- a/.github/workflows/qq-jobs-sync.yml +++ b/.github/workflows/qq-jobs-sync.yml @@ -2,7 +2,7 @@ name: 同步招聘信息 on: schedule: - - cron: "0 */3 * * *" + - cron: "17 * * * *" workflow_dispatch: inputs: probe_only: @@ -49,9 +49,11 @@ jobs: with: python-version: "3.12" - - name: 还原专用账号会话 + - name: 还原滚动专用账号会话 shell: bash env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} QQ_STORAGE_STATE_B64: ${{ secrets.QQ_DOCS_STORAGE_STATE_B64 }} run: | if [[ -z "$QQ_STORAGE_STATE_B64" ]]; then @@ -59,10 +61,47 @@ jobs: exit 1 fi STORAGE_STATE="$RUNNER_TEMP/qq-docs-storage-state.json" - printf '%s' "$QQ_STORAGE_STATE_B64" | base64 --decode > "$STORAGE_STATE" + SESSION_ARTIFACT_DIR="$RUNNER_TEMP/qq-docs-session-artifacts" + mkdir -p "$SESSION_ARTIFACT_DIR" + + session_key="$(printf '%s' "$QQ_STORAGE_STATE_B64" | sha256sum | cut -d ' ' -f 1)" + restored=false + while IFS= read -r run_id; do + candidate_dir="$SESSION_ARTIFACT_DIR/$run_id" + if gh run download "$run_id" \ + --name qq-docs-storage-state \ + --dir "$candidate_dir" >/dev/null 2>&1; then + encrypted_state="$candidate_dir/qq-docs-storage-state.json.enc" + if [[ ! -f "$encrypted_state" ]]; then + continue + fi + if SESSION_KEY="$session_key" openssl enc -d -aes-256-cbc -pbkdf2 \ + -in "$encrypted_state" \ + -out "$STORAGE_STATE" \ + -pass env:SESSION_KEY; then + restored=true + echo "已恢复最近一次成功运行刷新后的专用账号会话。" + break + fi + echo "::warning::滚动会话无法用当前 Secret 解密，继续查找或回退到初始会话。" + fi + done < <( + gh run list \ + --workflow qq-jobs-sync.yml \ + --branch "$DEFAULT_BRANCH" \ + --status success \ + --limit 20 \ + --json databaseId \ + --jq '.[].databaseId' + ) + + if [[ "$restored" != "true" ]]; then + printf '%s' "$QQ_STORAGE_STATE_B64" | base64 --decode > "$STORAGE_STATE" + echo "未找到滚动会话，已使用初始专用账号会话。" + fi chmod 600 "$STORAGE_STATE" STORAGE_STATE="$STORAGE_STATE" \ - python3 -c 'import json, os; json.load(open(os.environ["STORAGE_STATE"], encoding="utf-8"))' + python3 -c 'import json, os; state = json.load(open(os.environ["STORAGE_STATE"], encoding="utf-8")); assert isinstance(state.get("cookies"), list) and isinstance(state.get("origins"), list)' echo "STORAGE_STATE=$STORAGE_STATE" >> "$GITHUB_ENV" - name: 安装 Playwright Chromium @@ -90,6 +129,26 @@ jobs: fi python3 scripts/sync_qq_jobs.py "${args[@]}" + - name: 加密刷新后的专用账号会话 + shell: bash + env: + QQ_STORAGE_STATE_B64: ${{ secrets.QQ_DOCS_STORAGE_STATE_B64 }} + run: | + session_key="$(printf '%s' "$QQ_STORAGE_STATE_B64" | sha256sum | cut -d ' ' -f 1)" + SESSION_KEY="$session_key" openssl enc -aes-256-cbc -pbkdf2 -salt \ + -in "$STORAGE_STATE" \ + -out "$RUNNER_TEMP/qq-docs-storage-state.json.enc" \ + -pass env:SESSION_KEY + chmod 600 "$RUNNER_TEMP/qq-docs-storage-state.json.enc" + + - name: 保存滚动专用账号会话 + uses: actions/upload-artifact@v4 + with: + name: qq-docs-storage-state + path: ${{ runner.temp }}/qq-docs-storage-state.json.enc + if-no-files-found: error + retention-days: 30 + - name: 检查生成文件范围 id: changes if: ${{ !(github.event_name == 'workflow_dispatch' && inputs.probe_only) }} diff --git a/scripts/sync_qq_jobs.py b/scripts/sync_qq_jobs.py index 977260b..e2b2480 100644 --- a/scripts/sync_qq_jobs.py +++ b/scripts/sync_qq_jobs.py @@ -545,6 +545,7 @@ def schedule_capture(response): source_url=_url_with_tab(self.source_url, sheet_id), document_id=DOCUMENT_ID, ) + await context.storage_state(path=str(self.storage_state)) return snapshots finally: await browser.close() diff --git a/tests/test_sync_qq_jobs.py b/tests/test_sync_qq_jobs.py index 7fadb0d..1fb584b 100644 --- a/tests/test_sync_qq_jobs.py +++ b/tests/test_sync_qq_jobs.py @@ -1,13 +1,16 @@ +import asyncio import base64 import http.server import json import sys import tempfile import threading +import types import unittest import zlib from html.parser import HTMLParser from pathlib import Path +from unittest.mock import patch ROOT = Path(__file__).resolve().parents[1] @@ -16,6 +19,7 @@ from sync_qq_jobs import ( DatasetSpec, IntegrityError, + QQDocsSource, _resolve_sheet_id, assert_matching_scans, check_link_accessibility, @@ -203,6 +207,112 @@ def test_stable_sheet_id_disambiguates_duplicate_view_names(self): ) +class QQDocsSourceSessionTests(unittest.TestCase): + def test_complete_collection_persists_refreshed_browser_session(self): + class FakePage: + def on(self, _event, _callback): + pass + + def set_default_timeout(self, _timeout): + pass + + async def goto(self, _url, **_kwargs): + pass + + class FakeContext: + def __init__(self): + self.saved_path = None + + async def new_page(self): + return FakePage() + + async def storage_state(self, *, path): + self.saved_path = Path(path) + self.saved_path.write_text( + json.dumps({"cookies": [{"name": "refreshed"}], "origins": []}), + encoding="utf-8", + ) + + class FakeBrowser: + def __init__(self, context): + self.context = context + + async def new_context(self, **_kwargs): + return self.context + + async def close(self): + pass + + class FakeChromium: + def __init__(self, browser): + self.browser = browser + + async def launch(self, **_kwargs): + return self.browser + + class FakePlaywrightManager: + def __init__(self, chromium): + self.playwright = types.SimpleNamespace(chromium=chromium) + + async def __aenter__(self): + return self.playwright + + async def __aexit__(self, *_args): + pass + + class ImmediateSource(QQDocsSource): + async def _wait_for_workbook(self, _page, _captures): + return [{"id": "sheet_daily", "name": "每日更新"}] + + async def _wait_for_pages( + self, _page, _captures, _sheet_id, encourage_loading=False + ): + pass + + snapshot = { + "source": {"sheet_id": "sheet_daily", "view_name": "每日更新"}, + "schema": [], + "snapshot": { + "source_total": 0, + "fetched_count": 0, + "pagination_complete": True, + }, + "records": [], + } + with tempfile.TemporaryDirectory() as directory: + storage_state = Path(directory) / "storage-state.json" + storage_state.write_text('{"cookies": [], "origins": []}', encoding="utf-8") + spec = DatasetSpec( + "每日更新", + Path(directory) / "daily.json", + Path(directory) / "每日更新.md", + 1, + ) + context = FakeContext() + manager = FakePlaywrightManager(FakeChromium(FakeBrowser(context))) + async_api = types.ModuleType("playwright.async_api") + async_api.async_playwright = lambda: manager + playwright_module = types.ModuleType("playwright") + playwright_module.__path__ = [] + + with patch.dict( + sys.modules, + {"playwright": playwright_module, "playwright.async_api": async_api}, + ), patch("sync_qq_jobs.parse_sheet_pages", return_value=snapshot): + source = ImmediateSource( + "https://docs.qq.com/smartsheet/example", + storage_state, + timeout_seconds=0, + ) + asyncio.run(source.collect((spec,))) + + self.assertEqual(storage_state, context.saved_path) + self.assertEqual( + "refreshed", + json.loads(storage_state.read_text(encoding="utf-8"))["cookies"][0]["name"], + ) + + class MergeHistoryTests(unittest.TestCase): def test_first_missing_observation_keeps_record_and_marks_pending(self): previous = {