diff --git a/MANIFEST.in b/MANIFEST.in index 485e14d2c..7b97aacb3 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -63,3 +63,4 @@ graft src/lingtai/tools/bash/manual graft src/lingtai/tools/avatar/manual graft src/lingtai/tools/daemon/manual graft src/lingtai/tools/mcp/manual +graft src/lingtai/tools/web_search/manual diff --git a/src/lingtai/tools/ANATOMY.md b/src/lingtai/tools/ANATOMY.md index 07c32cba8..62febabc2 100644 --- a/src/lingtai/tools/ANATOMY.md +++ b/src/lingtai/tools/ANATOMY.md @@ -35,6 +35,7 @@ composes them. | `i18n/` | `en/zh/wen` string catalogs for every tool; registers into the kernel i18n cache via `register_strings` on import | | `_catalog.py` | Shared scan/manifest helpers for `knowledge` + `skills` | | `_file_paths.py` | `resolve_workdir_path` — shared by the five file tools | +| `_manual.py` | `load_installed_manual` — read-only loader for tools exposing an already-installed intrinsic manual skill | | `_media_host.py`, `_zhipu_mode.py` | Provider-host / z.ai-mode helpers for `vision` + `web_search` | **Tool sub-packages:** `email/`, `system/`, `psyche/`, `soul/`, `notification/` diff --git a/src/lingtai/tools/_manual.py b/src/lingtai/tools/_manual.py new file mode 100644 index 000000000..e9926b746 --- /dev/null +++ b/src/lingtai/tools/_manual.py @@ -0,0 +1,29 @@ +"""Shared loader for manuals installed in an agent's intrinsic skill catalog.""" +from __future__ import annotations + + +def load_installed_manual(agent, skill_name: str) -> dict: + """Return one installed intrinsic manual without mutating agent state.""" + manual_path = ( + agent._working_dir + / ".library" + / "intrinsic" + / "capabilities" + / skill_name + / "SKILL.md" + ) + if not manual_path.is_file(): + return { + "status": "degraded", + "manual": "", + "manual_path": str(manual_path), + "error": ( + f"{skill_name} manual missing — initializer may have failed or " + "capability not installed correctly" + ), + } + return { + "status": "ok", + "manual": manual_path.read_text(encoding="utf-8"), + "manual_path": str(manual_path), + } diff --git a/src/lingtai/tools/bash/__init__.py b/src/lingtai/tools/bash/__init__.py index 03ddec367..72e43de5d 100644 --- a/src/lingtai/tools/bash/__init__.py +++ b/src/lingtai/tools/bash/__init__.py @@ -22,6 +22,7 @@ from typing import TYPE_CHECKING from ._shell_dialect import ShellDialect, ShellInvocation, extract_posix_commands +from .._manual import load_installed_manual from ._async_supervisor import ( load_state, @@ -234,7 +235,7 @@ def get_description( "Returns exit_code, stdout, stderr, plus ok (bool) and command_status ('success'/'failed'). IMPORTANT: top-level status stays 'ok' even when the command FAILS — it only means the shell ran. " "Always check exit_code/ok and read the warning field (it names nonzero exits, Python tracebacks, and missing modules); never assume success from status alone. " "Avoid broad recursive scans (find … -name, rglob, os.walk, glob('**')) — they time out; prefer `rg --files`. Parse JSONL line-by-line, not as one JSON blob. " - "Supports async mode (async=true → job_id, then poll/cancel). Before using this tool, read the `shell-manual` skill — it covers async hygiene and advanced usage; no exceptions.") + "Supports async mode (async=true → job_id, then poll/cancel). Call shell(action='manual') to return the installed shell-manual skill. Before ordinary shell work, read that manual — it covers async hygiene and advanced usage; no exceptions.") def get_schema(lang: str = "en") -> dict: @@ -243,8 +244,8 @@ def get_schema(lang: str = "en") -> dict: "properties": { "action": { "type": "string", - "enum": ["run", "poll", "cancel"], - "description": "Action to perform: 'run' (default) executes a command, 'poll' checks async job status, 'cancel' kills an async job", + "enum": ["run", "poll", "cancel", "manual"], + "description": "Action to perform: 'run' (default) executes a command, 'poll' checks async job status, 'cancel' kills an async job, 'manual' returns the installed shell-manual skill", "default": "run", }, "command": { @@ -280,7 +281,7 @@ def get_schema(lang: str = "en") -> dict: "default": False, }, }, - "required": ["reminder"], # command/job_id are enforced per action; handler defaults omitted reminder for runtime compatibility + "required": [], # action-specific inputs are enforced by the handler; manual needs none } @@ -479,6 +480,8 @@ def _validate_reminder(value) -> tuple[float | None, dict | None]: def handle(self, args: dict) -> dict: action = args.get("action", "run") + if action == "manual": + return load_installed_manual(self._agent, "shell") if action == "poll": return self._handle_poll(args) if action == "cancel": diff --git a/src/lingtai/tools/daemon/__init__.py b/src/lingtai/tools/daemon/__init__.py index 2a0a46d96..67adaed84 100644 --- a/src/lingtai/tools/daemon/__init__.py +++ b/src/lingtai/tools/daemon/__init__.py @@ -39,6 +39,7 @@ from lingtai.kernel.tool_executor import ToolExecutor from lingtai.kernel.meta_block import attach_daemon_agent_meta from lingtai.kernel.trace_redaction import redact_text +from .._manual import load_installed_manual from lingtai.adapters.posix.process_identity import ( process_identity, process_identity_matches, @@ -1005,7 +1006,7 @@ def __getattr__(self, n): def get_description(lang: str = "en") -> str: - return 'Daemon (神識) — delegate work to ephemeral subagents for context isolation. Each is a disposable LLM session sharing your working directory, retaining no memory after completion. Use for noisy work where you only need the conclusion. Results truncated to ~2000 chars — instruct the emanation to write detailed output to a file. Actions: emanate (dispatch), list (status), ask (follow-up), check (inspect recent events), reclaim (kill all). Every terminal outcome is push-notified exactly once — done, failed, cancelled, or timed out — so after you dispatch you can safely go idle and wait for the notification; do not poll for "is it done". The notification carries the daemon id, terminal status, task summary, and the result/error path; act on it with daemon(action="check", id=...). LingTai daemons also receive compact; compact(action="manual") is read-only procedures, while explicit compact(action="run", _reason="...") is the repeatable sole-call context reset; action is required. Before using this tool, read the `daemon-manual` skill — it covers inspection patterns, polling cadence, preset/capability inheritance, and compact procedures; no exceptions.' + return 'Daemon (神識) — delegate work to ephemeral subagents for context isolation. Each is a disposable LLM session sharing your working directory, retaining no memory after completion. Use for noisy work where you only need the conclusion. Results truncated to ~2000 chars — instruct the emanation to write detailed output to a file. Actions: emanate (dispatch), list (status), ask (follow-up), check (inspect recent events), reclaim (kill all), manual (return the installed daemon-manual skill). Every terminal outcome is push-notified exactly once — done, failed, cancelled, or timed out — so after you dispatch you can safely go idle and wait for the notification; do not poll for "is it done". The notification carries the daemon id, terminal status, task summary, and the result/error path; act on it with daemon(action="check", id=...). LingTai daemons also receive compact; compact(action="manual") is read-only procedures, while explicit compact(action="run", _reason="...") is the repeatable sole-call context reset; action is required. Before using this tool, read the `daemon-manual` skill — it covers inspection patterns, polling cadence, preset/capability inheritance, and compact procedures; no exceptions.' def get_schema(lang: str = "en") -> dict: @@ -1014,8 +1015,8 @@ def get_schema(lang: str = "en") -> dict: "properties": { "action": { "type": "string", - "enum": ["emanate", "list", "ask", "check", "reclaim"], - "description": "Action to perform: 'emanate' (dispatch subagents), 'list' (show status), 'ask' (follow-up message), 'check' (read recent events of one emanation), 'reclaim' (kill all). LingTai emanations additionally receive compact(action='manual') for read-only procedures or explicit compact(action='run', _reason='...') for the non-terminal sole-call reset; action is required.", + "enum": ["emanate", "list", "ask", "check", "reclaim", "manual"], + "description": "Action to perform: 'emanate' (dispatch subagents), 'list' (show status), 'ask' (follow-up message), 'check' (read recent events of one emanation), 'reclaim' (kill all), 'manual' (return the installed daemon-manual skill). LingTai emanations additionally receive compact(action='manual') for read-only compaction procedures or explicit compact(action='run', _reason='...') for the non-terminal sole-call reset; action is required.", }, "tasks": { "type": "array", @@ -1527,6 +1528,8 @@ def _terminal_notification_text_from_state( def handle(self, args: dict) -> dict: action = args.get("action") + if action == "manual": + return load_installed_manual(self._agent, "daemon") backend = _normalize_backend(args.get("backend", "lingtai")) if action == "emanate": return self._handle_emanate( diff --git a/src/lingtai/tools/edit/__init__.py b/src/lingtai/tools/edit/__init__.py index a354cc6fd..b8d975ddb 100644 --- a/src/lingtai/tools/edit/__init__.py +++ b/src/lingtai/tools/edit/__init__.py @@ -7,25 +7,27 @@ from typing import TYPE_CHECKING from .._file_paths import resolve_workdir_path +from .._manual import load_installed_manual if TYPE_CHECKING: from lingtai.kernel.base_agent import BaseAgent def get_description(lang: str = "en") -> str: - return 'Replace an exact string in a file. Fails if old_string is not found or is ambiguous.' + return "Replace an exact string in a file. Fails if old_string is not found or is ambiguous. Call edit(action='manual') to return the installed file-manual skill." def get_schema(lang: str = "en") -> dict: return { "type": "object", "properties": { - "file_path": {"type": "string", "description": 'Absolute path to the file to edit'}, - "old_string": {"type": "string", "description": 'The exact text to find and replace'}, - "new_string": {"type": "string", "description": 'The replacement text'}, + "action": {"type": "string", "enum": ["manual"], "description": "Use action='manual' to return the installed file-manual skill without editing a file."}, + "file_path": {"type": "string", "description": "Absolute path to the file to edit. Required for ordinary edits; omit for action='manual'."}, + "old_string": {"type": "string", "description": "The exact text to find and replace. Required for ordinary edits; omit for action='manual'."}, + "new_string": {"type": "string", "description": "The replacement text. Required for ordinary edits; omit for action='manual'."}, "replace_all": {"type": "boolean", "description": 'Replace all occurrences', "default": False}, }, - "required": ["file_path", "old_string", "new_string"], + "required": [], } @@ -34,12 +36,18 @@ def setup(agent: "BaseAgent") -> None: """Set up the edit capability on an agent.""" def handle_edit(args: dict) -> dict: + if args.get("action") == "manual": + return load_installed_manual(agent, "file-manual") path = args.get("file_path", "") if not path: return {"status": "error", "message": "file_path is required"} + if "old_string" not in args: + return {"status": "error", "message": "old_string is required"} + if "new_string" not in args: + return {"status": "error", "message": "new_string is required"} path = resolve_workdir_path(agent, path) - old = args.get("old_string", "") - new = args.get("new_string", "") + old = args["old_string"] + new = args["new_string"] replace_all = args.get("replace_all", False) try: content = agent._file_io.read(path) diff --git a/src/lingtai/tools/email/__init__.py b/src/lingtai/tools/email/__init__.py index a84a618be..ec65e8ed0 100644 --- a/src/lingtai/tools/email/__init__.py +++ b/src/lingtai/tools/email/__init__.py @@ -27,6 +27,7 @@ from lingtai.kernel.notifications import register_generic_dismiss_guard +from .._manual import load_installed_manual register_generic_dismiss_guard( "email", @@ -83,6 +84,8 @@ def handle(agent, args: dict) -> dict: calls handle() before boot() in a test harness), return a clear error. """ action = args.get("action") + if action == "manual": + return load_installed_manual(agent, "email") if action == "unread": return { "status": "error", diff --git a/src/lingtai/tools/email/schema.py b/src/lingtai/tools/email/schema.py index bd3043408..1f3b4dbb0 100644 --- a/src/lingtai/tools/email/schema.py +++ b/src/lingtai/tools/email/schema.py @@ -5,7 +5,7 @@ def get_description(lang: str = "en") -> str: - return 'LingTai email protocol within your .lingtai/ network — NOT real internet email (for Gmail/Outlook use the imap tool). Addresses are bare paths under .lingtai/ with no @ signs (e.g. human for the operator). Reply discipline: always reply on the channel the message arrived on; prefer reply over send. Never reply via text output — that is your private diary, not a comms channel. Always address people by sender_nickname if set, else sender_name. For detailed usage, see the email-manual skill.' + return "LingTai email protocol within your .lingtai/ network — NOT real internet email (for Gmail/Outlook use the imap tool). Addresses are bare paths under .lingtai/ with no @ signs (e.g. human for the operator). Reply discipline: always reply on the channel the message arrived on; prefer reply over send. Never reply via text output — that is your private diary, not a comms channel. Always address people by sender_nickname if set, else sender_name. Call email(action='manual') to return the installed email-manual skill." def get_schema(lang: str = "en") -> dict: @@ -18,8 +18,9 @@ def get_schema(lang: str = "en") -> dict: "send", "check", "read", "dismiss", "reply", "reply_all", "search", "archive", "delete", "contacts", "add_contact", "remove_contact", "edit_contact", + "manual", ], - "description": 'send: send with optional cc/bcc (requires address, message; message body max 50,000 chars because unread bodies are injected in full into persistent notifications). check: list mailbox with preview of each email (up to 500 chars). read: fetch inbox emails by ID list (email_id=[id1, id2, ...]) AND marks each as read; ordinary unread content is already injected in notification_persistent.email, so prefer dismiss when you only need to clear handled mail. dismiss: same read-state effect as read but returns no bodies — preferred after handling content visible in persistent notification. reply: reply to email (requires email_id, message). reply_all: reply to all recipients. search: regex search mailbox. archive/delete: move/remove from inbox or archive. contacts/add_contact/remove_contact/edit_contact manage contacts.', + "description": 'send: send with optional cc/bcc (requires address, message; message body max 50,000 chars because unread bodies are injected in full into persistent notifications). check: list mailbox with preview of each email (up to 500 chars). read: fetch inbox emails by ID list (email_id=[id1, id2, ...]) AND marks each as read; ordinary unread content is already injected in notification_persistent.email, so prefer dismiss when you only need to clear handled mail. dismiss: same read-state effect as read but returns no bodies — preferred after handling content visible in persistent notification. reply: reply to email (requires email_id, message). reply_all: reply to all recipients. search: regex search mailbox. archive/delete: move/remove from inbox or archive. contacts/add_contact/remove_contact/edit_contact manage contacts. manual returns the installed email-manual skill without reading or changing mailbox state.', }, "address": { "oneOf": [ diff --git a/src/lingtai/tools/glob/__init__.py b/src/lingtai/tools/glob/__init__.py index 4ebfeb4de..fb89a3fb8 100644 --- a/src/lingtai/tools/glob/__init__.py +++ b/src/lingtai/tools/glob/__init__.py @@ -7,24 +7,26 @@ from typing import TYPE_CHECKING from .._file_paths import resolve_workdir_path +from .._manual import load_installed_manual if TYPE_CHECKING: from lingtai.kernel.base_agent import BaseAgent def get_description(lang: str = "en") -> str: - return "Find files matching a glob pattern. Use '**/' for recursive search (e.g. '**/*.py' finds all Python files). Returns sorted list of matching file paths." + return "Find files matching a glob pattern. Use '**/' for recursive search (e.g. '**/*.py' finds all Python files). Returns sorted list of matching file paths. Call glob(action='manual') to return the installed file-manual skill." def get_schema(lang: str = "en") -> dict: return { "type": "object", "properties": { - "pattern": {"type": "string", "description": "Glob pattern (e.g., '**/*.py')"}, + "action": {"type": "string", "enum": ["manual"], "description": "Use action='manual' to return the installed file-manual skill without searching."}, + "pattern": {"type": "string", "description": "Glob pattern (e.g., '**/*.py'). Required for ordinary searches; omit for action='manual'."}, "path": {"type": "string", "description": 'Directory to search in'}, "summary": {"type": "boolean", "description": 'Optional. Default false. When true, this tool runs normally and the raw result is preserved in the durable log (retrievable by tool_call_id), but before the result enters your context it is replaced by an LLM-generated summary driven by your `reasoning` field — so make `reasoning` specific about what to retain. Set true only when the output is expected to be large (>10k chars) and you do NOT need the exact raw text. Leave false when you need exact line/file/diff/stderr text. The summary is non-canonical; if the raw exceeds 500,000 chars no summary is generated and you get a refusal pointing at the preserved raw.', "default": False}, }, - "required": ["pattern"], + "required": [], } @@ -33,6 +35,8 @@ def setup(agent: "BaseAgent") -> None: """Set up the glob capability on an agent.""" def handle_glob(args: dict) -> dict: + if args.get("action") == "manual": + return load_installed_manual(agent, "file-manual") pattern = args.get("pattern", "") if not pattern: return {"status": "error", "message": "pattern is required"} diff --git a/src/lingtai/tools/grep/__init__.py b/src/lingtai/tools/grep/__init__.py index 2154faaa1..9f5a7e3ce 100644 --- a/src/lingtai/tools/grep/__init__.py +++ b/src/lingtai/tools/grep/__init__.py @@ -7,26 +7,28 @@ from typing import TYPE_CHECKING, Any from .._file_paths import resolve_workdir_path +from .._manual import load_installed_manual if TYPE_CHECKING: from lingtai.kernel.base_agent import BaseAgent def get_description(lang: str = "en") -> str: - return 'Search file contents for lines matching a regex pattern. Returns matching lines with file path and line number. Searches recursively when given a directory. Use the glob filter to narrow to specific file types.' + return "Search file contents for lines matching a regex pattern. Returns matching lines with file path and line number. Searches recursively when given a directory. Use the glob filter to narrow to specific file types. Call grep(action='manual') to return the installed file-manual skill." def get_schema(lang: str = "en") -> dict: return { "type": "object", "properties": { - "pattern": {"type": "string", "description": 'Regex pattern to search for'}, + "action": {"type": "string", "enum": ["manual"], "description": "Use action='manual' to return the installed file-manual skill without searching."}, + "pattern": {"type": "string", "description": "Regex pattern to search for. Required for ordinary searches; omit for action='manual'."}, "path": {"type": "string", "description": 'File or directory to search in'}, "glob": {"type": "string", "description": "File glob filter (e.g., '*.py')", "default": "*"}, "max_matches": {"type": "integer", "description": 'Maximum matches to return', "default": 200}, "summary": {"type": "boolean", "description": 'Optional. Default false. When true, this tool runs normally and the raw result is preserved in the durable log (retrievable by tool_call_id), but before the result enters your context it is replaced by an LLM-generated summary driven by your `reasoning` field — so make `reasoning` specific about what to retain. Set true only when the output is expected to be large (>10k chars) and you do NOT need the exact raw text. Leave false when you need exact line/file/diff/stderr text. The summary is non-canonical; if the raw exceeds 500,000 chars no summary is generated and you get a refusal pointing at the preserved raw.', "default": False}, }, - "required": ["pattern"], + "required": [], } @@ -35,6 +37,8 @@ def setup(agent: "BaseAgent") -> None: """Set up the grep capability on an agent.""" def handle_grep(args: dict) -> dict: + if args.get("action") == "manual": + return load_installed_manual(agent, "file-manual") pattern = args.get("pattern", "") if not pattern: return {"status": "error", "message": "pattern is required"} diff --git a/src/lingtai/tools/psyche/__init__.py b/src/lingtai/tools/psyche/__init__.py index fca5565a1..6c2d51d96 100644 --- a/src/lingtai/tools/psyche/__init__.py +++ b/src/lingtai/tools/psyche/__init__.py @@ -31,6 +31,7 @@ # Molt (the public surface) from ._molt import _context_molt, _name_set, _name_nickname, context_forget # noqa: F401 +from .._manual import load_installed_manual # --------------------------------------------------------------------------- @@ -39,7 +40,7 @@ def get_description(lang: str = "en") -> str: - return 'Identity, pad, and context management — three objects, all molt-surviving. lingtai: your 灵台 (character) — update after significant work or before molt. pad: system-prompt sketchboard (system/pad.md) — plans, tasks, notes. context: molt (凝蜕) — shed conversation, keep stores. name: set true name (once) or change nickname. See psyche-manual skill for full molt/pad guidance.' + return 'Identity, pad, and context management — three objects, all molt-surviving. lingtai: your 灵台 (character) — update after significant work or before molt. pad: system-prompt sketchboard (system/pad.md) — plans, tasks, notes. context: molt (凝蜕) — shed conversation, keep stores. name: set true name (once) or change nickname. Call psyche(action="manual") to return the installed psyche-manual skill.' def get_schema(lang: str = "en") -> dict: @@ -53,7 +54,7 @@ def get_schema(lang: str = "en") -> dict: }, "action": { "type": "string", - "description": 'lingtai: update | load. update auto-loads.\npad: edit | load | append. edit auto-loads. append pins files as read-only reference.\ncontext: molt. Requires `summary` — tend the four stores BEFORE molting. See psyche-manual.\nname: set (true name, once) | nickname (display name, mutable).', + "description": 'lingtai: update | load. update auto-loads.\npad: edit | load | append. edit auto-loads. append pins files as read-only reference.\ncontext: molt. Requires `summary` — tend the four stores BEFORE molting. See psyche-manual.\nname: set (true name, once) | nickname (display name, mutable).\nmanual: return the installed psyche-manual skill; object is omitted.', }, "content": { "type": "string", @@ -82,7 +83,7 @@ def get_schema(lang: str = "en") -> dict: "description": 'REQUIRED for context molt. The path to the session-journal entry you wrote for the just-finished segment BEFORE molting: knowledge/session-journal//KNOWLEDGE.md (a per-segment sub-entry, NOT the parent index). Must be inside your workdir, exist, be non-empty UTF-8, have valid YAML frontmatter with `name` and `description`, and identify itself as session knowledge via `type: session-journal` or `session_journal: true`. The molt is refused before any context is shed if this is missing or invalid. See psyche-manual §4.', }, }, - "required": ["object", "action"], + "required": ["action"], } @@ -112,6 +113,9 @@ def handle(agent, args: dict) -> dict: obj = args.get("object", "") action = args.get("action", "") + if action == "manual": + return load_installed_manual(agent, "psyche-manual") + valid = _VALID_ACTIONS.get(obj) if valid is None: return { diff --git a/src/lingtai/tools/read/__init__.py b/src/lingtai/tools/read/__init__.py index adf061c3e..b68b0055a 100644 --- a/src/lingtai/tools/read/__init__.py +++ b/src/lingtai/tools/read/__init__.py @@ -10,6 +10,7 @@ from lingtai.kernel.tool_result_artifacts import PREVENTIVE_MAX_CHARS from .._file_paths import resolve_workdir_path +from .._manual import load_installed_manual if TYPE_CHECKING: from lingtai.kernel.base_agent import BaseAgent @@ -51,20 +52,21 @@ def _resolve_call_cap(agent: "BaseAgent", requested_max_chars: object) -> int: def get_description(lang: str = "en") -> str: - return "Read the contents of a text file. Returns numbered lines. Text files only — cannot read binary, images, or audio. Before using read, especially for large files, complete-content workflows, truncation, or line_truncated handling, read the read-manual skill. If the file is non-UTF-8 or needs careful search/edit workflow, read file-manual first. Use offset/limit for line windows and optional max_chars to choose the per-call character budget. Default read budget is 100 000 characters; max_chars can raise/lower that per call but is clamped by the non-configurable runtime hard cap of 200 000 characters. A successful read can still be truncated: check truncated=true, cap_chars, returned_chars, next_offset, remaining_lines_estimate, and line_truncated. Continue with next_offset until done. If line_truncated=true, the shown physical line is only a prefix; next_offset skips to the next line and does not recover the hidden tail. Use the read-manual's bash/Python metadata/stats workflow (file size, line count, longest line) and targeted grep/sed/Python processing for such content." + return "Read the contents of a text file. Returns numbered lines. Text files only — cannot read binary, images, or audio. Call read(action='manual') to return the installed read-manual skill. Before ordinary reads, especially for large files, complete-content workflows, truncation, or line_truncated handling, read that manual. If the file is non-UTF-8 or needs careful search/edit workflow, read file-manual first. Use offset/limit for line windows and optional max_chars to choose the per-call character budget. Default read budget is 100 000 characters; max_chars can raise/lower that per call but is clamped by the non-configurable runtime hard cap of 200 000 characters. A successful read can still be truncated: check truncated=true, cap_chars, returned_chars, next_offset, remaining_lines_estimate, and line_truncated. Continue with next_offset until done. If line_truncated=true, the shown physical line is only a prefix; next_offset skips to the next line and does not recover the hidden tail. Use the read-manual's bash/Python metadata/stats workflow (file size, line count, longest line) and targeted grep/sed/Python processing for such content." def get_schema(lang: str = "en") -> dict: return { "type": "object", "properties": { - "file_path": {"type": "string", "description": 'Absolute path to the file to read'}, + "action": {"type": "string", "enum": ["manual"], "description": "Use action='manual' to return the installed read-manual skill without reading a target file."}, + "file_path": {"type": "string", "description": "Absolute path to the file to read. Required for ordinary reads; omit for action='manual'."}, "offset": {"type": "integer", "description": 'Line number to start from (1-based)', "default": 1}, "limit": {"type": "integer", "description": 'Max lines to read', "default": 2000}, "max_chars": {"type": "integer", "description": 'Optional per-call character budget for read content. Defaults to 100 000; values above the runtime hard cap are clamped to 200 000. Use read-manual before setting this for large files.'}, "summary": {"type": "boolean", "description": 'Optional. Default false. When true, this tool runs normally and the raw result is preserved in the durable log (retrievable by tool_call_id), but before the result enters your context it is replaced by an LLM-generated summary driven by your `reasoning` field — so make `reasoning` specific about what to retain. Set true only when the output is expected to be large (>10k chars) and you do NOT need the exact raw text. Leave false when you need exact line/file/diff/stderr text. The summary is non-canonical; if the raw exceeds 500,000 chars no summary is generated and you get a refusal pointing at the preserved raw.', "default": False}, }, - "required": ["file_path"], + "required": [], } @@ -129,6 +131,8 @@ def setup(agent: "BaseAgent") -> None: """Set up the read capability on an agent.""" def handle_read(args: dict) -> dict: + if args.get("action") == "manual": + return load_installed_manual(agent, "read-manual") path = args.get("file_path", "") if not path: return {"status": "error", "message": "file_path is required"} diff --git a/src/lingtai/tools/soul/__init__.py b/src/lingtai/tools/soul/__init__.py index 8a5b4a0dd..ab6349066 100644 --- a/src/lingtai/tools/soul/__init__.py +++ b/src/lingtai/tools/soul/__init__.py @@ -22,6 +22,7 @@ # Re-export constants from config.py from lingtai.kernel.config import DEFAULT_SOUL_DELAY_SECONDS +from .._manual import load_installed_manual from .config import ( SOUL_DELAY_MIN_SECONDS, CONSULTATION_PAST_COUNT_MIN, @@ -79,7 +80,7 @@ def get_description(lang: str = "en") -> str: - return "Your inner voice. flow is OPT-IN and DISABLED by default: it runs only when the operator sets env LINGTAI_SOUL_FLOW_ENABLED=1 (then refreshes). While disabled, soul(action='flow') returns status='disabled' (not an error — do not retry); inquiry/config/voice/dismiss still work. When enabled, flow fires periodic past-self consultation every soul_delay seconds while IDLE — M=1+K parallel LLM calls (1 stepped-back read of current chat + K past-snapshot voices) arrive as an involuntary soul(action='flow') pair. delay_seconds is only the cadence after opt-in, NOT an off switch. inquiry: ask a deep copy of yourself a question; answer returns in the tool result. config: tune flow knobs at runtime (delay_seconds, consultation_past_count) — does not enable flow. dismiss: clear the current flow notification. See soul-manual skill." + return "Your inner voice. flow is OPT-IN and DISABLED by default: it runs only when the operator sets env LINGTAI_SOUL_FLOW_ENABLED=1 (then refreshes). While disabled, soul(action='flow') returns status='disabled' (not an error — do not retry); inquiry/config/voice/dismiss still work. When enabled, flow fires periodic past-self consultation every soul_delay seconds while IDLE — M=1+K parallel LLM calls (1 stepped-back read of current chat + K past-snapshot voices) arrive as an involuntary soul(action='flow') pair. delay_seconds is only the cadence after opt-in, NOT an off switch. inquiry: ask a deep copy of yourself a question; answer returns in the tool result. config: tune flow knobs at runtime (delay_seconds, consultation_past_count) — does not enable flow. dismiss: clear the current flow notification. manual: return the installed soul-manual skill. See soul-manual for details." def get_schema(lang: str = "en") -> dict: @@ -88,8 +89,8 @@ def get_schema(lang: str = "en") -> dict: "properties": { "action": { "type": "string", - "enum": ["inquiry", "flow", "config", "voice", "dismiss"], - "description": "inquiry: ask yourself a question — the answer returns in the tool result. Requires 'inquiry' parameter. Always available. flow: OPT-IN, DISABLED by default. Only runs when the operator sets env LINGTAI_SOUL_FLOW_ENABLED=1 (case-insensitive true/yes/on) and refreshes/restarts. While DISABLED, invoking flow returns {status:'disabled', enabled:false} with an explanation — this is expected config state, NOT an error, so do NOT retry it; the operator must set the env var first. When ENABLED, flow fires automatically every soul_delay seconds while IDLE — appears in history as a soul(action='flow') call you did not initiate, with voices from past selves and a stepped-back read of your current chat. You may ALSO invoke flow voluntarily while ACTIVE: the call returns immediately with a success acknowledgement, and the actual voices arrive shortly after as a separate involuntary soul(action='flow') pair. If a fire is already running when you invoke, the call is rejected with 'soul flow ongoing, request rejected' — wait for the current fire to land, then try again. config: tune flow knobs — pass any subset of delay_seconds (wall-clock cadence, min 30s), consultation_past_count (K voices per fire, 0–5). At least one field required. Persists to init.json. config does NOT enable flow — delay_seconds is only cadence after the env opt-in, never an off switch. voice: choose how your own soul-flow voice sounds. Bare (no 'set') reads the current voice + the resolved prompt. Pass set='inner' or set='observer' to switch presets. Pass set='custom' with a 'prompt' field to write your own — speak to yourself as the soul, describe how you want to be framed when reading your own diary. Persists to init.json. This is yours; the operator does not choose it for you. dismiss: clear the current soul flow notification from the notification panel. Use when you've read the voices and want to dismiss them before the next fire replaces them. inquiry/config/voice/dismiss all work whether or not flow is enabled. See soul-manual skill for enabling/disabling, troubleshooting, and the privacy/cost rationale.", + "enum": ["inquiry", "flow", "config", "voice", "dismiss", "manual"], + "description": "inquiry: ask yourself a question — the answer returns in the tool result. Requires 'inquiry' parameter. Always available. flow: OPT-IN, DISABLED by default. Only runs when the operator sets env LINGTAI_SOUL_FLOW_ENABLED=1 (case-insensitive true/yes/on) and refreshes/restarts. While DISABLED, invoking flow returns {status:'disabled', enabled:false} with an explanation — this is expected config state, NOT an error, so do NOT retry it; the operator must set the env var first. When ENABLED, flow fires automatically every soul_delay seconds while IDLE — appears in history as a soul(action='flow') call you did not initiate, with voices from past selves and a stepped-back read of your current chat. You may ALSO invoke flow voluntarily while ACTIVE: the call returns immediately with a success acknowledgement, and the actual voices arrive shortly after as a separate involuntary soul(action='flow') pair. If a fire is already running when you invoke, the call is rejected with 'soul flow ongoing, request rejected' — wait for the current fire to land, then try again. config: tune flow knobs — pass any subset of delay_seconds (wall-clock cadence, min 30s), consultation_past_count (K voices per fire, 0–5). At least one field required. Persists to init.json. config does NOT enable flow — delay_seconds is only cadence after the env opt-in, never an off switch. voice: choose how your own soul-flow voice sounds. Bare (no 'set') reads the current voice + the resolved prompt. Pass set='inner' or set='observer' to switch presets. Pass set='custom' with a 'prompt' field to write your own — speak to yourself as the soul, describe how you want to be framed when reading your own diary. Persists to init.json. This is yours; the operator does not choose it for you. dismiss: clear the current soul flow notification from the notification panel. Use when you've read the voices and want to dismiss them before the next fire replaces them. inquiry/config/voice/dismiss all work whether or not flow is enabled. manual returns the installed soul-manual skill without changing soul state. See that manual for enabling/disabling, troubleshooting, and the privacy/cost rationale.", }, "inquiry": { "type": "string", @@ -130,6 +131,9 @@ def handle(agent, args: dict) -> dict: """ action = args.get("action", "") + if action == "manual": + return load_installed_manual(agent, "soul-manual") + if action == "flow": # Opt-in gate: soul flow is disabled by default. When disabled, # return an explicit, stable "disabled" status BEFORE touching the @@ -243,6 +247,6 @@ def _fire(): return { "error": ( f"Unknown soul action: {action}. Use inquiry, config, voice, dismiss, " - "or wait for flow (mechanical)." + "manual, or wait for flow (mechanical)." ) } diff --git a/src/lingtai/tools/system/__init__.py b/src/lingtai/tools/system/__init__.py index 78bfb6d63..be41f0ee3 100644 --- a/src/lingtai/tools/system/__init__.py +++ b/src/lingtai/tools/system/__init__.py @@ -10,6 +10,7 @@ clear — force a full molt on another agent (requires karma) nirvana — permanently destroy an agent's working directory (requires nirvana) presets — list available presets in the agent's library + manual — return the installed system-manual skill without mutation summarize — record an agent-authored compact replacement for a prior tool-result block in runtime history. Pick targets from ``_meta.agent_meta.agent_state.current_tool_result_chars.top_results``. @@ -44,6 +45,7 @@ # Schema (tool registration) from .schema import get_description, get_schema # noqa: F401 +from .._manual import load_installed_manual # Summarize — agent-authored context summarization from .summarize import _summarize, SUMMARIZE_MARKER # noqa: F401 @@ -86,6 +88,11 @@ # --------------------------------------------------------------------------- +def _manual(agent, args: dict) -> dict: + """Return the installed system manual without changing runtime state.""" + return load_installed_manual(agent, "system-manual") + + def handle(agent, args: dict) -> dict: """Handle system tool — runtime, lifecycle, synchronization. @@ -110,6 +117,7 @@ def handle(agent, args: dict) -> dict: "nirvana": _nirvana, "presets": _presets, "summarize": _summarize, + "manual": _manual, }.get(action) if handler is None: return {"status": "error", "message": f"Unknown system action: {action}"} diff --git a/src/lingtai/tools/system/schema.py b/src/lingtai/tools/system/schema.py index 2a7453936..7c6c4b9e7 100644 --- a/src/lingtai/tools/system/schema.py +++ b/src/lingtai/tools/system/schema.py @@ -3,7 +3,7 @@ def get_description(lang: str = "en") -> str: - return 'Runtime inspection, lifecycle control, synchronization, and inter-agent management.\n\nSelf-actions (no permissions needed): sleep, refresh, summarize.\nKarma actions (require admin.karma=True): lull, interrupt, suspend, cpr, clear.\nNirvana (require admin.karma=True AND admin.nirvana=True): nirvana.\n\nNotification verbs (check/dismiss) are NOT here — they live on the standalone notification tool. See system-manual skill for detailed usage of each action.' + return 'Runtime inspection, lifecycle control, synchronization, and inter-agent management.\n\nSelf-actions (no permissions needed): sleep, refresh, summarize.\nKarma actions (require admin.karma=True): lull, interrupt, suspend, cpr, clear.\nNirvana (require admin.karma=True AND admin.nirvana=True): nirvana.\n\nNotification verbs (check/dismiss) are NOT here — they live on the standalone notification tool. Call system(action="manual") to return the installed system-manual skill.' def get_schema(lang: str = "en") -> dict: @@ -12,8 +12,8 @@ def get_schema(lang: str = "en") -> dict: "properties": { "action": { "type": "string", - "enum": ["refresh", "sleep", "lull", "interrupt", "suspend", "cpr", "clear", "nirvana", "presets", "summarize"], - "description": "refresh: rebuild from init.json — same identity, preserved conversation. Reloads MCP, capabilities, addons, LLM, prompt sections. See system-manual.\n\npresets: list available presets with tags, connectivity, capabilities. See system-manual.\n\nsleep: go to sleep until mail wakes you. Self only.\n\nlull: put another agent to sleep (karma).\n\nsuspend: freeze another agent (karma).\n\ncpr: resuscitate suspended agent (karma).\n\ninterrupt: cancel another agent's turn (karma).\n\nclear: force molt on another agent (karma). See system-manual.\n\nnirvana: permanently destroy an agent (karma + nirvana). See system-manual.\n\nsummarize: record an agent-authored compact replacement for one or more prior tool-result blocks in runtime history. Pass items=[{tool_call_id, summary}, ...]. The original result remains in events.jsonl; the active provider context may still contain the old raw result until a rebuild applies it (manual rebuild=true, or the runtime's forced rebuild at the 1.0 full-context hard boundary). Use after digesting a large result to free context budget. Pass rebuild=true (default false) to also request a provider-context rebuild that applies the pending summaries now; rebuild=true with no items is a pure rebuild of already-pending summaries. Do not loop rebuild/summarize. Choose the tool_call_ids to compress from _meta.agent_meta.agent_state.current_tool_result_chars.top_results (the ranked list of the largest formal results in context) — large results are surfaced there, not pushed as notifications. (Legacy: if a stale large_tool_result reminder still exists in system.json, a successful summarize of its tool_call_id also clears it.) To read or dismiss notifications, use the notification tool. See system-manual.", + "enum": ["refresh", "sleep", "lull", "interrupt", "suspend", "cpr", "clear", "nirvana", "presets", "summarize", "manual"], + "description": "refresh: rebuild from init.json — same identity, preserved conversation. Reloads MCP, capabilities, addons, LLM, prompt sections. See system-manual.\n\npresets: list available presets with tags, connectivity, capabilities. See system-manual.\n\nsleep: go to sleep until mail wakes you. Self only.\n\nlull: put another agent to sleep (karma).\n\nsuspend: freeze another agent (karma).\n\ncpr: resuscitate suspended agent (karma).\n\ninterrupt: cancel another agent's turn (karma).\n\nclear: force molt on another agent (karma). See system-manual.\n\nnirvana: permanently destroy an agent (karma + nirvana). See system-manual.\n\nsummarize: record an agent-authored compact replacement for one or more prior tool-result blocks in runtime history. Pass items=[{tool_call_id, summary}, ...]. The original result remains in events.jsonl; the active provider context may still contain the old raw result until a rebuild applies it (manual rebuild=true, or the runtime's forced rebuild at the 1.0 full-context hard boundary). Use after digesting a large result to free context budget. Pass rebuild=true (default false) to also request a provider-context rebuild that applies the pending summaries now; rebuild=true with no items is a pure rebuild of already-pending summaries. Do not loop rebuild/summarize. Choose the tool_call_ids to compress from _meta.agent_meta.agent_state.current_tool_result_chars.top_results (the ranked list of the largest formal results in context) — large results are surfaced there, not pushed as notifications. (Legacy: if a stale large_tool_result reminder still exists in system.json, a successful summarize of its tool_call_id also clears it.) To read or dismiss notifications, use the notification tool. manual returns the installed system-manual skill without changing runtime state.", }, "reason": { "type": "string", diff --git a/src/lingtai/tools/web_search/ANATOMY.md b/src/lingtai/tools/web_search/ANATOMY.md index f81272e41..9b6c02b73 100644 --- a/src/lingtai/tools/web_search/ANATOMY.md +++ b/src/lingtai/tools/web_search/ANATOMY.md @@ -2,6 +2,7 @@ related_files: - src/lingtai/tools/ANATOMY.md - src/lingtai/tools/web_search/__init__.py + - src/lingtai/tools/web_search/manual/SKILL.md - src/lingtai/services/websearch/ANATOMY.md - src/lingtai/tools/web_search/glossary-en.md - src/lingtai/tools/web_search/glossary-zh.md @@ -23,12 +24,14 @@ Web search capability — web lookup via pluggable SearchService backends. | File | LOC | Role | |---|---|---| -| `__init__.py` | 130 | `WebSearchManager`, `setup()`, provider registry, tool schema | +| `__init__.py` | 146 | `WebSearchManager`, `setup()`, provider registry, search/manual tool schema | +| `manual/` | 26 files | TUI-derived `web-search-manual` router, references, assets, and extraction scripts | **Key symbols:** - `PROVIDERS` (L20-24) — supported: `duckduckgo`, `minimax`, `zhipu`, `gemini`, `anthropic`, `openai`. Default: `duckduckgo`. Fallback on inherit: `duckduckgo`. -- `WebSearchManager` (L41) — handles tool calls; delegates to `SearchService.search()`. -- `setup()` (L77) — entry point. Creates manager, registers `"web_search"` tool (L124-129). +- `WebSearchManager` — returns the installed manual before touching query/service state, otherwise delegates to `SearchService.search()`. +- `setup()` — entry point. Creates manager and registers the `"web_search"` tool. +- `manual/SKILL.md` — collision-safe `web-search-manual` root synchronized from the TUI web-browsing bundle; relative scripts/references ship together. ## Connections @@ -36,20 +39,24 @@ Web search capability — web lookup via pluggable SearchService backends. - **→ `lingtai.services.websearch.SearchService`** (L15) — abstract service interface + `create_search_service()` factory. - **→ `capabilities._media_host.resolve_media_host`** (L110) — injected for non-duckduckgo providers. - **→ `capabilities._zhipu_mode.resolve_z_ai_mode`** (L113) — injected for `zhipu` provider. -- **→ `lingtai.kernel.base_agent.BaseAgent`** — type-only (L18). +- **→ `lingtai.kernel.base_agent.BaseAgent`** — type-only. +- **→ `tools._manual.load_installed_manual`** — read-only `action="manual"` loader for `.library/intrinsic/capabilities/web_search/SKILL.md`. +- **→ `Agent._install_intrinsic_manuals()`** — copies `manual/` wholesale into each agent’s intrinsic skill catalog. - **← `capabilities.__init__`** — registered as `".web_search"` in `_BUILTIN`. ## Composition -Single file. No internal state — `WebSearchManager` instances hold agent + service refs. +One code module plus a self-contained manual bundle. `WebSearchManager` instances hold agent + service refs; manual files are immutable package data copied into the agent library. ## State - `WebSearchManager._agent` / `_search_service` (L49-50) — per-agent instance. Service can be `None` (returns error on call, L57-64). - `PROVIDERS` dict is module-level constant. +- `action="manual"` reads installed package data only; it neither constructs nor calls a search service. ## Notes - Graceful fallback (L97-105): unsupported providers fall back to `duckduckgo` (with `api_key=None`). Unlike vision, this never skips — always provides search. - No-provider default (L119-120): if neither `search_service` nor `provider` is given, defaults to `duckduckgo`. -- Results are formatted as markdown `**title**\nurl\nsnippet` (L71-73). +- Results are formatted as markdown `**title**\nurl\nsnippet`. +- The manual bundle keeps `` references portable and uses the distinct `web-search-manual` frontmatter name so a TUI `web-browsing` utility can coexist. diff --git a/src/lingtai/tools/web_search/CONTRACT.md b/src/lingtai/tools/web_search/CONTRACT.md index 28811b656..89e7db17a 100644 --- a/src/lingtai/tools/web_search/CONTRACT.md +++ b/src/lingtai/tools/web_search/CONTRACT.md @@ -5,6 +5,7 @@ contract_version: 1 related_files: - src/lingtai/tools/web_search/__init__.py - src/lingtai/tools/web_search/ANATOMY.md + - src/lingtai/tools/web_search/manual/SKILL.md maintenance: | Keep related_files as repo-relative paths to real files. If behavior and this contract disagree, the code is the source of truth — fix the contract in the @@ -13,9 +14,10 @@ maintenance: | # Web search capability contract -`web_search` performs a single web lookup via a `SearchService` and returns -formatted results. It is a thin tool over a provider service; provider selection -and fallback happen at setup time. The implementation lives in +`web_search` either returns its installed manual bundle or performs a single +web lookup via a `SearchService` and returns formatted results. The ordinary +search path is a thin tool over a provider service; provider selection and +fallback happen at setup time. The implementation lives in `src/lingtai/tools/web_search/`; the code is the source of truth. ## Routing Card @@ -23,7 +25,7 @@ and fallback happen at setup time. The implementation lives in **Use this when:** - You are editing the web-search tool surface, its `SearchService` wiring, or the provider-resolution / graceful-fallback logic in `setup`. -- You are reviewing result formatting or the default/duckduckgo behavior. +- You are reviewing result formatting, the default/duckduckgo behavior, or the packaged web-search manual bundle. **Do not use this for:** - Image understanding: use `vision` (see `src/lingtai/tools/vision/CONTRACT.md`). @@ -31,13 +33,13 @@ and fallback happen at setup time. The implementation lives in - Adding a provider service implementation: that lives under `src/lingtai/services/websearch/`, imported lazily from here. -**Fast paths:** tool schema -> §Tool surface; provider list / default -> -§Scope; lazy-import DAG rule -> §Cross-platform invariants. +**Fast paths:** tool schema/manual action -> §Tool surface; provider list / default -> +§Scope; bundle/package and lazy-import invariants -> §Cross-platform invariants. ## Scope - Canonical tool name: `web_search`. -- One tool, one call — no actions, no persistent state. +- One tool, two modes, no persistent state: omit `action` for a search or use `action="manual"` for the installed manual bundle. - Advertised providers (`PROVIDERS["providers"]`): `duckduckgo`, `minimax`, `zhipu`, `gemini`, `anthropic`, `openai`. Default provider is `duckduckgo`, and `fallback_on_inherit` is `duckduckgo`. @@ -45,22 +47,26 @@ and fallback happen at setup time. The implementation lives in `capability_fallback` and falls back to `duckduckgo` (with no credentials) — it never raises. -**Non-goals:** the capability does not fetch or crawl page bodies, deduplicate, -or rank beyond what the underlying `SearchService` returns. It keeps no state. +**Non-goals:** the ordinary search path does not fetch or crawl page bodies, +deduplicate, or rank beyond what the underlying `SearchService` returns. The +manual action only returns documentation and bundled helper scripts; it does not +execute those scripts, install dependencies, or authorize external side effects. ## Tool surface -Schema requires `query`. +The flat provider-compatible schema exposes optional `action` and `query`; +handlers enforce the mode-specific requirement at runtime. -| Inputs | Optional inputs | Success output | Error shapes | +| Mode | Inputs | Success output | Error shapes | |---|---|---|---| -| `query` (required) | — | `{status: "ok", results: }` (each result rendered as `**title**\n\n`, joined by blank lines; `"No results found."` when empty) | `{status: "error", message}` — missing `query` (`Missing required parameter: query`), no configured service (`No SearchService configured. ...`), or `Web search failed: ` | +| Search (omit `action`) | `query` (runtime-required) | `{status: "ok", results: }` (each result rendered as `**title**\n\n`, joined by blank lines; `"No results found."` when empty) | `{status: "error", message}` — missing `query` (`Missing required parameter: query`), no configured service (`No SearchService configured. ...`), or `Web search failed: ` | +| Manual (`action="manual"`) | none | `{status: "ok", manual, manual_path}` for `.library/intrinsic/capabilities/web_search/SKILL.md` | `{status: "degraded", manual: "", manual_path, error}` when the installed bundle is missing | ## State & storage -None. `web_search` issues one query through the configured `SearchService` and -returns the formatted results inline. It writes no files and keeps no per-call -state under the agent working directory. +None. Ordinary search issues one query through the configured `SearchService` +and returns formatted results inline. Manual mode reads the installer-owned +`SKILL.md` and writes nothing. Neither mode keeps per-call state. ## Cross-platform invariants @@ -75,6 +81,11 @@ DOCUMENT ONLY — do not change these assumptions and do not propose Windows wor - Provider resolution is graceful: an unsupported provider degrades to the `duckduckgo` fallback rather than raising, so an agent always ends up with a working search service. +- The TUI-derived 26-file manual bundle ships in wheels via the generic package-data + glob and in sdists via the explicit `MANIFEST.in` graft. Active helper examples + use `/scripts`; only the historical v2 migration note names a TUI path. +- Root frontmatter is `name: web-search-manual`, avoiding a catalog collision with + a separately installed TUI `web-browsing` skill. There are no subprocess/shell/PTY/binary-spawn assumptions in this tool. @@ -85,6 +96,7 @@ There are no subprocess/shell/PTY/binary-spawn assumptions in this tool. | `setup` registers the `web_search` tool | `src/lingtai/tools/web_search/__init__.py` | `tests/test_web_search_capability.py::test_web_search_added_by_capability`, `::test_web_search_with_provider_kwarg` | | The manager delegates the query to the `SearchService` | `src/lingtai/tools/web_search/__init__.py` | `tests/test_web_search_capability.py::test_web_search_manager_uses_search_service`, `::test_web_search_with_dedicated_service` | | Missing `query` is a structured error | `src/lingtai/tools/web_search/__init__.py` | `tests/test_web_search_capability.py::test_web_search_missing_query` | +| Manual mode returns the installed bundle before query/service validation | `src/lingtai/tools/web_search/__init__.py`, `manual/SKILL.md` | `tests/test_intrinsic_manual_actions.py::test_manual_actions_return_their_installed_skills`, `tests/test_skills.py::test_skills_setup_hard_copies_intrinsics` | | Service exceptions are caught and returned as errors | `src/lingtai/tools/web_search/__init__.py` | `tests/test_web_search_capability.py::test_web_search_service_exception` | | `api_key_env` overrides the raw key at setup | `src/lingtai/tools/web_search/__init__.py` | `tests/test_web_search_capability.py::test_web_search_setup_api_key_env_overrides_raw_key`, `::test_web_search_setup_resolves_api_key_env` | | MiniMax `api_host` / Zhipu mode are injected; Gemini omits `api_host` | `src/lingtai/tools/web_search/__init__.py` | `tests/test_web_search_capability.py::test_web_search_setup_passes_api_host_for_minimax`, `::test_web_search_setup_passes_zhipu_mode_without_api_host`, `::test_web_search_setup_omits_api_host_for_gemini` | @@ -98,18 +110,19 @@ There are no subprocess/shell/PTY/binary-spawn assumptions in this tool. | Unsupported providers fall back to duckduckgo (never raise) | fallback path in `tests/test_web_search_capability.py` | Configure an unknown provider, confirm `capability_fallback` log + duckduckgo | Setup crashes; agent loses search entirely | | Query delegation + result formatting | `tests/test_web_search_capability.py::test_web_search_manager_uses_search_service` | Run a query, confirm `**title**` formatting | Malformed/empty results surfaced to the model | | Errors are structured, never raised to the agent loop | `tests/test_web_search_capability.py::test_web_search_service_exception` | Point at an unreachable endpoint, confirm `status: error` | Tool-call crash instead of a recoverable error | +| Manual bundle is complete in wheel and sdist | `tests/test_tools_package_data.py` | Inspect installed `capabilities/web_search/` | Manual action degrades or routed assets/scripts disappear | | Lazy import preserves the lingtai.tools → lingtai DAG rule | import-time absence of `lingtai.services` at module load | `grep` for top-level `lingtai.services` imports (none) | Import cycle / layering violation | Run before merging web_search changes: ```bash -python -m pytest tests/test_web_search_capability.py -q +python -m pytest tests/test_web_search_capability.py tests/test_intrinsic_manual_actions.py -q ``` ## Schema and glossary ownership - **Canonical identifiers:** function names, JSON property names, action/enum - values, required fields, defaults, and bounds are canonical English literals. + values, mode-specific required fields, defaults, and bounds are canonical English literals. The schema (`get_schema()`) and description (`get_description()`) are language-independent; the optional `lang` argument is accepted for source compatibility but ignored. diff --git a/src/lingtai/tools/web_search/__init__.py b/src/lingtai/tools/web_search/__init__.py index 442104f83..ec01d924d 100644 --- a/src/lingtai/tools/web_search/__init__.py +++ b/src/lingtai/tools/web_search/__init__.py @@ -11,6 +11,7 @@ from typing import TYPE_CHECKING, Any +from .._manual import load_installed_manual if TYPE_CHECKING: from lingtai.kernel.base_agent import BaseAgent @@ -23,16 +24,17 @@ } def get_description(lang: str = "en") -> str: - return 'Search the web for current information. Use for real-time data, recent events, documentation, or anything beyond your training knowledge. Returns ranked search results with titles, URLs, and snippets. Before using this tool, read the `web-browsing` skill — it covers fetching specific URLs, PDF download, JS-rendered pages, scraping with stealth, and fallback APIs; no exceptions.' + return "Search the web for current information. Use for real-time data, recent events, documentation, or anything beyond your training knowledge. Returns ranked search results with titles, URLs, and snippets. Call web_search(action='manual') to return the installed web-search-manual bundle. Before ordinary searches, read that manual — it covers fetching specific URLs, PDF download, JS-rendered pages, scraping with stealth, and fallback APIs; no exceptions." def get_schema(lang: str = "en") -> dict: return { "type": "object", "properties": { - "query": {"type": "string", "description": 'Search query'}, + "action": {"type": "string", "enum": ["manual"], "description": "Use action='manual' to return the installed web-search-manual bundle without performing a search."}, + "query": {"type": "string", "description": "Search query. Required for ordinary searches; omit for action='manual'."}, }, - "required": ["query"], + "required": [], } @@ -49,6 +51,8 @@ def __init__( self._search_service = search_service def handle(self, args: dict) -> dict: + if args.get("action") == "manual": + return load_installed_manual(self._agent, "web_search") query = args.get("query") if not query: return {"status": "error", "message": "Missing required parameter: query"} diff --git a/src/lingtai/tools/web_search/manual/SKILL.md b/src/lingtai/tools/web_search/manual/SKILL.md new file mode 100644 index 000000000..1b4beb89b --- /dev/null +++ b/src/lingtai/tools/web_search/manual/SKILL.md @@ -0,0 +1,128 @@ +--- +name: web-search-manual +description: > + Fetch, extract, scrape, or search web content. First try + `python3 /scripts/extract_page.py `: it auto-tiers across + PDFs, metadata APIs, trafilatura, BeautifulSoup, Playwright, Jina, and AI + search. Read this router when the script fails, you need site/tier routing, + or you are composing a multi-step web/research pipeline. +version: 3.1.0 +last_changed_at: "2026-07-16T18:50:00-07:00" +related_files: + - src/lingtai/tools/web_search/__init__.py + - src/lingtai/tools/web_search/ANATOMY.md + - src/lingtai/tools/web_search/CONTRACT.md + - src/lingtai/tools/web_search/manual/scripts/extract_page.py +maintenance: | + Kernel-packaged manual synchronized from the TUI web-browsing bundle. Keep + its scripts, assets, and routed references together; use bundle-relative + paths rather than TUI installation paths. If information is stale, use the + lingtai-issue-report workflow and never include secrets or private paths. +--- + +# web-browsing — Router + +This is the kernel-packaged `web_search` manual. Its root skill name is +`web-search-manual` so it can coexist with a separately installed TUI +`web-browsing` utility; all `` examples resolve from this bundle. + +> **Browse the web with progressive disclosure.** Start with the bundled +> auto-tier extractor. Drill into nested references only when the script fails, +> you need a custom extraction shape, or you are changing the skill itself. + +## Try this first + +For most fetches the bundled script is the right answer. It auto-tiers, falls +back on failure, and handles PDFs / APIs / static articles / dynamic pages +without you writing custom code: + +```bash +# Auto-tier: extractor picks the cheapest viable strategy +python3 /scripts/extract_page.py "https://example.com/article" + +# Fallback chain: try, escalate on each failure +python3 /scripts/extract_page.py "https://example.com" --fallback + +# Force a specific tier when you know better than the auto-router +python3 /scripts/extract_page.py "https://example.com" --tier 3 + +# Search mode (no URL, just a query) +python3 /scripts/extract_page.py "quantum computing" --search + +# Save as JSON +python3 /scripts/extract_page.py "https://example.com" --json out.json +``` + +Read further only if that returns nothing useful, you need a custom extraction +shape, or you are composing a multi-step pipeline such as academic search → DOI +→ free PDF → text. + +## Nested reference catalog + +`web-browsing` owns these nested references. They are parent-owned drill-down +files, not standalone top-level skills. Existing deep-dive `.md` files under +`reference/` remain available and are indexed from the nested references. + +```yaml +- name: web-browsing-tier-quick-refs + location: reference/tier-quick-refs/SKILL.md + description: | + Manual commands for each extraction tier: PDF direct download, metadata + APIs, Trafilatura, BeautifulSoup, Playwright stealth, Jina/Firecrawl, and + AI-native search. +- name: web-browsing-routing-and-sites + location: reference/routing-and-sites/SKILL.md + description: | + Auto-tier decision tree, per-site recommendations, known limitations and + gotchas, and real-time data endpoints. +- name: web-browsing-maintenance-bundles + location: reference/maintenance-bundles/SKILL.md + description: | + Maintenance protocol, semantic sweeps, dirty-first testing, bundled JSON + JSON asset files, deep-dive reference files, and explicit decision flowchart. +``` + +## Quick decision tree + +```text +URL arrives → run scripts/extract_page.py first + ├─ PDF? → Tier 0; details in tier quick refs + ├─ Known API? → Tier 1; details in tier quick refs + ├─ Static HTML article? → Tier 1.5 Trafilatura + ├─ Needs structured scraping? → Tier 2 BeautifulSoup + ├─ JS-rendered/protected? → Tier 3 Playwright stealth + ├─ Still failing? → Tier 4 Jina Reader / Firecrawl + └─ Need to discover content? → Tier 5 search / AI-native search +``` + +## Router table + +| Need / keywords | Read | +|---|---| +| Specific tier commands; manual PDF/API/Trafilatura/BeautifulSoup/Playwright/Jina/Firecrawl/search examples | `reference/tier-quick-refs/SKILL.md` | +| Auto-tier misroutes a page; choose a tier; per-site recommendations; limitations; real-time data endpoints | `reference/routing-and-sites/SKILL.md` | +| Editing or validating this skill; bundled JSON asset files; deep-dive reference index; semantic sweep and dirty-first testing | `reference/maintenance-bundles/SKILL.md` | + +## Tier overview + +| Tier | Method | Speed | Tools | Reference | +|------|--------|-------|-------|-----------| +| **0** | PDF Direct Download | ~1s | `curl` + `fitz` | [tier-0-pdf.md](reference/tier-0-pdf.md) | +| **1** | API Metadata Queries | ~0.5s | `requests` | [tier-1-apis.md](reference/tier-1-apis.md) | +| **1.5** | Trafilatura Fast Extraction | ~2s | `trafilatura` | [tier-1-5-trafilatura.md](reference/tier-1-5-trafilatura.md) | +| **2** | BeautifulSoup Structured Extraction | ~5s | `requests` + `BS4` | [tier-2-beautifulsoup.md](reference/tier-2-beautifulsoup.md) | +| **3** | Playwright Stealth | ~15s | `playwright` + stealth | [tier-3-playwright.md](reference/tier-3-playwright.md) | +| **4** | API Fallback | ~3s | Jina / Firecrawl | [tier-4-jina-firecrawl.md](reference/tier-4-jina-firecrawl.md) | +| **5** | AI-Native Search | ~5s | `ddgs` / Tavily / Exa | [tier-5-ai-search.md](reference/tier-5-ai-search.md) | + +## Core rules to keep resident + +- Use the bundled `extract_page.py` before hand-writing scrapers unless you have + a clear reason not to. +- Escalate tiers only on failure or when the site class demands it; each tier is + heavier than the previous. +- Prefer source-specific APIs for structured/current data when available. +- Do not use web browsing for content already in the conversation or when an MCP + or first-class tool covers the source more cleanly. +- When changing this skill, run the maintenance reference's semantic sweep so the + script, JSON asset files, and docs stay aligned. diff --git a/src/lingtai/tools/web_search/manual/assets/api-endpoints.json b/src/lingtai/tools/web_search/manual/assets/api-endpoints.json new file mode 100644 index 000000000..579b1f797 --- /dev/null +++ b/src/lingtai/tools/web_search/manual/assets/api-endpoints.json @@ -0,0 +1,359 @@ +{ + "_comment": "API 端点汇总 v3.0 — 覆盖学术、搜索、实时数据、提取服务", + "academic": { + "arXiv": { + "base": "https://export.arxiv.org/api/query", + "example": "https://export.arxiv.org/api/query?id_list=1706.03762", + "params": { + "id_list": "arXiv ID 列表,逗号分隔", + "search_query": "搜索词 (ti:标题, au:作者, abs:摘要)", + "max_results": "最大结果数(默认10)", + "start": "起始索引", + "sortBy": "relevance | lastUpdatedDate | submittedDate", + "sortOrder": "ascending | descending" + }, + "returns": "Atom XML,含标题/摘要/作者/分类/PDF链接", + "rate_limit": "无明确限制,建议 <1 req/s", + "tier": 1 + }, + "OpenAlex": { + "base": "https://api.openalex.org", + "works_by_doi": "https://api.openalex.org/works/https://doi.org/{DOI}", + "works_search": "https://api.openalex.org/works?search={关键词}&per_page=25", + "authors_search": "https://api.openalex.org/authors?search={姓名}", + "params": { + "search": "全文搜索", + "filter": "过滤器(author.id, year, topic, is_oa, etc.)", + "sort": "排序(cited_by_count:desc, publication_date:desc)", + "per_page": "每页数量(最大400)", + "select": "返回字段白名单" + }, + "returns": "JSON,含标题/作者/摘要/引用数/主题/PDF链接/期刊", + "rate_limit": "无限制(建议加 mailto 参数获取更快响应)", + "tier": 1, + "notes": "最强学术API,完全免费,数据质量高" + }, + "CrossRef": { + "base": "https://api.crossref.org", + "works": "https://api.crossref.org/works/{DOI}", + "works_search": "https://api.crossref.org/works?query={关键词}&rows=10", + "params": { + "query": "搜索词", + "rows": "结果数(最大100)", + "mailto": "联系邮箱(礼貌性,建议填写获得更快速率)", + "filter": "过滤器(from-pub-date, type, member, etc.)", + "select": "返回字段白名单" + }, + "returns": "JSON,含标题/作者/期刊/DOI/License/发布日期", + "rate_limit": "无限制,但建议附上 mailto", + "tier": 1, + "notes": "元数据权威来源,无全文" + }, + "SemanticScholar": { + "base": "https://api.semanticscholar.org/graph/v1", + "paper_by_doi": "https://api.semanticscholar.org/graph/v1/paper/{DOI}?fields=title,authors,year,abstract,citationCount,openAccessPdf", + "paper_search": "https://api.semanticscholar.org/graph/v1/paper/search?query={关键词}&fields=title,authors,year,abstract,citationCount,openAccessPdf&limit=10", + "params": { + "fields": "逗号分隔字段列表(title,authors,year,abstract,citationCount,openAccessPdf,references,citations)" + }, + "returns": "JSON,含标题/作者/摘要/引用数/开放获取PDF链接", + "rate_limit": "无需Key:100 req/5min;有Key:10000/day", + "tier": 1, + "notes": "申请Key:https://www.semanticscholar.org/product/api#api-key-form" + }, + "PubMed": { + "base": "https://eutils.ncbi.nlm.nih.gov/entrez/eutils", + "search": "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=pubmed&term={query}&retmax=10", + "fetch": "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=pubmed&id={PMIDs}&rettype=abstract&retmode=json", + "params": { + "db": "pubmed | pmc", + "term": "搜索词", + "retmax": "最大结果数", + "rettype": "abstract | full | uilist", + "retmode": "json | xml" + }, + "returns": "JSON/XML,含标题/摘要/作者/期刊", + "rate_limit": "3 req/s(无key),10 req/s(有key)", + "tier": 1, + "notes": "网页版403,必须用E-utilities API" + }, + "NASA_ADS": { + "base": "https://ui.adsabs.harvard.edu/abs", + "export": "https://ui.adsabs.harvard.edu/abs/{arxiv_id}/exportcitation", + "bibtex": "https://ui.adsabs.harvard.edu/abs/{arxiv_id}/bibtex", + "returns": "BibTeX / ASCII / JSON", + "rate_limit": "建议加延迟", + "tier": 1, + "notes": "天文物理必备,数据权威" + }, + "CORE": { + "base": "https://api.core.ac.uk/v3", + "search": "https://api.core.ac.uk/v3/search/works?q={query}&limit=10", + "work_by_id": "https://api.core.ac.uk/v3/works/{id}", + "params": { + "q": "搜索词", + "limit": "结果数", + "offset": "起始索引" + }, + "returns": "JSON,含标题/作者/摘要/fullText/downloadUrl", + "rate_limit": "1000/day free (需API key)", + "tier": 1, + "auth": "Bearer token in Authorization header", + "notes": "全球最大开放获取论文集合,30M+ 篇,部分有全文", + "key_register": "https://core.ac.uk/services/api" + }, + "Unpaywall": { + "base": "https://api.unpaywall.org/v2", + "by_doi": "https://api.unpaywall.org/v2/{DOI}?email=lingtai@users.noreply.github.com", + "params": { + "email": "联系邮箱(必填)" + }, + "returns": "JSON: is_oa, best_oa_location, oa_locations (url, url_for_pdf, host_type, version)", + "rate_limit": "100k/day (with email)", + "tier": 1, + "notes": "为任何论文找免费PDF,补充CrossRef" + }, + "EuropePMC": { + "base": "https://www.ebi.ac.uk/europepmc/webservices/rest", + "search": "https://www.ebi.ac.uk/europepmc/webservices/rest/search?query={query}&format=json&pageSize=25", + "by_doi": "https://www.ebi.ac.uk/europepmc/webservices/rest/search?query=DOI:{doi}&format=json", + "fulltext": "https://www.ebi.ac.uk/europepmc/webservices/rest/{pmcid}/fullTextXML", + "params": { + "query": "搜索词,支持 DOI: PMCID: AUTHOR: 等前缀", + "format": "json | xml", + "pageSize": "结果数(最大1000)" + }, + "returns": "JSON,含标题/作者/摘要/PMCID/DOI/引用列表", + "rate_limit": "合理使用", + "tier": 1, + "notes": "覆盖PubMed + 欧洲生物医学文献,PMC全文可获取" + }, + "DBLP": { + "base": "https://dblp.org/search", + "publ_search": "https://dblp.org/search/publ/api?q={query}&format=json&h=10", + "author_search": "https://dblp.org/search/author/api?q={name}&format=json", + "venue_search": "https://dblp.org/search/venue/api?q={venue}&format=json", + "params": { + "q": "搜索词", + "format": "json | xml", + "h": "结果数(最大1000)", + "f": "偏移量" + }, + "returns": "JSON,含title/authors/venue/year/DOI/URL/ee", + "rate_limit": "合理使用", + "tier": 1, + "notes": "计算机科学首选,会议论文最准确" + }, + "PapersWithCode": { + "base": "https://paperswithcode.com/api/v1", + "search": "https://paperswithcode.com/api/v1/search/?q={query}", + "paper": "https://paperswithcode.com/api/v1/papers/{paper_id}", + "datasets": "https://paperswithcode.com/api/v1/datasets/", + "tasks": "https://paperswithcode.com/api/v1/tasks/", + "methods": "https://paperswithcode.com/api/v1/methods/", + "params": { + "q": "搜索词", + "page": "页码", + "items_per_page": "每页数量" + }, + "returns": "JSON,含paper title/code_repo/benchmark_results", + "rate_limit": "合理使用", + "tier": 1, + "notes": "ML/AI论文+代码仓库+基准排行榜" + }, + "DOAJ": { + "base": "https://doaj.org/api/search/articles", + "search": "https://doaj.org/api/search/articles/{query}?page=1&pageSize=10", + "returns": "JSON,开放获取期刊文章", + "rate_limit": "合理使用", + "tier": 1, + "notes": "Directory of Open Access Journals" + }, + "Zenodo": { + "base": "https://zenodo.org/api/records", + "search": "https://zenodo.org/api/records?q={query}&size=10", + "by_id": "https://zenodo.org/api/records/{id}", + "returns": "JSON,研究数据/软件/报告", + "rate_limit": "合理使用", + "tier": 1, + "notes": "CERN运营的研究数据仓库" + } + }, + "search": { + "DuckDuckGo": { + "method": "Python library (pip install duckduckgo-search)", + "usage": "from duckduckgo_search import DDGS; DDGS().text('query', max_results=10)", + "also": "ddgs.news(), ddgs.images(), ddgs.videos()", + "rate_limit": "无API key,但有限速", + "tier": 5, + "notes": "免费无key搜索首选", + "free": true + }, + "Tavily": { + "base": "https://api.tavily.com/search", + "method": "POST", + "params": { + "api_key": "API key", + "query": "搜索词", + "search_depth": "basic | advanced", + "include_answer": "true/false — AI生成答案", + "include_raw_content": "true/false — 完整页面内容", + "max_results": "结果数量" + }, + "returns": "JSON: answer + results (title, url, content, raw_content, score)", + "rate_limit": "1000 req/month free", + "tier": 5, + "notes": "AI agent搜索首选,搜索+提取+答案一体化", + "free_tier": true + }, + "Exa": { + "base": "https://api.exa.ai/search", + "method": "POST", + "headers": {"x-api-key": "API key"}, + "params": { + "query": "搜索词", + "type": "neural | keyword | auto", + "numResults": "结果数量", + "contents": "{\"text\": {\"maxCharacters\": 3000}}", + "includeDomains": "限定域名列表", + "excludeDomains": "排除域名列表" + }, + "returns": "JSON: results (title, url, text, score)", + "rate_limit": "1000 req/month free", + "tier": 5, + "notes": "语义搜索,按含义而非关键词匹配", + "free_tier": true + }, + "Google_Custom_Search": { + "base": "https://www.googleapis.com/customsearch/v1", + "params": { + "key": "API key", + "cx": "搜索引擎 ID", + "q": "搜索词" + }, + "rate_limit": "100 queries/day free", + "tier": 5, + "notes": "需要配置: https://programmablesearchengine.google.com/" + }, + "Serper": { + "base": "https://google.serper.dev/search", + "method": "POST", + "headers": {"X-API-KEY": "API key"}, + "params": {"q": "搜索词", "gl": "国家代码", "hl": "语言"}, + "rate_limit": "2500 queries one-time free", + "tier": 5, + "notes": "Google质量搜索结果,简单API" + }, + "Brave": { + "base": "https://api.search.brave.com/res/v1/web/search", + "headers": {"X-Subscription-Token": "API key"}, + "params": {"q": "搜索词", "summary": "true — AI摘要"}, + "rate_limit": "2000 queries/month free", + "tier": 5, + "notes": "独立搜索索引,非Google/Bing" + }, + "SearXNG": { + "base": "http://localhost:8888/search", + "params": {"q": "搜索词", "format": "json", "categories": "general|news|science|files"}, + "rate_limit": "无限制(自托管)", + "tier": 5, + "notes": "自托管元搜索,聚合Google+Bing+DDG等", + "install": "docker run -p 8888:8080 searxng/searxng" + } + }, + "extraction": { + "JinaReader": { + "base": "https://r.jina.ai/{url}", + "method": "GET", + "headers": { + "Accept": "text/markdown | application/json", + "X-Return-Format": "markdown | html | text", + "X-With-Links": "true | false", + "X-With-Images": "true | false" + }, + "returns": "Markdown / HTML / 纯文本", + "rate_limit": "20 req/min free, unlimited with key", + "tier": 4, + "notes": "万能fallback,服务端渲染JS", + "free": true, + "batch": "https://r.jina.ai/{url1},{url2}" + }, + "Firecrawl": { + "sdk": "pip install firecrawl-py", + "endpoints": { + "scrape": "/scrape — 单页提取", + "crawl": "/crawl — 整站爬取", + "search": "/search — 搜索+提取一体化", + "map": "/map — 获取站点URL结构", + "extract": "/extract — LLM结构化提取" + }, + "rate_limit": "500 credits/month free", + "tier": 4, + "notes": "生产级爬取,反bot+代理处理" + } + }, + "realtime": { + "GoogleNewsRSS": { + "url": "https://news.google.com/rss/search?q={query}&hl=en&gl=US&ceid=US:en", + "returns": "RSS XML", + "rate_limit": "无", + "tier": 2, + "free": true + }, + "RedditJSON": { + "pattern": "在Reddit URL后加 .json", + "example": "https://www.reddit.com/r/programming/hot.json?limit=25", + "headers": {"User-Agent": "必须有描述性UA"}, + "rate_limit": "~60 req/min", + "tier": 2, + "free": true + }, + "HackerNews": { + "base": "https://hacker-news.firebaseio.com/v0", + "top": "https://hacker-news.firebaseio.com/v0/topstories.json", + "item": "https://hacker-news.firebaseio.com/v0/item/{id}.json", + "rate_limit": "无", + "tier": 1, + "free": true + }, + "GitHub": { + "base": "https://api.github.com", + "search": "https://api.github.com/search/repositories?q={query}&sort=stars", + "rate_limit": "60/hr无token, 5000/hr有token", + "tier": 1, + "notes": "Header: Authorization: token {github_token}" + }, + "StackExchange": { + "base": "https://api.stackexchange.com/2.3", + "search": "https://api.stackexchange.com/2.3/search?intitle={query}&site=stackoverflow", + "rate_limit": "300/day无key, 10000/day有key", + "tier": 1, + "notes": "180+ 站点(stackoverflow, serverfault, etc.)" + }, + "Wikipedia": { + "base": "https://en.wikipedia.org/api/rest_v1", + "summary": "https://en.wikipedia.org/api/rest_v1/page/summary/{title}", + "plain": "https://en.wikipedia.org/api/rest_v1/page/plain/{title}", + "rate_limit": "合理使用", + "tier": 1, + "free": true + }, + "WaybackMachine": { + "url": "https://archive.org/wayback/available?url={url}", + "returns": "JSON: archived_snapshots", + "tier": 1, + "free": true + }, + "yfinance": { + "method": "Python library (pip install yfinance)", + "usage": "yf.Ticker('AAPL').history(period='1mo') / .info / .news", + "tier": 1, + "free": true + }, + "OpenMeteo": { + "url": "https://api.open-meteo.com/v1/forecast?latitude={lat}&longitude={lon}¤t_weather=true", + "tier": 1, + "free": true + } + } +} diff --git a/src/lingtai/tools/web_search/manual/assets/css-selectors.json b/src/lingtai/tools/web_search/manual/assets/css-selectors.json new file mode 100644 index 000000000..765f6da8d --- /dev/null +++ b/src/lingtai/tools/web_search/manual/assets/css-selectors.json @@ -0,0 +1,113 @@ +{ + "_comment": "常见 CSS 选择器模式库 v3.0 — Tier 1.5+ 使用", + "article_body": [ + "article", + "article .c-article-body", + "article .article-body", + "article main", + "main.article-content", + "div[itemprop='articleBody']", + "div.post-content", + "div.entry-content", + "div.available-content", + "div.article-content", + "div[class*='article']", + "div[class*='post-body']", + "div[class*='story-body']", + "[data-component='text-block']", + "[data-testid='ArticleBody']", + "div.markdown-body" + ], + "title": [ + "h1", + "h1.title", + "h1.post-title", + "h1.article__title", + "h1.firstHeading", + "h1.headline", + "article h1", + "span.titleline > a", + "meta[property='og:title']", + "meta[name='citation_title']" + ], + "abstract": [ + "blockquote.abstract", + "div.abstract", + "section.abstract", + "meta[name='description']", + "meta[property='og:description']" + ], + "metadata": { + "authors": [ + "meta[name='citation_author']", + "meta[name='DC.creator']", + "div.authors a", + "span.author-name", + "a.author", + "a.author-name", + "[data-component='byline']", + "span.byline__name" + ], + "doi": [ + "meta[name='citation_doi']", + "meta[name='DC.identifier']", + "a[href*='doi.org']", + "span[class*='doi']" + ], + "published_date": [ + "meta[name='citation_publication_date']", + "meta[name='citation_date']", + "time[datetime]", + "time.post-date", + "span.date", + "time" + ], + "journal": [ + "meta[name='citation_journal_title']", + "meta[property='og:site_name']" + ] + }, + "pdf_link": [ + "a[href*='.pdf']", + "a[href*='/pdf/']", + "a[title*='PDF']", + "a.download-pdf", + "a[href*='download']", + "a[class*='pdf']" + ], + "jsonld": { + "selector": "script[type='application/ld+json']", + "types": ["Article", "NewsArticle", "ScholarlyArticle", "BlogPosting", "TechArticle", "WebPage"], + "notes": "JSON-LD structured data - most modern sites embed this. Returns: headline, author, datePublished, image, etc." + }, + "opengraph": { + "title": "meta[property='og:title']", + "description": "meta[property='og:description']", + "image": "meta[property='og:image']", + "url": "meta[property='og:url']", + "type": "meta[property='og:type']", + "site_name": "meta[property='og:site_name']", + "twitter_card": "meta[name='twitter:card']", + "twitter_title": "meta[name='twitter:title']", + "twitter_description": "meta[name='twitter:description']", + "notes": "OG + Twitter Card metadata - available on most pages without JS rendering" + }, + "navigation": [ + "nav", + "header", + "footer", + "aside", + "div.sidebar", + "div[class*='nav']", + "div[class*='menu']" + ], + "exclude_patterns": [ + "nav", "header", "footer", "aside", ".sidebar", + ".nav", ".menu", ".cookie", ".popup", ".modal", + ".ad", ".ads", ".advertisement", ".sponsor", + ".social-share", ".share-buttons", ".comments", + ".related-posts", ".recommended", ".newsletter", + "[class*='paywall']", "[class*='subscribe']", + "script", "style", "noscript", "svg" + ] +} diff --git a/src/lingtai/tools/web_search/manual/assets/extraction-pipeline.json b/src/lingtai/tools/web_search/manual/assets/extraction-pipeline.json new file mode 100644 index 000000000..7f8f67223 --- /dev/null +++ b/src/lingtai/tools/web_search/manual/assets/extraction-pipeline.json @@ -0,0 +1,132 @@ +{ + "_comment": "完整提取流水线配置 v3.0 — 7层渐进式提取策略", + "tiers": { + "tier0_pdf": { + "name": "PDF Direct Download", + "speed": "~1s", + "tools": ["curl", "fitz/pymupdf"], + "trigger": ["URL ends with .pdf", "Known PDF direct link"], + "weight": 1, + "fallback_to": "tier1" + }, + "tier1_api": { + "name": "API Metadata Queries", + "speed": "~0.5s", + "tools": ["requests"], + "trigger": ["arXiv ID", "DOI", "PMID", "PMC ID", "Known API endpoint"], + "apis": ["arXiv", "OpenAlex", "CrossRef", "Semantic Scholar", "PubMed E-utilities", + "CORE", "Unpaywall", "Europe PMC", "DBLP", "Papers With Code", + "DOAJ", "Zenodo", "NASA ADS", "Wikipedia REST"], + "weight": 2, + "fallback_to": "tier1_5" + }, + "tier1_5_trafilatura": { + "name": "Trafilatura Fast Extraction", + "speed": "~0.05s", + "tools": ["trafilatura (已安装)"], + "trigger": ["Static HTML article", "Blog post", "News article", "Documentation page"], + "weight": 3, + "fallback_to": "tier2", + "notes": "10-50x faster than BeautifulSoup, 200x faster than browser. Default for most non-API URLs." + }, + "tier2_bs": { + "name": "BeautifulSoup Structured Extraction", + "speed": "~0.15s", + "tools": ["requests", "beautifulsoup4", "lxml"], + "trigger": ["Need structured data (lists, tables)", "Site has known CSS selectors", + "Google Scholar", "Reddit", "GitHub HTML", "News lists"], + "weight": 5, + "fallback_to": "tier3" + }, + "tier3_playwright": { + "name": "Playwright Stealth", + "speed": "~3-5s", + "tools": ["playwright", "playwright-stealth (已安装)"], + "trigger": ["JS-rendered content", "Login-gated pages", "Anti-bot protection", + "403 from simple requests", "Dynamic content loading"], + "weight": 10, + "fallback_to": "tier4", + "critical_notes": [ + "Use domcontentloaded NOT networkidle for Nature/Springer", + "Block images/styles/fonts for speed", + "Add init scripts to override fingerprinting", + "Smart delay between requests (2s + jitter)" + ] + }, + "tier4_api_fallback": { + "name": "API Fallback (Jina Reader / Firecrawl)", + "speed": "~2-5s", + "tools": ["requests (Jina Reader)", "firecrawl-py (Firecrawl)"], + "trigger": ["All local methods failed", "403/429/CAPTCHA", "Heavy JS protection"], + "services": { + "JinaReader": { + "url": "https://r.jina.ai/{target_url}", + "free": true, + "rate_limit": "20 req/min", + "returns": "Markdown" + }, + "Firecrawl": { + "install": "pip install firecrawl-py", + "free_credits": 500/month, + "features": ["scrape", "crawl", "search", "extract"] + } + }, + "weight": 20, + "fallback_to": "tier5" + }, + "tier5_ai_search": { + "name": "AI-Native Search (Tavily / Exa / DuckDuckGo)", + "speed": "~3-5s", + "tools": ["requests", "duckduckgo-search"], + "trigger": ["Need to discover content (not extract known URL)", + "Search + extract in one call", + "Semantic/meaning-based search"], + "services": { + "DuckDuckGo": {"free": true, "no_key": true, "python_lib": "duckduckgo-search"}, + "Tavily": {"free": "1000/month", "features": "search+answer+content"}, + "Exa": {"free": "1000/month", "features": "neural search"} + }, + "weight": 30, + "fallback_to": null + } + }, + "auto_tier_rules": { + "pdf_pattern": {"match": [".pdf"], "tier": 0}, + "arxiv_abs": {"match": ["arxiv.org/abs"], "tier": 1}, + "arxiv_pdf": {"match": ["arxiv.org/pdf"], "tier": 0}, + "doi": {"match": ["doi.org/10."], "tier": 1}, + "openalex": {"match": ["openalex.org"], "tier": 1}, + "crossref": {"match": ["api.crossref.org"], "tier": 1}, + "pubmed": {"match": ["pubmed.ncbi"], "tier": 1}, + "semanticscholar": {"match": ["semanticscholar.org"], "tier": 1}, + "dblp": {"match": ["dblp.org/search"], "tier": 1}, + "paperswithcode": {"match": ["paperswithcode.com/api"], "tier": 1}, + "wikipedia": {"match": ["wikipedia.org/wiki"], "tier": 1}, + "hackernews": {"match": ["news.ycombinator.com"], "tier": 1}, + "medium": {"match": ["medium.com"], "tier": 1.5}, + "substack": {"match": ["substack.com"], "tier": 1.5}, + "bbc": {"match": ["bbc.com"], "tier": 1.5}, + "reuters": {"match": ["reuters.com"], "tier": 1.5}, + "blog": {"match": ["blog.", "/blog/"], "tier": 1.5}, + "scholar": {"match": ["scholar.google"], "tier": 2}, + "nature": {"match": ["nature.com"], "tier": 2}, + "reddit": {"match": ["reddit.com"], "tier": 2}, + "springer": {"match": ["springer.com"], "tier": 2}, + "github_html": {"match": ["github.com"], "tier": 2}, + "twitter": {"match": ["twitter.com", "x.com"], "tier": 3}, + "linkedin": {"match": ["linkedin.com"], "tier": 3}, + "facebook": {"match": ["facebook.com"], "tier": 3}, + "default": {"tier": 1.5, "fallback_chain": [1.5, 2, 3, 4]} + }, + "fallback_chain": { + "description": "Default auto-escalation when a tier fails", + "steps": [ + "Try auto_tier → execute recommended tier", + "On failure → escalate to next tier", + "tier1.5 fails → try tier2 (BS)", + "tier2 fails → try tier3 (Playwright)", + "tier3 fails → try tier4 (Jina Reader)", + "tier4 fails → try tier5 (AI search to find alternative URLs)" + ] + } +} diff --git a/src/lingtai/tools/web_search/manual/assets/regex-patterns.json b/src/lingtai/tools/web_search/manual/assets/regex-patterns.json new file mode 100644 index 000000000..2cdbe7655 --- /dev/null +++ b/src/lingtai/tools/web_search/manual/assets/regex-patterns.json @@ -0,0 +1,130 @@ +{ + "_comment": "学术 ID 正则模板 v3.0 — URL 指纹触发,按需加载", + "academic_ids": { + "doi": { + "pattern": "10\\.\\d{4,}/[^\\s\"'<>]+", + "examples": ["10.1038/nature12373", "10.5194/egusphere-egu23-11253"], + "template": "https://doi.org/{id}", + "api_templates": { + "crossref": "https://api.crossref.org/works/{id}", + "openalex": "https://api.openalex.org/works/https://doi.org/{id}", + "unpaywall": "https://api.unpaywall.org/v2/{id}?email=lingtai@users.noreply.github.com", + "semantic_scholar": "https://api.semanticscholar.org/graph/v1/paper/{id}?fields=title,authors,year,abstract,citationCount,openAccessPdf", + "europe_pmc": "https://www.ebi.ac.uk/europepmc/webservices/rest/search?query=DOI:{id}&format=json" + } + }, + "arxiv": { + "pattern": "(?:arXiv:)?(\\d{4}\\.\\d{4,5}(?:v\\d+)?)", + "examples": ["1706.03762", "arXiv:2301.12345", "2401.00001v2"], + "html_template": "https://arxiv.org/abs/{id}", + "pdf_template": "https://arxiv.org/pdf/{id}.pdf", + "api_template": "https://export.arxiv.org/api/query?id_list={id}" + }, + "pmid": { + "pattern": "(?:PMID:|pmid:)?(\\d{6,8})", + "examples": ["38153247", "PMID:38153247"], + "api_template": "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=pubmed&id={id}&rettype=abstract&retmode=json", + "europe_pmc": "https://www.ebi.ac.uk/europepmc/webservices/rest/search?query=EXT_ID:{id}%20AND%20SRC:MED&format=json" + }, + "pmcid": { + "pattern": "(?:PMC|pmc:?)(\\d{6,7})", + "examples": ["PMC1234567"], + "api_template": "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=pmc&id=PMC{id}", + "fulltext_xml": "https://www.ebi.ac.uk/europepmc/webservices/rest/PMC{id}/fullTextXML" + }, + "isbn": { + "pattern": "(?:ISBN[:\\s]?)?(?:\\d[\\-\\s]?){9}[\\dXx]|(?:\\d[\\-\\s]?){12}\\d", + "examples": ["978-3-642-15581-0", "ISBN 9780134685991"], + "api_template": "https://openlibrary.org/api/books?bibkeys=ISBN:{id}&format=json&jscmd=data" + } + }, + "url_fingerprints": { + "arxiv_html": { + "match": ["arxiv.org/abs", "arxiv.org_ARXIV"], + "id_extract": "arxiv", + "tier": 1 + }, + "arxiv_pdf": { + "match": ["arxiv.org/pdf"], + "id_extract": "arxiv", + "tier": 0 + }, + "doi_org": { + "match": ["doi.org/10."], + "id_extract": "doi", + "tier": 1 + }, + "nature": { + "match": ["nature.com/articles"], + "tier": 2, + "notes": "og meta可用curl提取" + }, + "springer": { + "match": ["link.springer.com/article", "link.springer.com/content/pdf"], + "tier": 2, + "fallback_tier": 3, + "notes": "付费文章需Playwright+cookie" + }, + "scholar": { + "match": ["scholar.google.com"], + "tier": 2, + "notes": "curl+BS对搜索结果可用,详情页需登录态" + }, + "pubmed": { + "match": ["pubmed.ncbi.nlm.nih.gov"], + "tier": 1, + "notes": "网页版403,用E-utilities API" + }, + "medium": { + "match": ["medium.com"], + "tier": 1.5, + "notes": "trafilatura通常可提取全文" + }, + "substack": { + "match": ["substack.com"], + "tier": 1.5, + "notes": "trafilatura提取" + }, + "reddit": { + "match": ["reddit.com"], + "tier": 2, + "alt_api": "append .json to URL", + "notes": ".json API更可靠" + }, + "github": { + "match": ["github.com"], + "tier": 1, + "alt_api": "https://api.github.com", + "notes": "API优先" + }, + "wikipedia": { + "match": ["wikipedia.org/wiki"], + "tier": 1, + "notes": "REST API /page/summary/{title} 最简洁" + }, + "hackernews": { + "match": ["news.ycombinator.com"], + "tier": 1, + "alt_api": "https://hacker-news.firebaseio.com/v0", + "notes": "Firebase API" + } + }, + "pdf_url_patterns": { + "arxiv": { + "url_pattern": "arxiv\\.org/(?:abs|pdf)/(\\d{4}\\.\\d{4,5})", + "pdf_transform": "https://arxiv.org/pdf/{id}.pdf" + }, + "springer": { + "url_pattern": "link\\.springer\\.com/(?:article|content/pdf)/([^/?#]+)", + "pdf_transform": "https://link.springer.com/content/pdf/{id}.pdf" + }, + "pnas": { + "url_pattern": "pnas\\.org/doi/([^/?#]+)", + "pdf_transform": "https://www.pnas.org/doi/pdf/{id}" + }, + "nature": { + "url_pattern": "nature\\.com/articles/([^/?#]+)", + "pdf_notes": "Nature主文PDF不直接暴露,需从文章页内找PDF链接。用Unpaywall查OA版本。" + } + } +} diff --git a/src/lingtai/tools/web_search/manual/assets/search-providers.json b/src/lingtai/tools/web_search/manual/assets/search-providers.json new file mode 100644 index 000000000..e9a387c2c --- /dev/null +++ b/src/lingtai/tools/web_search/manual/assets/search-providers.json @@ -0,0 +1,141 @@ +{ + "_comment": "搜索引擎 API 配置 v3.0 — 搜索策略决策参考", + "strategy_decision_tree": { + "free_no_setup": "DuckDuckGo (duckduckgo-search Python库)", + "ai_agent_workflow": "Tavily (搜索+提取+AI答案 一体化)", + "semantic_meaning": "Exa (神经/语义搜索)", + "google_quality": "Serper 或 Google Custom Search", + "self_hosted_unlimited": "SearXNG (Docker自托管)", + "academic": "OpenAlex > Semantic Scholar > DBLP (CS)", + "news": "Google News RSS (免费) 或 DuckDuckGo news()" + }, + "providers": { + "DuckDuckGo": { + "type": "python_library", + "install": "pip install duckduckgo-search", + "usage": "from duckduckgo_search import DDGS; results = list(DDGS().text('query', max_results=10))", + "also_supports": ["text (网页搜索)", "news (新闻)", "images (图片)", "videos (视频)"], + "auth": "无需API key", + "rate_limit": "有限速但无明确数字", + "cost": "完全免费", + "best_for": "免费无key搜索首选", + "tier": 5 + }, + "Tavily": { + "type": "rest_api", + "endpoint": "https://api.tavily.com/search", + "method": "POST", + "auth": "api_key in JSON body", + "key_params": { + "query": "搜索词", + "search_depth": "basic (快) | advanced (深)", + "include_answer": "true → AI生成答案", + "include_raw_content": "true → 完整页面内容", + "max_results": "结果数 (默认10)" + }, + "returns": "answer (AI答案) + results[] (title, url, content, raw_content, score)", + "rate_limit": "1000 req/month free", + "cost": "免费1000次/月", + "best_for": "AI agent搜索,搜索+提取+答案一体化", + "tier": 5 + }, + "Exa": { + "type": "rest_api", + "endpoint": "https://api.exa.ai/search", + "method": "POST", + "auth": "x-api-key header", + "key_params": { + "query": "搜索词", + "type": "neural (语义) | keyword (关键词) | auto (自动)", + "numResults": "结果数", + "contents": "{\"text\": {\"maxCharacters\": 3000}} → 提取内容", + "includeDomains": "限定域名", + "excludeDomains": "排除域名" + }, + "returns": "results[] (title, url, text, score)", + "rate_limit": "1000 req/month free", + "cost": "免费1000次/月", + "best_for": "语义搜索,按含义匹配", + "tier": 5 + }, + "Google_Custom_Search": { + "type": "rest_api", + "endpoint": "https://www.googleapis.com/customsearch/v1", + "method": "GET", + "auth": "key param + cx param", + "key_params": { + "key": "API key", + "cx": "自定义搜索引擎 ID", + "q": "搜索词", + "num": "结果数 (最大10)" + }, + "setup": "https://programmablesearchengine.google.com/", + "rate_limit": "100 queries/day free", + "cost": "免费100次/天", + "best_for": "精确Google搜索结果", + "tier": 5 + }, + "Serper": { + "type": "rest_api", + "endpoint": "https://google.serper.dev/search", + "method": "POST", + "auth": "X-API-KEY header", + "key_params": { + "q": "搜索词", + "gl": "国家代码 (us, cn, etc.)", + "hl": "语言 (en, zh, etc.)", + "num": "结果数" + }, + "returns": "organic[] (title, link, snippet, position) + knowledgeGraph + featuredSnippet", + "rate_limit": "2500 queries one-time free", + "cost": "一次性2500次免费", + "best_for": "Google质量搜索结果,简单API", + "tier": 5 + }, + "Brave": { + "type": "rest_api", + "endpoint": "https://api.search.brave.com/res/v1/web/search", + "method": "GET", + "auth": "X-Subscription-Token header", + "key_params": { + "q": "搜索词", + "count": "结果数", + "summary": "true → AI生成摘要" + }, + "returns": "web.results[] (title, url, description)", + "rate_limit": "2000 queries/month free", + "cost": "免费2000次/月", + "best_for": "独立搜索索引(非Google/Bing)", + "tier": 5 + }, + "SearXNG": { + "type": "self_hosted", + "endpoint": "http://localhost:8888/search?q={query}&format=json", + "install": "docker run -d -p 8888:8080 searxng/searxng", + "key_params": { + "q": "搜索词", + "format": "json (必须)", + "categories": "general | news | science | files | images", + "engines": "google, bing, duckduckgo, etc.", + "pageno": "页码" + }, + "returns": "results[] (title, url, content, engine)", + "rate_limit": "无限制(自托管)", + "cost": "完全免费(需自托管)", + "best_for": "私有无限搜索,聚合多引擎", + "tier": 5 + } + }, + "query_tips": [ + "site: operator: 'machine learning' site:arxiv.org", + "filetype: operator: filetype:pdf", + "Exact phrase: \"deep learning\"", + "Exclude: -ads -sponsored", + "OR operator: \"deep learning\" OR \"neural networks\"", + "intitle: for title matches: intitle:transformer", + "Date range (Google): after:2024-01-01 before:2025-01-01", + "Academic: combine search with DOI extraction and API enrichment", + "For DuckDuckGo: same query syntax as Google mostly works", + "For Tavily: use search_depth=advanced for complex queries" + ] +} diff --git a/src/lingtai/tools/web_search/manual/assets/site-templates.json b/src/lingtai/tools/web_search/manual/assets/site-templates.json new file mode 100644 index 000000000..cb88f275c --- /dev/null +++ b/src/lingtai/tools/web_search/manual/assets/site-templates.json @@ -0,0 +1,209 @@ +{ + "_comment": "已知网站的 CSS 选择器模板 v3.0 — 域名识别后自动加载", + "arxiv.org": { + "name": "arXiv", + "tier": 1, + "selectors": { + "title": "h1.title", + "abstract": "blockquote.abstract", + "authors": "div.authors a", + "subjects": "td.subjects", + "pdf_link": "a[href*='/pdf/']", + "doi": "meta[name='citation_doi']" + }, + "pdf_path_template": "https://arxiv.org/pdf/{arxiv_id}.pdf", + "notes": "HTML页无直接PDF链接,需从ID推导。例:1706.03762 → https://arxiv.org/pdf/1706.03762.pdf" + }, + "scholar.google.com": { + "name": "Google Scholar", + "tier": 2, + "selectors": { + "paper_card": "div.gs_ri", + "title": "h3.gs_rt", + "title_link": "h3.gs_rt a", + "abstract": "div.gs_rs", + "meta": "div.gs_fl", + "pdf_link": "div.gs_or_gtxtr span.gs_ctg", + "citation_count": "div.gs_fl a:nth-child(1)" + }, + "notes": "搜索结果页curl+BS可用(IP需干净)。详情页需登录态。避免高频请求。" + }, + "nature.com": { + "name": "Nature", + "tier": 2, + "selectors": { + "title_og": "meta[property='og:title']", + "description_og": "meta[property='og:description']", + "doi": "meta[name='citation_doi']", + "authors": "meta[name='citation_author']", + "journal": "meta[name='citation_journal_title']", + "published_date": "meta[name='citation_publication_date']", + "article_body": "article .c-article-body" + }, + "playwright_notes": "Playwright用 domcontentloaded 而非 networkidle(会超时)。正文需JS渲染。", + "notes": "og meta可拿DOI和标题,无需JS。" + }, + "link.springer.com": { + "name": "Springer Link", + "tier": 3, + "selectors": { + "title": "meta[name='citation_title']", + "doi": "meta[name='citation_doi']", + "authors": "meta[name='citation_author']", + "journal": "meta[name='citation_journal_title']", + "published_date": "meta[name='citation_publication_date']" + }, + "pdf_path_template": "https://link.springer.com/content/pdf/{doi}.pdf", + "notes": "付费文章全程404。开放获取文章可用。curl持续404时需Playwright+cookie。" + }, + "pubmed.ncbi.nlm.nih.gov": { + "name": "PubMed", + "tier": 1, + "api": "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi", + "selectors": { + "pmid": "meta[name='pubmedid']", + "doi": "meta[name='citation_doi']", + "title": "meta[name='citation_title']", + "abstract": "#abstract > div", + "authors": "meta[name='citation_author']" + }, + "notes": "网页版403,需用 PubMed E-utilities API。" + }, + "iopscience.iop.org": { + "name": "IOP Science", + "tier": 2, + "selectors": { + "title": "meta[name='citation_title']", + "doi": "meta[name='citation_doi']", + "authors": "meta[name='citation_author']" + }, + "notes": "IOP出版的部分太阳物理期刊在此。" + }, + "ui.adsabs.harvard.edu": { + "name": "NASA ADS", + "tier": 1, + "selectors": { + "title": "meta[name='citation_title']", + "bibtex_export": "/abs/{id}/bibtex", + "abstract_export": "/abs/{id}/exportcitation" + }, + "notes": "天文必备。curl访问HTML困难,应用API端点如bibtex导出。" + }, + "reddit.com": { + "name": "Reddit", + "tier": 2, + "json_api": "在URL后加 .json,必须加 User-Agent header", + "old_reddit": "old.reddit.com 更易抓取", + "selectors": { + "post_card": "div.thing.linkflair", + "title": "a.title", + "score": "div.score", + "comments_count": "a.comments", + "post_body": "div.expando .usertext-body", + "comment": "div.comment" + }, + "rate_limit": "~60 req/min", + "notes": "JSON API更可靠:append .json to URL, add User-Agent header" + }, + "old.reddit.com": { + "name": "Reddit (Old)", + "tier": 2, + "selectors": { + "post_card": "div.thing", + "title": "a.title", + "score": "div.score.unvoted", + "comments": "a.bylink.comments" + }, + "notes": "Old Reddit HTML更简洁,更易抓取" + }, + "github.com": { + "name": "GitHub", + "tier": 1, + "api": "https://api.github.com", + "selectors": { + "readme": "article.markdown-body", + "file_content": "div.Box-body pre", + "repo_name": "strong a.h3", + "description": "p.mb-1", + "stars": "span.Counter", + "issues": "span.Counter" + }, + "api_search": "https://api.github.com/search/repositories?q={query}&sort=stars", + "rate_limit": "60/hr无token, 5000/hr有token", + "notes": "API优先;HTML抓取用old.github.com或直接API" + }, + "medium.com": { + "name": "Medium", + "tier": 1.5, + "selectors": { + "title": "h1", + "author": "a.author span", + "date": "time", + "article_body": "article" + }, + "notes": "trafilatura通常能提取全文(包括付费墙后)。?source=friends_link可尝试绕过付费墙。" + }, + "substack.com": { + "name": "Substack", + "tier": 1.5, + "selectors": { + "title": "h1.post-title", + "author": "a.author-name", + "date": "time.post-date", + "article_body": "div.available-content" + }, + "notes": "trafilatura通常可提取全文" + }, + "news.ycombinator.com": { + "name": "Hacker News", + "tier": 1, + "api": "https://hacker-news.firebaseio.com/v0", + "selectors": { + "story_row": "tr.athing", + "title": "span.titleline > a", + "score": "span.score", + "comments": "a[href^='item?id=']" + }, + "notes": "Firebase API更可靠:/topstories.json + /item/{id}.json" + }, + "bbc.com": { + "name": "BBC", + "tier": 1.5, + "selectors": { + "title": "h1", + "article_body": "article [data-component='text-block']", + "date": "time", + "author": "[data-component='byline']" + } + }, + "reuters.com": { + "name": "Reuters", + "tier": 1.5, + "selectors": { + "title": "h1", + "article_body": "article [data-testid='ArticleBody']", + "date": "time" + } + }, + "techcrunch.com": { + "name": "TechCrunch", + "tier": 1.5, + "selectors": { + "title": "h1.article__title", + "article_body": "div.article-content", + "author": "a.article__byline" + } + }, + "wikipedia.org": { + "name": "Wikipedia", + "tier": 1, + "api": "https://en.wikipedia.org/api/rest_v1/page/summary/{title}", + "selectors": { + "title": "h1.firstHeading", + "body_text": "div.mw-parser-output > p", + "infobox": "table.infobox", + "toc": "div.toc" + }, + "notes": "REST API /page/summary/{title} 最简洁高效" + } +} diff --git a/src/lingtai/tools/web_search/manual/reference/academic-pipeline.md b/src/lingtai/tools/web_search/manual/reference/academic-pipeline.md new file mode 100644 index 000000000..7b926e11e --- /dev/null +++ b/src/lingtai/tools/web_search/manual/reference/academic-pipeline.md @@ -0,0 +1,645 @@ +# Academic Search Pipeline + +> Part of the [web-browsing](../SKILL.md) skill. +> Complete pipeline for finding, enriching, and acquiring academic papers. + +# academic-search-pipeline + +> **Academic paper search, resolution, and acquisition — from a DOI string to a full PDF with metadata.** +> Part of the `web-browsing-manual` v3.0 skill family. For general web browsing, see the parent skill. + +--- + +## Decision Tree: What Do You Have? + +``` +Input arrives ────────┐ + │ + ┌─────────────────┼──────────────────────────────┐ + │ │ │ + DOI string arXiv ID Keywords only + (10.xxx/...) (2401.12345) │ + │ │ │ + ▼ ▼ ▼ + Unpaywall arXiv API What field? + (free PDF?) (metadata+PDF) │ + │ │ ┌───────────┼───────────┐ + ▼ ▼ │ │ │ + CrossRef Direct PDF CS/ML Biomedical General + (metadata) download │ │ │ + │ ▼ ▼ ▼ + ▼ DBLP PubMed/ OpenAlex → + OpenAlex arXiv EuropePMC CrossRef + (citations, Semantic CORE Semantic + concepts) Scholar Scholar + │ │ │ │ + └──────────────┬───────────┘ │ │ + ▼ ▼ ▼ + Papers With Code Zenodo DOAJ + (if ML+code) (datasets) (OA journals) +``` + +### Quick Routing Table + +| Input Type | First API | Fallback 1 | Fallback 2 | +|-----------|-----------|------------|------------| +| DOI (`10.xxx/...`) | Unpaywall → CrossRef | OpenAlex | Semantic Scholar | +| arXiv ID (`2401.12345`) | arXiv API | Semantic Scholar | OpenAlex | +| PMID (`12345678`) | PubMed E-utilities | Europe PMC | CrossRef (by DOI) | +| Keywords + CS | DBLP | arXiv | Semantic Scholar | +| Keywords + Biomedical | PubMed | Europe PMC | CORE | +| Keywords + ML/AI | Papers With Code | arXiv | Semantic Scholar | +| Keywords + General | OpenAlex | CrossRef | Semantic Scholar | +| Keywords + Dataset | Zenodo | DOAJ | OpenAlex | + +--- + +## PDF Acquisition Chain + +The goal: get a free PDF for any paper. Try in this order: + +### 1. Unpaywall (DOI → OA PDF) + +**When to use:** You have a DOI. First check for open access. +**Speed:** ~0.5s | **Free:** ✅ (100k/day with email) | **Key needed:** No (just email) + +```python +import requests + +def unpaywall_find_pdf(doi, email="lingtai@users.noreply.github.com"): + """Find free PDF for any paper via Unpaywall. + + Returns dict with pdf_url if found, None if no OA version exists. + + NOTE: Unpaywall requires a real-looking email address. Generic addresses + like test@example.com or research@example.com will be rejected with 422. + Pass your actual email, or at minimum something plausible. + """ + url = f"https://api.unpaywall.org/v2/{doi}?email={email}" + try: + r = requests.get(url, timeout=15) + r.raise_for_status() + data = r.json() + + if data.get("is_oa"): + best = data.get("best_oa_location", {}) + pdf_url = best.get("url_for_pdf") or best.get("url") + if pdf_url: + return { + "pdf_url": pdf_url, + "version": best.get("version"), # publishedVersion, acceptedVersion + "host_type": best.get("host_type"), # publisher, repository + "license": best.get("license"), + } + + # Check all OA locations + for loc in data.get("oa_locations", []): + if loc.get("url_for_pdf"): + return { + "pdf_url": loc["url_for_pdf"], + "version": loc.get("version"), + "host_type": loc.get("host_type"), + } + + return None # No OA version available + except Exception as e: + print(f"[Unpaywall error] {e}") + return None +``` + +**When NOT to use:** No DOI available. Fall through to other methods. + +### 2. arXiv (arXiv ID → PDF + Metadata) + +**When to use:** CS/Physics/Math papers, arXiv ID known or discoverable. +**Speed:** ~1s | **Free:** ✅ | **Key needed:** No + +```python +import xml.etree.ElementTree as ET + +def arxiv_search(query, max_results=10, sort_by="relevance"): + """Search arXiv for papers. + + sort_by: 'relevance', 'lastUpdatedDate', 'submittedDate' + """ + url = "https://export.arxiv.org/api/query" + params = { + "search_query": query, + "max_results": max_results, + "sortBy": sort_by, + "sortOrder": "descending" + } + try: + r = requests.get(url, params=params, timeout=30) + r.raise_for_status() + root = ET.fromstring(r.text) + ns = {"atom": "http://www.w3.org/2005/Atom", "arxiv": "http://arxiv.org/schemas/atom"} + + papers = [] + for entry in root.findall("atom:entry", ns): + pdf_url = None + for link in entry.findall("atom:link", ns): + if link.get("title") == "pdf": + pdf_url = link.get("href") + + papers.append({ + "title": entry.find("atom:title", ns).text.strip().replace("\n", " "), + "authors": [a.find("atom:name", ns).text for a in entry.findall("atom:author", ns)], + "abstract": entry.find("atom:summary", ns).text.strip(), + "pdf_url": pdf_url, + "arxiv_id": entry.find("atom:id", ns).text.split("/abs/")[-1], + "published": entry.find("atom:published", ns).text, + "updated": entry.find("atom:updated", ns).text, + "categories": [c.get("term") for c in entry.findall("atom:category", ns)], + }) + return papers + except Exception as e: + print(f"[arXiv error] {e}") + return [] + +def arxiv_by_id(arxiv_id): + """Get a specific arXiv paper by ID (e.g., '2401.12345' or '2401.12345v2').""" + url = "https://export.arxiv.org/api/query" + params = {"id_list": arxiv_id, "max_results": 1} + try: + r = requests.get(url, params=params, timeout=30) + r.raise_for_status() + root = ET.fromstring(r.text) + ns = {"atom": "http://www.w3.org/2005/Atom", "arxiv": "http://arxiv.org/schemas/atom"} + papers = [] + for entry in root.findall("atom:entry", ns): + pdf_url = None + for link in entry.findall("atom:link", ns): + if link.get("title") == "pdf": + pdf_url = link.get("href") + papers.append({ + "title": entry.find("atom:title", ns).text.strip().replace("\n", " "), + "authors": [a.find("atom:name", ns).text for a in entry.findall("atom:author", ns)], + "abstract": entry.find("atom:summary", ns).text.strip(), + "pdf_url": pdf_url, + "arxiv_id": entry.find("atom:id", ns).text.split("/abs/")[-1], + "published": entry.find("atom:published", ns).text, + "updated": entry.find("atom:updated", ns).text, + "categories": [c.get("term") for c in entry.findall("atom:category", ns)], + }) + return papers + except Exception as e: + print(f"[arXiv error] {e}") + return [] +``` + +**Direct PDF download:** `https://arxiv.org/pdf/{ID}.pdf` + +### 3. CORE (Open Access Full Text) + +**When to use:** Need full text of open access papers. 30M+ OA articles. +**Speed:** ~1s | **Free:** ✅ (1000/day) | **Key needed:** Recommended (free at core.ac.uk) + +```python +def core_search(query, api_key=None, limit=10): + """Search CORE for open access papers with full text. + + CORE is the world's largest collection of OA research papers. + Many entries include fullText directly in the API response. + """ + url = "https://api.core.ac.uk/v3/search/works" + headers = {"Authorization": f"Bearer {api_key}"} if api_key else {} + params = {"q": query, "limit": limit} + try: + r = requests.get(url, headers=headers, params=params, timeout=30) + r.raise_for_status() + results = r.json().get("results", []) + return [{ + "title": p.get("title"), + "authors": p.get("authors", []), + "abstract": p.get("abstract"), + "fullText": p.get("fullText"), # Often available! + "downloadUrl": p.get("downloadUrl"), + "year": p.get("yearPublished"), + "doi": p.get("doi"), + "journals": p.get("journals", []), + } for p in results] + except Exception as e: + print(f"[CORE error] {e}") + return [] +``` + +### 4. Europe PMC (Biomedical + PMC Full Text) + +**When to use:** Biomedical/life sciences papers. PMC full text XML available. +**Speed:** ~1s | **Free:** ✅ | **Key needed:** No + +```python +def europe_pmc_search(query, page_size=25): + """Search Europe PMC for biomedical literature.""" + url = "https://www.ebi.ac.uk/europepmc/webservices/rest/search" + params = {"query": query, "format": "json", "pageSize": page_size} + try: + r = requests.get(url, params=params, timeout=30) + r.raise_for_status() + data = r.json() + return [{ + "title": r.get("title"), + "authors": r.get("authorString"), + "doi": r.get("doi"), + "pmid": r.get("pmid"), + "pmcid": r.get("pmcid"), + "journal": r.get("journalTitle"), + "year": r.get("pubYear"), + "isOpenAccess": r.get("isOpenAccess") == "Y", + } for r in data.get("resultList", {}).get("result", [])] + except Exception as e: + print(f"[Europe PMC error] {e}") + return [] + +def europe_pmc_fulltext(pmcid): + """Get full text XML for a PMC article.""" + url = f"https://www.ebi.ac.uk/europepmc/webservices/rest/{pmcid}/fullTextXML" + try: + r = requests.get(url, timeout=30) + if r.status_code == 200: + return r.text # XML full text + return None + except Exception: + return None +``` + +--- + +## DOI Resolution Chain + +Given a DOI, extract metadata in order of richness: + +### 1. CrossRef (DOI → Metadata + BibTeX) + +**When to use:** First stop for any DOI. Most comprehensive metadata. +**Speed:** ~0.5s | **Free:** ✅ | **Key needed:** No (polite to add mailto) + +```python +def crossref_metadata(doi): + """Get rich metadata for a DOI from CrossRef. + + Returns: title, authors, journal, year, abstract, references count, type. + """ + url = f"https://api.crossref.org/works/{doi}" + headers = {"User-Agent": "LingTai/3.0 (mailto:lingtai@users.noreply.github.com)"} + try: + r = requests.get(url, headers=headers, timeout=15) + if r.status_code == 404: + return None # DOI not found + r.raise_for_status() + m = r.json()["message"] + return { + "doi": doi, + "title": m.get("title", [""])[0], + "authors": [f"{a.get('given', '')} {a.get('family', '')}".strip() + for a in m.get("author", [])], + "journal": m.get("container-title", [""])[0], + "year": (m.get("published-print") or m.get("published-online") or + {}).get("date-parts", [[None]])[0][0], + "type": m.get("type"), + "abstract": m.get("abstract"), + "references_count": len(m.get("reference", [])), + "cited_by_count": m.get("is-referenced-by-count"), + "license": [l.get("URL") for l in m.get("license", [])], + } + except Exception as e: + print(f"[CrossRef error] {e}") + return None + +def crossref_bibtex(doi): + """Get BibTeX citation for a DOI via CrossRef content negotiation.""" + url = f"https://api.crossref.org/works/{doi}/transform/application/x-bibtex" + headers = {"Accept": "application/x-bibtex"} + try: + r = requests.get(url, headers=headers, timeout=15) + if r.status_code == 200: + return r.text # Raw BibTeX string + return None + except Exception: + return None +``` + +### 2. OpenAlex (DOI → Citations + Concepts + OA Status) + +**When to use:** Need citation counts, research concepts/topics, OA URL. +**Speed:** ~0.5s | **Free:** ✅ | **Key needed:** No + +```python +def openalex_work(doi): + """Get OpenAlex data for a DOI — citations, concepts, OA status.""" + url = f"https://api.openalex.org/works/https://doi.org/{doi}" + try: + r = requests.get(url, timeout=15) + if r.status_code == 404: + return None + r.raise_for_status() + w = r.json() + return { + "title": w.get("title"), + "doi": doi, + "authors": [a["author"]["display_name"] for a in w.get("authorships", [])], + "cited_by_count": w.get("cited_by_count"), + "concepts": [{"name": c["display_name"], "score": c["score"]} + for c in w.get("concepts", [])[:5]], + "open_access_url": (w.get("open_access") or {}).get("oa_url"), + "type": w.get("type"), + "publication_year": w.get("publication_year"), + "host_venue": (w.get("host_venue") or {}).get("display_name"), + "referenced_works_count": len(w.get("referenced_works", [])), + } + except Exception as e: + print(f"[OpenAlex error] {e}") + return None +``` + +### 3. Semantic Scholar (DOI → AI Summary + Citation Graph) + +**When to use:** AI/ML papers, need TLDR summary or citation graph. +**Speed:** ~1s | **Free:** ✅ (100/5min without key) | **Key needed:** Recommended + +```python +def semantic_scholar_paper(doi): + """Get Semantic Scholar data — includes AI-generated TLDR summary.""" + url = f"https://api.semanticscholar.org/graph/v1/paper/DOI:{doi}" + params = { + "fields": "title,authors,abstract,citationCount,referenceCount," + "year,openAccessPdf,tldr,venue,publicationTypes" + } + try: + r = requests.get(url, params=params, timeout=15) + if r.status_code == 404: + return None + r.raise_for_status() + p = r.json() + return { + "title": p.get("title"), + "doi": doi, + "authors": [a.get("name") for a in p.get("authors", [])], + "abstract": p.get("abstract"), + "citations": p.get("citationCount"), + "references": p.get("referenceCount"), + "year": p.get("year"), + "venue": p.get("venue"), + "pdf": (p.get("openAccessPdf") or {}).get("url"), + "tldr": (p.get("tldr") or {}).get("text"), # AI-generated summary! + "publication_types": p.get("publicationTypes"), + } + except Exception as e: + print(f"[Semantic Scholar error] {e}") + return None +``` + +### 4. DBLP (CS Conference Papers) + +**When to use:** Computer science conference papers specifically. +**Speed:** ~0.5s | **Free:** ✅ | **Key needed:** No + +```python +def dblp_search(query, max_results=10): + """Search DBLP for CS publications.""" + url = "https://dblp.org/search/publ/api" + params = {"q": query, "format": "json", "h": max_results} + try: + r = requests.get(url, params=params, timeout=15) + r.raise_for_status() + hits = r.json().get("result", {}).get("hits", {}).get("hit", []) + return [{ + "title": h.get("info", {}).get("title"), + "authors": h.get("info", {}).get("authors", {}).get("author", []), + "venue": h.get("info", {}).get("venue"), + "year": h.get("info", {}).get("year"), + "doi": h.get("info", {}).get("doi"), + "url": h.get("info", {}).get("url"), + "type": h.get("info", {}).get("type"), + } for h in hits] + except Exception as e: + print(f"[DBLP error] {e}") + return [] +``` + +### 5. Papers With Code (ML/AI + Code + Benchmarks) + +**When to use:** ML/AI papers that may have code implementations. +**Speed:** ~1s | **Free:** ✅ | **Key needed:** No + +```python +def pwc_search(query, limit=10): + """Search Papers With Code for ML papers with code.""" + url = "https://paperswithcode.com/api/v1/search/" + params = {"q": query, "page": 1, "items_per_page": limit} + try: + r = requests.get(url, params=params, timeout=15) + r.raise_for_status() + return r.json().get("results", []) + except Exception as e: + print(f"[PWC error] {e}") + return [] +``` + +--- + +## BibTeX / Citation Export + +```python +def get_bibtex(doi): + """Get BibTeX for a DOI via CrossRef content negotiation.""" + return crossref_bibtex(doi) + +def get_ris(doi): + """Get RIS citation for a DOI.""" + url = f"https://api.crossref.org/works/{doi}/transform/application/x-research-info-systems" + try: + r = requests.get(url, timeout=15) + return r.text if r.status_code == 200 else None + except Exception: + return None +``` + +--- + +## End-to-End Pipeline + +```python +import re + +def academic_pipeline(query_or_id): + """Complete pipeline: identify input → resolve → enrich → get PDF. + + Accepts: DOI, arXiv ID, PMID, or keyword search query. + Returns: dict with metadata, pdf_url (if found), and sources queried. + """ + result = {"input": query_or_id, "metadata": {}, "pdf_url": None, "sources": []} + + # ── Step 1: Identify input type ── + doi_pattern = re.compile(r'10\.\d{4,}/[^\s"\'<>)]+') + arxiv_pattern = re.compile(r'\d{4}\.\d{4,5}(?:v\d+)?') + + input_type = "keywords" + if doi_pattern.search(query_or_id): + input_type = "doi" + result["doi"] = doi_pattern.search(query_or_id).group(0).rstrip("/") + elif arxiv_pattern.search(query_or_id): + input_type = "arxiv" + result["arxiv_id"] = arxiv_pattern.search(query_or_id).group(0) + elif query_or_id.isdigit() and len(query_or_id) <= 8: + input_type = "pmid" + result["pmid"] = query_or_id + + # ── Step 2: Get metadata ── + if input_type == "doi": + doi = result["doi"] + + # CrossRef first (richest metadata) + cr = crossref_metadata(doi) + if cr: + result["metadata"].update(cr) + result["sources"].append("crossref") + + # OpenAlex (citations + concepts) + oa = openalex_work(doi) + if oa: + result["metadata"]["cited_by"] = oa.get("cited_by_count") + result["metadata"]["concepts"] = oa.get("concepts") + result["metadata"]["oa_url"] = oa.get("open_access_url") + result["sources"].append("openalex") + + # Semantic Scholar (TLDR + citation graph) + ss = semantic_scholar_paper(doi) + if ss: + result["metadata"]["tldr"] = ss.get("tldr") + result["metadata"]["ss_citations"] = ss.get("citations") + if ss.get("pdf"): + result["pdf_url"] = ss["pdf"] + result["sources"].append("semantic_scholar") + + # ── Step 3: Try PDF acquisition ── + if not result["pdf_url"]: + upw = unpaywall_find_pdf(doi) + if upw and upw.get("pdf_url"): + result["pdf_url"] = upw["pdf_url"] + result["metadata"]["oa_version"] = upw.get("version") + result["sources"].append("unpaywall") + + if not result["pdf_url"]: + core = core_search(f"doi:{doi}", limit=1) + if core and core[0].get("downloadUrl"): + result["pdf_url"] = core[0]["downloadUrl"] + result["sources"].append("core") + + # BibTeX + result["bibtex"] = get_bibtex(doi) + + elif input_type == "arxiv": + papers = arxiv_by_id(result['arxiv_id']) + if papers: + result["metadata"] = papers[0] + result["pdf_url"] = papers[0].get("pdf_url") + result["sources"].append("arxiv") + + elif input_type == "pmid": + # PubMed lookup + url = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi" + try: + r = requests.get(url, params={ + "db": "pubmed", "id": result["pmid"], + "rettype": "abstract", "retmode": "xml" + }, timeout=15) + import xml.etree.ElementTree as ET + root = ET.fromstring(r.text) + article = root.find(".//PubmedArticle/MedlineCitation/Article") + if article is not None: + result["metadata"]["title"] = article.find("ArticleTitle").text + abstract = article.find("Abstract/AbstractText") + if abstract is not None: + result["metadata"]["abstract"] = abstract.text + result["sources"].append("pubmed") + + # Try to find DOI for further enrichment + doi_el = root.find(".//ArticleId[@IdType='doi']") + if doi_el is not None: + result["doi"] = doi_el.text + # Recurse with DOI for more metadata + except Exception as e: + print(f"[PubMed error] {e}") + + else: # keywords + # Try OpenAlex first (broadest) + oa_url = f"https://api.openalex.org/works?search={query_or_id}&per_page=5" + try: + r = requests.get(oa_url, timeout=15) + if r.status_code == 200: + results = r.json().get("results", []) + if results: + result["search_results"] = [{ + "title": w.get("title"), + "doi": w.get("doi"), + "year": w.get("publication_year"), + "cited_by": w.get("cited_by_count"), + "oa_url": (w.get("open_access") or {}).get("oa_url"), + } for w in results] + result["sources"].append("openalex") + except Exception: + pass + + # Also try arXiv if CS-related + arxiv_results = arxiv_search(query_or_id, max_results=5) + if arxiv_results: + result.setdefault("search_results", []) + result["search_results"].extend([{ + "title": p["title"], + "arxiv_id": p["arxiv_id"], + "pdf_url": p.get("pdf_url"), + "year": p.get("published", "")[:4], + } for p in arxiv_results]) + result["sources"].append("arxiv") + + return result +``` + +--- + +## Failure Modes & Fallback Table + +| Failure | Cause | Fallback | +|---------|-------|----------| +| DOI not in CrossRef | Non-standard DOI, very new paper | Try OpenAlex → Semantic Scholar | +| Unpaywall returns no OA | Paper is behind paywall | Try CORE full text → Europe PMC (if biomedical) → Playwright (Tier 3) on publisher page | +| arXiv API timeout | arXiv servers slow | Retry once (3s delay) → Semantic Scholar by title | +| Semantic Scholar 404 | Paper not indexed | CrossRef → Google Scholar via SerpAPI | +| CORE requires key | Rate limit exceeded without key | Get free key at core.ac.uk/services/api | +| All APIs fail | Obscure paper, network issues | Last resort: Playwright stealth on publisher page, or Google Scholar search | +| BibTeX not available | CrossRef content negotiation fails | Construct manually from metadata | + +--- + +## Rate Limits Summary + +| API | Free Tier | Rate Limit | Key Required? | +|-----|-----------|------------|---------------| +| Unpaywall | 100k/day | Generous | No (email param) | +| arXiv | Unlimited | Be reasonable | No | +| CrossRef | Unlimited | Be reasonable | No (add mailto) | +| OpenAlex | Unlimited | 10 req/s | No (polite pool) | +| Semantic Scholar | 100/5min | 1 req/s with key | Recommended | +| CORE | 1000/day | Higher with key | Recommended | +| Europe PMC | Unlimited | Reasonable use | No | +| DBLP | Unlimited | Reasonable use | No | +| Papers With Code | Unlimited | Reasonable use | No | +| PubMed E-utilities | 10/sec (no key) | Higher with key | No | + +--- + +## Dependencies + +```bash +# All academic search functions use only requests (standard) +pip install requests beautifulsoup4 lxml + +# Optional for PDF text extraction +pip install pymupdf # fitz - extract text from downloaded PDFs +``` + +--- + +*This sub-skill is part of `web-browsing-manual` v3.0. For general web browsing, search strategies, or stealth techniques, see the parent skill and other sub-skills.* diff --git a/src/lingtai/tools/web_search/manual/reference/maintenance-bundles/SKILL.md b/src/lingtai/tools/web_search/manual/reference/maintenance-bundles/SKILL.md new file mode 100644 index 000000000..4f721472d --- /dev/null +++ b/src/lingtai/tools/web_search/manual/reference/maintenance-bundles/SKILL.md @@ -0,0 +1,148 @@ +--- +name: web-browsing-maintenance-bundles +description: > + Nested web-browsing reference for maintenance protocol, semantic sweeps, + dirty-first testing, bundled JSON asset files, deep-dive reference files, and the + explicit decision flowchart. +version: 1.0.0 +last_changed_at: "2026-06-01T01:47:09-07:00" +maintenance: "If you find stale or incorrect information here, use the lingtai-issue-report skill to assemble evidence and obtain per-issue human consent before filing an issue. Never include secrets, credentials, tokens, or private paths." +--- + +# Web Browsing Maintenance and Assets Reference + +Nested web-browsing reference. Open this when changing the skill, validating +propagation, or choosing which bundled asset/reference file to inspect. + +## Maintenance Protocol + +When modifying any code or pattern in this skill, follow these three rules **without exception**: + +### Rule 1: Grep Before You Ship — Semantic Sweep + +Fixing a bug in one file is not enough. After every fix, search the entire codebase for **all occurrences of the same pattern** — not just the same text, but the same **semantic class**. + +```bash +# Fix propagation sweep (run after every code change) +# 1. Same text +grep -rn 'bad_pattern' scripts/ reference/ SKILL.md --include='*.json' . +# 2. Same semantics (e.g., "placeholder email", "outdated API", "hardcoded path") +grep -rn 'example\.com' | grep -iE 'email|mailto|user-agent' +grep -rn 'stealth_sync|bare_extraction.*\.get' +``` + +If a bad pattern appears in `scripts/`, it almost certainly also appears in `reference/`, `SKILL.md`, or bundled JSON asset files. Find them all or they will drift apart. + +### Rule 2: Dirty-First Testing + +Smoke tests must include **dirty inputs** — real-world edge cases that expose runtime failures, not just clean decision-logic checks. Every tier's test must include at least one non-mock verification. + +Minimum dirty test coverage: +- Non-standard URLs (e.g., `/pdf/ID` without `.pdf` suffix) +- Non-dict return types (e.g., `bare_extraction()` returning Document) +- API version incompatibilities (e.g., stealth v1 vs v2) +- Rejected placeholder values (e.g., Unpaywall 422 on `test@example.com`) +- Deprecated endpoints (e.g., Wikipedia `/page/related/`) + +### Rule 3: Single Source of Truth + +If the same logic appears in multiple places (e.g., `auto_tier()` in both `extract_page.py` and `SKILL.md`), **the script is the truth** and documentation should point to it, not duplicate it. Duplicated code drifts; references don't. + +--- + +## Bundled Assets + +| File | Contents | +|------|----------| +| `api-endpoints.json` in the bundled asset directory | Full API endpoints + parameters for every provider | +| `site-templates.json` in the bundled asset directory | CSS-selector templates for known sites | +| `css-selectors.json` in the bundled asset directory | Common-pattern CSS selector library | +| `regex-patterns.json` in the bundled asset directory | Regex templates for DOI / arXiv / PMID / PMC / ISBN | +| `search-providers.json` in the bundled asset directory | Search engine API configurations | +| `extraction-pipeline.json` in the bundled asset directory | Full extraction pipeline configuration | +| `scripts/extract_page.py` | Executable v3.0 script: `--tier 0-5 + auto`, `--search`, `--fallback` | +| `scripts/cached_get.py` | File-based HTTP cache with TTL support | + +--- + +## Reference Files (Deep-Dives) + +For more than quick-reference snippets, load the appropriate reference file: + +| Reference | When to load | +|-----------|-------------| +| [tier-0-pdf.md](../tier-0-pdf.md) | PDF download + fitz extraction details | +| [tier-1-apis.md](../tier-1-apis.md) | All academic/metadata APIs, ID resolution chains | +| [tier-1-5-trafilatura.md](../tier-1-5-trafilatura.md) | Trafilatura configuration, batch mode, dedup | +| [tier-2-beautifulsoup.md](../tier-2-beautifulsoup.md) | BS4 patterns, CSS selectors, site templates | +| [tier-3-playwright.md](../tier-3-playwright.md) | Playwright stealth setup, resource blocking | +| [tier-4-jina-firecrawl.md](../tier-4-jina-firecrawl.md) | Jina Reader + Firecrawl API details | +| [tier-5-ai-search.md](../tier-5-ai-search.md) | DDG / Tavily / Exa search integration | +| [academic-pipeline.md](../academic-pipeline.md) | Full academic search: find → enrich → get PDF | +| [search-strategies.md](../search-strategies.md) | Engine selection, query optimization, pagination | +| [news-and-rss.md](../news-and-rss.md) | Google News RSS, Reddit JSON, RSS parsing | +| [social-media.md](../social-media.md) | Reddit, HN, Mastodon, X/Twitter, GitHub | +| [realtime-data.md](../realtime-data.md) | Financial, weather, Stack Exchange, Wikipedia | +| [stealth.md](../stealth.md) | Anti-detection, fingerprinting, proxy strategies | +| [migration-from-v2.md](../migration-from-v2.md) | What changed from v2 → v3 | + +--- + +## Explicit Decision Flowchart + +``` + ┌──────────────────┐ + │ URL or query │ + └────────┬─────────┘ + │ + ┌──────────────────────────┐ + │ Is it a URL or a keyword? │ + └─────┬──────────────┬─────┘ + URL │ │ keyword + ▼ ▼ + ┌──────────────┐ ┌──────────────┐ + │ Is it PDF? │ │ Tier 5: │ + └──┬──────┬───┘ │ AI Search │ + yes│ no│ │ (DDG/Tavily/ │ + ▼ │ │ Exa) │ + ┌──────────┐ │ └──────────────┘ + │ Tier 0: │ │ + │ PDF+fitz │ │ + └──────────┘ │ + ▼ + ┌──────────────────┐ + │ Known API? │ + │ (arXiv, DOI, │ + │ PubMed, etc.) │ + └──┬──────────┬───┘ + yes│ no│ + ▼ ▼ + ┌──────────┐ ┌──────────────────┐ + │ Tier 1: │ │ Static HTML? │ + │ API query│ │ (article/blog) │ + └──────────┘ └──┬──────────┬─────┘ + yes│ no│ + ▼ ▼ + ┌──────────┐ ┌──────────────────┐ + │ Tier 1.5:│ │ Structured data? │ + │trafilat. │ │ (tables, lists) │ + └──────────┘ └──┬──────────┬─────┘ + yes│ no│ + ▼ ▼ + ┌──────────┐ ┌──────────────┐ + │ Tier 2: │ │ JS-rendered? │ + │ BS4 │ │ Protected? │ + └──────────┘ └──┬──────┬─────┘ + yes│ no│ + ▼ ▼ + ┌──────────┐ ┌──────────┐ + │ Tier 3: │ │ Tier 4: │ + │Playwright│ │ Jina │ + │ stealth │ │ Reader │ + └──────────┘ └──────────┘ +``` + +The compact decision tree near the top is the rule list; this flowchart shows the order of decisions and what each "no" branch does. Read `scripts/extract_page.py::auto_tier()` for the same logic in code (source of truth when this manual drifts). + +--- +> **Found a bug or issue?** If you encounter any problems with this skill, load the `lingtai-issue-report` skill and follow its instructions to report it. diff --git a/src/lingtai/tools/web_search/manual/reference/migration-from-v2.md b/src/lingtai/tools/web_search/manual/reference/migration-from-v2.md new file mode 100644 index 000000000..f437e7ba3 --- /dev/null +++ b/src/lingtai/tools/web_search/manual/reference/migration-from-v2.md @@ -0,0 +1,77 @@ +# Migration from v2 to v3 + +> Part of the [web-browsing](../SKILL.md) skill. + +## Architecture Overview: What Changed + +In **v2**, web browsing capabilities were spread across multiple separate sub-skills living in independent directories: + +``` +~/.lingtai-tui/utilities/ +├── web-content-extractor/ ← core extraction (tiers 0–3) +├── academic-search-pipeline/ ← academic APIs +├── search-strategies/ ← search engine selection +├── news-and-rss/ ← news acquisition +├── social-media-extraction/ ← social media +├── realtime-data/ ← real-time sources +└── stealth-browsing/ ← anti-detection +``` + +In **v3**, all of these have been **merged into a single unified skill** (`web-browsing-manual`). The tier system was expanded from 4 tiers (0–3) to 7 tiers (0–5, with 1.5), and new capabilities were added at every level. The sub-skills still exist as reference directories, but the main `SKILL.md` is now the single source of truth. + +### What's New in v3 + +| Feature | Tier | Description | +|---------|------|-------------| +| **Tier 1.5 — trafilatura** | NEW | Fast article extraction using the `trafilatura` library. 10–50× faster than BeautifulSoup (Tier 2). Now the default auto-tier choice for generic URLs. | +| **Tier 4 — Jina Reader / Firecrawl** | NEW | Cloud-based extraction APIs as fallback when all local methods fail (403, CAPTCHA, heavy JS). Jina Reader is free and should be the first fallback. | +| **Tier 5 — AI-Native Search** | NEW | Tavily and Exa APIs for discovering content (not extracting a known URL). Combines search + extraction in one call. | +| **Integrated search mode** | NEW | `--search --search-provider {ddg,tavily,exa}` flag brings search directly into the extraction script. | +| **Fallback chain** | NEW | `--fallback` flag enables automatic tier escalation on failure. | +| **Expanded academic APIs** | Enhanced | Tier 1 now also includes Semantic Scholar, Unpaywall, and PubMed E-utilities. | +| **Resource blocking in Tier 3** | Improved | Playwright now blocks images/CSS/fonts, making Tier 3 ~2× faster. | +| **Richer output** | Improved | Results now include `text_length`, `jsonld`, `opengraph`, `meta_*` fields. | +| **Built-in tests** | NEW | `--test` flag for verification. | + +### What Moved Where + +| v2 Location | v3 Equivalent | +|-------------|---------------| +| `web-content-extractor/SKILL.md` | Merged into `web-browsing-manual/SKILL.md` (tiers 0–5) | +| `academic-search-pipeline/` | Referenced as sub-skill; APIs integrated into Tier 1 | +| `search-strategies/` | Referenced as sub-skill; search integrated into Tier 5 | +| `news-and-rss/` | Referenced as sub-skill | +| `stealth-browsing/` | Referenced as sub-skill; techniques in Tier 3 | +| `social-media-extraction/` | Referenced as sub-skill | +| `realtime-data/` | Referenced as sub-skill | + +--- + +## Migration Notes: v2.0.0 → v3.0 + +| v2.0 Feature | v3.0 Equivalent | Changes | +|-------------|----------------|---------| +| `--tier 0/1/2/3/auto` | `--tier 0/1/1.5/2/3/4/5/auto` | Added Tier 1.5 (trafilatura), Tier 4 (Jina Reader), Tier 5 (AI Search) | +| `tier2()` = BS4 extraction | `tier1_5()` = trafilatura (new default for most sites) | **Default auto-tier changed from 2 to 1.5** for generic URLs — trafilatura is 10-50× faster | +| No search mode | `--search --search-provider {ddg,tavily,exa}` | Search is now integrated into the script | +| No fallback chain | `--fallback` flag | Automatic tier escalation on failure | +| `extract_page.py --save pdf.pdf` | Same, unchanged | PDF save still works | +| Academic APIs: arXiv, CrossRef | + Semantic Scholar, Unpaywall, PubMed E-utilities | Tier 1 now tries multiple APIs | +| No resource blocking in Tier 3 | Blocks images/CSS/fonts in Playwright | Tier 3 is ~2× faster | +| Site detection: 6 domains | 30+ domains in auto_tier() | Better auto-detection | +| Output: text preview only | + `text_length`, `jsonld`, `opengraph`, `meta_*` | Richer structured output | +| v2.0 `--tier 2` for nature.com | v3.0 auto picks Tier 2 for nature.com | Same, but now falls through to Tier 3 on failure | +| No smoke tests | `--test` flag | Built-in verification | + +### Breaking changes + +- **Auto-tier default changed**: Most generic URLs now route to Tier 1.5 (trafilatura) instead of Tier 2 (BS4). This is faster and usually better, but if you relied on BS4-specific extraction (CSS selectors, site templates), use `--tier 2` explicitly. +- **Tier numbering**: Tier 2 is still BS4. Tier 1.5 is new (trafilatura). Your old `--tier 2` calls still work identically. +- **Output format**: Results now include additional fields (`text_length`, `jsonld`, `opengraph`). Existing JSON parsing should be forward-compatible. + +### Post-release patches (2026-04-27) + +| Issue | Symptom | Fix | +|-------|---------|-----| +| Bare DOI input crash | `python3 extract_page.py "10.1038/xxx"` → `Invalid URL: No scheme supplied` | Added input normalizer: bare DOI → `https://doi.org/...`, bare arXiv ID → `https://arxiv.org/abs/...`, non-URL text → auto search mode | +| `duckduckgo_search` renamed | `RuntimeWarning: This package has been renamed to ddgs` on every search | Import now tries `from ddgs import DDGS` first, falls back to `from duckduckgo_search import DDGS` | diff --git a/src/lingtai/tools/web_search/manual/reference/news-and-rss.md b/src/lingtai/tools/web_search/manual/reference/news-and-rss.md new file mode 100644 index 000000000..aebe3cbb3 --- /dev/null +++ b/src/lingtai/tools/web_search/manual/reference/news-and-rss.md @@ -0,0 +1,1101 @@ +# News and RSS Feeds + +> Part of the [web-browsing](../SKILL.md) skill. +> Google News RSS, Reddit JSON news feeds, RSS feed discovery and parsing, paywall boundary handling, news archival via Wayback Machine. + +# News & RSS — 新闻获取与 RSS 订阅处理 + +本技能是 `web-browsing-manual` 的子技能,专注于新闻类内容的获取、RSS 订阅源的发现与解析、付费墙边界的处理、以及新闻归档。当你需要获取新闻、监控信息流、或解析 RSS/Atom feed 时,使用本技能。 + +--- + +## 1. 新闻获取决策树 + +面对"我需要获取新闻"这个需求时,按下图选择最佳路径: + +``` +新闻需求 + │ + ├─ 特定主题新闻? ──────────────→ Google News RSS(§2) + │ (如 "AI breakthroughs") + │ + ├─ 社区讨论与热点? ────────────→ Reddit JSON API(§3) + │ (如 r/worldnews, r/technology) + → Hacker News(见 social-media-extraction) + │ + ├─ 特定网站新闻? ──────────────→ RSS Feed 发现 + feedparser(§4) + │ (如 CNN, BBC, NYT) + │ + ├─ 已有 URL 但遇到付费墙? ─→ 付费墙绕行边界(§5) + │ 尝试链: trafilatura → Jina Reader → Wayback Machine + │ + └─ 需要归档或查历史? ────────→ Wayback Machine API(§6) + (如页面可能被删除、需要保存证据) +``` + +**快速选择指南:** + +| 需求 | 推荐方法 | 耗时 | 精度 | +|------|----------|------|------| +| 搜某个话题的最新新闻 | Google News RSS | <2s | 高 | +| 了解社区在讨论什么 | Reddit JSON API | <3s | 中 | +| 持续监控某个网站 | RSS Feed 发现 + 解析 | 首次<5s | 高 | +| 读到一半被付费墙挡住 | 付费墙绕行链 | 3-10s | 不确定 | +| 确保新闻链接不过期 | Wayback Machine 归档 | 5-60s | 高 | + +--- + +## 2. Google News RSS + +Google News 提供免费的 RSS 接口,无需 API Key,无速率限制,适合按主题搜索最新新闻。 + +### 2.1 完整代码 + +```python +import requests +import feedparser +from bs4 import BeautifulSoup + + +def google_news_rss(query, hl="en", gl="US", ceid="US:en"): + """Search Google News via RSS feed. + + Args: + query: Search query string (e.g. "artificial intelligence"). + hl: Language hint (e.g. "en", "zh-CN", "ja"). + gl: Country/region code (e.g. "US", "CN", "JP"). + ceid: Country:language edition (e.g. "US:en", "CN:zh-Hans"). + + Returns: + List of dicts with keys: title, url, published, source. + """ + url = "https://news.google.com/rss/search" + params = { + "q": query, + "hl": hl, + "gl": gl, + "ceid": ceid, + } + headers = {"User-Agent": "Mozilla/5.0 (compatible; LingTai/1.0)"} + + r = requests.get(url, params=params, timeout=15, headers=headers) + r.raise_for_status() + + feed = feedparser.parse(r.text) + articles = [] + + for entry in feed.entries[:20]: + # Google News titles often contain HTML tags + title = BeautifulSoup(entry.title, "lxml").get_text() + + source_name = None + source_tag = entry.get("source") + if isinstance(source_tag, dict): + source_name = source_tag.get("title") + elif isinstance(source_tag, str): + source_name = source_tag + + articles.append({ + "title": title, + "url": entry.link, + "published": entry.get("published"), + "source": source_name, + }) + + return articles +``` + +### 2.2 参数详解 + +| 参数 | 说明 | 示例 | +|------|------|------| +| `q` | 搜索关键词,支持引号精确匹配 | `"OpenAI GPT-5"` | +| `hl` | 界面语言 | `"en"`, `"zh-CN"`, `"ja"`, `"de"` | +| `gl` | 国家/地区 | `"US"`, `"CN"`, `"JP"`, `"DE"` | +| `ceid` | 版本标识,格式 `{国家}:{语言}` | `"US:en"`, `"CN:zh-Hans"` | + +**常用语言/地区组合:** + +``` +英语美国: hl=en gl=US ceid=US:en +中文中国: hl=zh-CN gl=CN ceid=CN:zh-Hans +日语日本: hl=ja gl=JP ceid=JP:ja +德语德国: hl=de gl=DE ceid=DE:de +法语法国: hl=fr gl=FR ceid=FR:fr +``` + +### 2.3 进阶用法 + +**按时间范围搜索:** 在查询中加入 `when:` 修饰符: + +```python +# 过去一天的新闻 +articles = google_news_rss("AI when:1d") + +# 过去一周的新闻 +articles = google_news_rss("climate change when:7d") +``` + +**排除特定来源:** 使用 `-source:` 修饰符: + +```python +articles = google_news_rss("technology -source:foxnews.com") +``` + +**获取特定新闻主题的 RSS:** 不用搜索,直接订阅主题 RSS: + +```python +def google_news_topic(topic="WORLD", hl="en", gl="US", ceid="US:en"): + """Get Google News topic feed. + + Valid topics: WORLD, NATION, BUSINESS, TECHNOLOGY, + ENTERTAINMENT, SPORTS, SCIENCE, HEALTH + """ + url = f"https://news.google.com/rss/topics/CAAqJggKIiBDQkFTRWdvSUwyMHZNRGx1YlY4U0FtVnVHZ0pWVXlnQVAB" + # Note: topic tokens change; use search-based approach for reliability. + # For topic feeds, it's easier to use the web interface and copy the RSS URL. + ... +``` + +> **提示:** topic token 不稳定,推荐始终使用 `search` 方式。 + +### 2.4 注意事项 + +- **无需 API Key**:Google News RSS 是免费公开接口。 +- **无速率限制**:但仍应遵守合理使用,避免过于频繁请求。 +- **返回 RSS XML**:用 `feedparser` 解析为结构化数据。 +- **标题含 HTML**:Google News 的标题字段中可能包含 `` 等 HTML 标签,需要用 BeautifulSoup 清理。 +- **URL 重定向**:返回的链接是 Google 的重定向 URL,最终会跳转到原始文章。 + +--- + +## 3. Reddit JSON API 新闻流 + +Reddit 提供免费的 JSON API,只需在 URL 后追加 `.json` 即可获取结构化数据。适合获取社区驱动的新闻与讨论。 + +### 3.1 完整代码 + +```python +import requests +import time + + +# Simple rate limiter — track last request time +_last_request_time = 0 +_MIN_INTERVAL = 2.0 # seconds between requests + + +def _rate_limited_get(url, **kwargs): + """Enforce minimum interval between Reddit API requests.""" + global _last_request_time + elapsed = time.time() - _last_request_time + if elapsed < _MIN_INTERVAL: + time.sleep(_MIN_INTERVAL - elapsed) + _last_request_time = time.time() + return requests.get(url, **kwargs) + + +def reddit_news(subreddit="worldnews", sort="hot", limit=25, timeframe="day"): + """Get news posts from a Reddit subreddit. + + Args: + subreddit: Subreddit name (without r/ prefix). + sort: Sort order — "hot", "new", "top", "rising". + limit: Number of posts to return (max 100). + timeframe: Time range for "top" sort — "hour", "day", "week", + "month", "year", "all". + + Returns: + List of dicts with post metadata. + """ + url = f"https://www.reddit.com/r/{subreddit}/{sort}.json" + headers = {"User-Agent": "LingTai/1.0 (News Aggregator)"} + params = {"limit": limit} + if sort == "top": + params["t"] = timeframe + + r = _rate_limited_get(url, headers=headers, params=params, timeout=15) + r.raise_for_status() + + data = r.json() + posts = [] + + for child in data["data"]["children"]: + post = child["data"] + posts.append({ + "title": post["title"], + "url": post["url"], + "score": post["score"], + "comments": post["num_comments"], + "author": post.get("author"), + "created_utc": post.get("created_utc"), + "selftext": post.get("selftext", "")[:500], + "is_self": post.get("is_self", False), + "subreddit": post.get("subreddit"), + "link_flair_text": post.get("link_flair_text"), + }) + + return posts + + +def reddit_search(query, sort="new", limit=25, subreddit=None): + """Search Reddit posts. + + Args: + query: Search query string. + sort: Sort order — "relevance", "new", "top", "comments". + limit: Number of results (max 100). + subreddit: Restrict to a specific subreddit (optional). + """ + if subreddit: + url = f"https://www.reddit.com/r/{subreddit}/search.json" + else: + url = "https://www.reddit.com/search.json" + headers = {"User-Agent": "LingTai/1.0 (News Aggregator)"} + params = {"q": query, "sort": sort, "limit": limit, "restrict_sr": "on" if subreddit else "off"} + + r = _rate_limited_get(url, headers=headers, params=params, timeout=15) + r.raise_for_status() + + data = r.json() + return [ + { + "title": child["data"]["title"], + "url": child["data"]["url"], + "score": child["data"]["score"], + "subreddit": child["data"].get("subreddit"), + "selftext": child["data"].get("selftext", "")[:500], + } + for child in data["data"]["children"] + ] +``` + +### 3.2 Reddit API 速率限制 + +| 条件 | 限制 | +|------|------| +| 无 OAuth 认证 | 约 60 请求/分钟 | +| 必须设置 User-Agent | 否则可能被拒绝或限速更严 | +| 建议请求间隔 | ≥ 2 秒 | +| 每次请求最大结果数 | 100 条 | + +**关键规则:** + +1. **必须设置 User-Agent**:不设或使用默认 User-Agent 会被严厉限速甚至封禁。 +2. **间隔请求**:在循环中抓取时,每次请求间隔至少 2 秒。 +3. **不要模拟登录**:使用 OAuth 认证提升限额是可以的,但不要破解或绕过认证。 +4. **分页**:使用 `after` 参数翻页: + ```python + # First page + params = {"limit": 25} + # Next page — use the "after" value from previous response + params = {"limit": 25, "after": data["data"]["after"]} + ``` + +### 3.3 常用新闻子板块 + +| 子板块 | 内容 | 适用场景 | +|--------|------|----------| +| r/worldnews | 国际新闻 | 全球时事 | +| r/news | 美国及国际新闻 | 综合新闻 | +| r/technology | 科技新闻 | 科技动态 | +| r/science | 科学新闻 | 科研进展 | +| r/politics | 美国政治 | 政治分析 | +| r/business | 商业财经 | 经济动态 | + +--- + +## 4. RSS Feed 发现与解析 + +当你想持续监控某个新闻网站时,RSS 是最稳定的方式。本节提供自动发现和解析 RSS 的完整工具。 + +### 4.1 Feed 发现 + +```python +import re +import requests +from bs4 import BeautifulSoup +from urllib.parse import urlparse, urljoin + + +def discover_rss(url): + """Find RSS/Atom feed URLs from a webpage. + + Strategy: + 1. Check tags. + 2. Check tags. + 3. Probe common RSS paths. + + Args: + url: Webpage URL to scan for feed links. + + Returns: + List of dicts: {"url": ..., "type": ..., "label": ...} + """ + headers = {"User-Agent": "Mozilla/5.0 (compatible; LingTai/1.0)"} + r = requests.get(url, timeout=15, headers=headers) + soup = BeautifulSoup(r.text, "lxml") + + feeds = [] + + # Strategy 1 & 2: tags with RSS/Atom types + for link in soup.find_all("link", rel=True): + rel = link.get("rel", []) + type_ = link.get("type", "") + if "alternate" in rel and ("rss" in type_ or "atom" in type_): + href = link.get("href", "") + if href: + # Resolve relative URLs + href = urljoin(url, href) + feeds.append({ + "url": href, + "type": type_, + "label": link.get("title", ""), + }) + + # Strategy 3: Probe common RSS paths + parsed = urlparse(url) + base = f"{parsed.scheme}://{parsed.netloc}" + common_paths = [ + "/feed", "/rss", "/feed.xml", "/rss.xml", "/index.xml", + "/feeds/all.xml", "/atom.xml", "/feed/", "/rss/", + "/news/rss", "/news/feed", "/api/rss", + ] + for path in common_paths: + probe_url = base + path + try: + probe_r = requests.head(probe_url, timeout=5, headers=headers, + allow_redirects=True) + content_type = probe_r.headers.get("Content-Type", "") + if probe_r.status_code == 200 and ("xml" in content_type or "rss" in content_type): + feeds.append({"url": probe_url, "type": "probed", "label": path}) + except requests.RequestException: + continue + + # Deduplicate + seen = set() + unique = [] + for f in feeds: + if f["url"] not in seen: + seen.add(f["url"]) + unique.append(f) + + return unique +``` + +### 4.2 Feed 解析 + +```python +import feedparser + + +def parse_rss(feed_url, max_entries=20): + """Parse RSS/Atom feed and return structured entries. + + Args: + feed_url: URL of the RSS/Atom feed. + max_entries: Maximum number of entries to return. + + Returns: + Dict with "title" (feed title), "entries" (list of article dicts). + """ + feed = feedparser.parse(feed_url) + + if not feed.entries: + # feedparser might have failed — check bozo + if feed.bozo and feed.bozo_exception: + return {"title": None, "entries": [], "error": str(feed.bozo_exception)} + return {"title": feed.feed.get("title"), "entries": []} + + entries = [] + for entry in feed.entries[:max_entries]: + # Extract best available content + content = None + if hasattr(entry, "content") and entry.content: + content = entry.content[0].get("value") + elif hasattr(entry, "summary"): + content = entry.summary + elif hasattr(entry, "description"): + content = entry.description + + # Clean HTML from content + if content: + content = BeautifulSoup(content, "lxml").get_text() + + entries.append({ + "title": entry.get("title"), + "url": entry.get("link"), + "published": entry.get("published"), + "updated": entry.get("updated"), + "summary": content[:1000] if content else None, + "author": entry.get("author"), + "tags": [t.get("term") for t in entry.get("tags", [])] if entry.get("tags") else [], + }) + + return { + "title": feed.feed.get("title"), + "link": feed.feed.get("link"), + "entries": entries, + } +``` + +### 4.3 使用示例 + +```python +# Example 1: Discover and parse CNN RSS +feeds = discover_rss("https://www.cnn.com") +if feeds: + articles = parse_rss(feeds[0]["url"]) + for article in articles["entries"][:5]: + print(f"[{article['published']}] {article['title']}") + print(f" {article['url']}") + +# Example 2: Direct known RSS feed +articles = parse_rss("https://feeds.bbci.co.uk/news/world/rss.xml") +print(f"Feed: {articles['title']}") +for a in articles["entries"][:10]: + print(f" - {a['title']} ({a['published']})") + +# Example 3: Batch monitor multiple feeds +FEEDS = [ + "https://feeds.bbci.co.uk/news/world/rss.xml", + "https://rss.nytimes.com/services/xml/rss/nyt/World.xml", + "https://feeds.reuters.com/reuters/topNews", +] +for feed_url in FEEDS: + result = parse_rss(feed_url, max_entries=5) + print(f"\n=== {result.get('title', feed_url)} ===") + for entry in result["entries"]: + print(f" {entry['title']}") +``` + +### 4.4 feedparser 诊断 + +`feedparser` 有内置的诊断机制: + +```python +feed = feedparser.parse("https://example.com/feed.xml") + +# Check if feed was parsed successfully +if feed.bozo: + print(f"Parse warning: {feed.bozo_exception}") + # Common: XML encoding issues, malformed dates + # feedparser still tries to extract data despite bozo errors + +# Check feed metadata +print(f"Feed title: {feed.feed.get('title')}") +print(f"Feed link: {feed.feed.get('link')}") +print(f"Entries found: {len(feed.entries)}") + +# Check entry fields available +if feed.entries: + print(f"Available fields: {[k for k in feed.entries[0].keys()]}") +``` + +--- + +## 5. 付费墙绕行边界 + +### 5.1 伦理边界 + +**明确什么能做,什么不能做:** + +| 能做 ✅ | 不能做 ❌ | +|---------|----------| +| 使用 RSS 获取公开摘要 | 破解登录验证系统 | +| 使用 Google 缓存 | 绕过服务器端付费墙 | +| 使用 Wayback Machine 查看归档 | 盗用付费订阅内容 | +| trafilatura 提取(对客户端付费墙有效) | 模拟已付费用户身份 | +| Jina Reader 转换页面 | 自动化破解 CAPTCHA | +| 提取 meta 标签和 OG 数据 | 共享付费内容的完整副本 | + +**判断原则:如果内容需要用户登录才能看到,应该停止尝试。** 客户端付费墙(内容在 HTML 中但被 JS 隐藏)与服务器端付费墙(内容根本不在 HTML 中)有本质区别。前者是技术策略,后者是明确的付费意图。 + +### 5.2 绕行策略链 + +```python +import requests +import trafilatura + + +def try_behind_paywall(url): + """Try to extract content from a potentially paywalled page. + + Tries strategies in order of respectfulness: + 1. trafilatura — handles client-side paywalls + 2. Jina Reader — lightweight conversion + 3. Wayback Machine — check archived version + + Args: + url: The article URL that may be behind a paywall. + + Returns: + Dict with "method" and content, or None if all strategies fail. + """ + headers = {"User-Agent": "Mozilla/5.0 (compatible; LingTai/1.0)"} + + # ── Strategy 1: trafilatura ──────────────────────────────────── + # Works against client-side paywalls where content is in HTML + # but hidden by JavaScript. + try: + html = trafilatura.fetch_url(url) + if html: + text = trafilatura.extract(html) + if text and len(text) > 200: + return {"method": "trafilatura", "content": text} + except Exception: + pass + + # ── Strategy 2: Jina Reader API ──────────────────────────────── + # Converts pages to clean markdown. Often bypasses simple paywalls. + try: + jina_url = f"https://r.jina.ai/{url}" + r = requests.get(jina_url, timeout=30) + if r.status_code == 200 and len(r.text) > 200: + return {"method": "jina", "content": r.text} + except Exception: + pass + + # ── Strategy 3: Wayback Machine ──────────────────────────────── + # Check if an archived version exists. Most respectful approach. + try: + wb_url = f"https://archive.org/wayback/available?url={url}" + wb = requests.get(wb_url, timeout=15) + if wb.status_code == 200: + data = wb.json() + snapshots = data.get("archived_snapshots", {}) + closest = snapshots.get("closest") + if closest and closest.get("available"): + archive_url = closest["url"] + # Fetch the archived page + ar = requests.get(archive_url, timeout=15) + if ar.status_code == 200: + text = trafilatura.extract(ar.text) + if text and len(text) > 200: + return { + "method": "wayback", + "archive_url": archive_url, + "timestamp": closest.get("timestamp"), + "content": text, + } + except Exception: + pass + + # ── All strategies failed — respect the paywall ──────────────── + # Try to extract at least the meta description / OG summary + try: + from bs4 import BeautifulSoup + r = requests.get(url, timeout=15, headers=headers) + soup = BeautifulSoup(r.text, "lxml") + og_desc = soup.find("meta", property="og:description") + meta_desc = soup.find("meta", attrs={"name": "description"}) + title = soup.find("title") + summary = None + if og_desc: + summary = og_desc.get("content") + elif meta_desc: + summary = meta_desc.get("content") + if summary: + return { + "method": "meta_only", + "title": title.get_text() if title else None, + "summary": summary, + "note": "Full content behind paywall — only summary available.", + } + except Exception: + pass + + return None # Give up — respect the paywall +``` + +### 5.3 常见付费墙类型 + +| 类型 | 特征 | trafilatura | Jina | Wayback | +|------|------|:-----------:|:----:|:-------:| +| 客户端 (JS overlay) | 内容在 HTML 中,被 JS 遮盖 | ✅ 常有效 | ✅ 常有效 | ✅ | +| 计量付费墙 (metered) | 允许 N 篇/月 | ✅ | ✅ | ✅ | +| 服务器端 | HTML 中无正文 | ❌ | ❌ | ✅ | +| 硬付费墙 | 必须登录 | ❌ | ❌ | ❌ | + +--- + +## 6. 新闻归档(Wayback Machine) + +Wayback Machine 提供免费的网页归档服务。适合保存新闻链接,防止链接失效或内容被修改。 + +### 6.1 检查归档 + +```python +import requests + + +def wayback_check(url): + """Check if an archived version of a URL exists. + + Args: + url: The URL to check. + + Returns: + Dict with "available", "url", "timestamp" if found. + """ + api_url = "https://archive.org/wayback/available" + params = {"url": url} + r = requests.get(api_url, params=params, timeout=15) + + data = r.json() + closest = data.get("archived_snapshots", {}).get("closest") + + if closest and closest.get("available"): + return { + "available": True, + "url": closest["url"], + "timestamp": closest["timestamp"], + "status": closest.get("status"), + } + + return {"available": False} + + +def wayback_get_all_snapshots(url): + """Get list of all archived snapshots for a URL. + + Returns: + List of dicts with "timestamp", "status", "url". + """ + api_url = "https://web.archive.org/cdx/search/cdx" + params = { + "url": url, + "output": "json", + "fl": "timestamp,statuscode,original", + "limit": 50, + } + r = requests.get(api_url, params=params, timeout=30) + r.raise_for_status() + + rows = r.json() + if len(rows) <= 1: + return [] + + # First row is header + snapshots = [] + for row in rows[1:]: + snapshots.append({ + "timestamp": row[0], + "status": row[1], + "url": f"https://web.archive.org/web/{row[0]}/{row[2]}", + }) + + return snapshots +``` + +### 6.2 保存页面 + +```python +def wayback_save(url): + """Request Wayback Machine to archive a page. + + Note: This is asynchronous. The archive may take minutes to become available. + + Args: + url: The URL to archive. + + Returns: + Dict with "saved" status and archive URL if successful. + """ + save_url = f"https://web.archive.org/save/{url}" + headers = {"User-Agent": "Mozilla/5.0 (compatible; LingTai/1.0)"} + + try: + r = requests.get(save_url, timeout=60, headers=headers) + if r.status_code == 200: + # The response headers may contain the archive path + archive_path = r.headers.get("Content-Location", "") + if archive_path: + archive_url = f"https://web.archive.org{archive_path}" + else: + # Try to parse from response + archive_url = f"https://web.archive.org/web/*/{url}" + return {"saved": True, "archive_url": archive_url} + return {"saved": False, "status_code": r.status_code} + except requests.RequestException as e: + return {"saved": False, "error": str(e)} + + +def wayback_save_and_verify(url, max_wait=120): + """Save a page and wait for it to become available. + + Args: + url: The URL to archive. + max_wait: Maximum seconds to wait for verification. + + Returns: + Dict with verified archive URL. + """ + import time + + result = wayback_save(url) + if not result["saved"]: + return result + + # Poll until available + start = time.time() + while time.time() - start < max_wait: + check = wayback_check(url) + if check["available"]: + return {"saved": True, "verified": True, "archive_url": check["url"]} + time.sleep(5) + + return {"saved": True, "verified": False, "archive_url": result.get("archive_url")} +``` + +### 6.3 使用场景 + +```python +# Scenario 1: Check if a news article has been archived +result = wayback_check("https://www.example.com/news/article-123") +if result["available"]: + print(f"Archived at: {result['url']} (snapshot: {result['timestamp']})") +else: + print("No archive found.") + +# Scenario 2: Archive a potentially ephemeral news article +result = wayback_save("https://www.example.com/breaking-news") +print(f"Archive requested: {result}") + +# Scenario 3: Get full history of a URL +snapshots = wayback_get_all_snapshots("https://www.example.com/article") +for s in snapshots: + print(f" {s['timestamp']} (status {s['status']}) → {s['url']}") +``` + +--- + +## 7. 失败模式与排查 + +### 7.1 Google News RSS + +| 症状 | 原因 | 解决方案 | +|------|------|----------| +| 返回空结果 | 查询词编码问题 | 简化关键词,避免特殊字符 | +| 返回空结果 | `ceid` 不匹配 | 确保语言/国家组合正确 | +| 连接超时 | 网络问题或被封锁 | 使用代理,或换用 Reddit | +| feedparser 报错 | 返回非 XML | 检查 HTTP 状态码,可能被重定向 | + +### 7.2 Reddit JSON API + +| 症状 | 原因 | 解决方案 | +|------|------|----------| +| HTTP 429 | 速率限制 | 降低频率,确保 User-Agent 已设置 | +| HTTP 403 | 被封禁或私有子板块 | 检查子板块是否存在和公开 | +| 返回空 `children` | 子板块无帖子或不存在 | Reddit 对不存在的子板块也返回 200+空列表,需检查 `dist` 或 `children` 是否为空;尝试 `sort="new"` 或换子板块 | +| JSON 解析失败 | 可能返回 HTML 错误页 | 检查 `r.status_code` 和 `Content-Type` | + +### 7.3 RSS Feed 发现 + +| 症状 | 原因 | 解决方案 | +|------|------|----------| +| 发现不到 feed | 网站不提供 RSS | 尝试手动拼接常见路径 | +| 发现到的 feed 解析失败 | URL 格式错误或需要认证 | 用 `feedparser` 的 `bozo` 诊断 | +| feed 内容过时 | feed 不再更新 | 检查 `updated` 字段,寻找替代 feed | +| 中文 feed 乱码 | 编码问题 | feedparser 通常自动处理,检查 `encoding` | + +### 7.4 付费墙 + +| 症状 | 原因 | 解决方案 | +|------|------|----------| +| trafilatura 返回极短文本 | 服务器端付费墙 | 尝试 Jina Reader 或 Wayback | +| Jina Reader 返回付费提示 | 内容确实需要登录 | 停止尝试,仅取摘要 | +| Wayback 无归档 | 页面从未被归档 | 返回 meta 摘要,标记不可用 | +| 所有策略失败 | 硬付费墙 | 返回 `None`,记录链接供后续参考 | + +### 7.5 Wayback Machine + +| 症状 | 原因 | 解决方案 | +|------|------|----------| +| `wayback_check` 返回 `available: False` | 从未被归档 | 使用 `wayback_save` 主动归档 | +| `wayback_save` 超时 | Wayback 服务繁忙 | 重试,或接受异步结果 | +| 归档页面加载慢 | Wayback CDN 负载高 | 等待或使用 `web_archive.org` 直接链接 | + +--- + +## 8. 重试退避与降级策略 + +外域接口诡变无常——速率越限、连接超时、令牌见弃皆为常态。本节提供系统性的重试、退避与降级方案。 + +### 8.1 通用重试装饰器 + +```python +import requests +import time +import logging + +logger = logging.getLogger(__name__) + + +def retry_with_backoff(max_retries=3, base_delay=2.0, backoff_factor=2.0, + retryable_statuses=(429, 500, 502, 503, 504)): + """Decorator: retry a function with exponential backoff. + + Args: + max_retries: Maximum retry attempts (0 = no retry). + base_delay: Initial delay in seconds. + backoff_factor: Multiplier for each subsequent delay. + retryable_statuses: HTTP status codes that trigger a retry. + """ + def decorator(func): + def wrapper(*args, **kwargs): + last_exception = None + for attempt in range(max_retries + 1): + try: + result = func(*args, **kwargs) + # If result is a requests.Response, check status + if isinstance(result, requests.Response): + if result.status_code in retryable_statuses: + retry_after = result.headers.get("Retry-After") + delay = float(retry_after) if retry_after else base_delay * (backoff_factor ** attempt) + logger.warning( + f"HTTP {result.status_code} on attempt {attempt + 1}/{max_retries + 1}, " + f"retrying in {delay:.1f}s" + ) + time.sleep(delay) + continue + result.raise_for_status() + return result + except (requests.Timeout, requests.ConnectionError) as e: + last_exception = e + if attempt < max_retries: + delay = base_delay * (backoff_factor ** attempt) + logger.warning( + f"Network error on attempt {attempt + 1}/{max_retries + 1}: {e}, " + f"retrying in {delay:.1f}s" + ) + time.sleep(delay) + else: + raise + raise last_exception # Should not reach here + return wrapper + return decorator +``` + +### 8.2 新闻降级链 + +当一个数据源失败时,自动降级到备选源: + +```python +def fetch_news_with_fallback(query, max_results=10): + """Fetch news with automatic fallback across sources. + + Priority chain: + 1. Google News RSS (broad, topic-based) + 2. Reddit search (community-driven) + 3. RSS feed of major outlet (curated) + + Returns: + List of article dicts, or empty list if all sources fail. + """ + articles = [] + + # ── Tier 1: Google News RSS ──────────────────────────────────── + try: + articles = google_news_rss(query, limit=max_results) + if articles: + return articles[:max_results] + except Exception as e: + logger.info(f"Google News failed, falling back: {e}") + + # ── Tier 2: Reddit Search ────────────────────────────────────── + try: + reddit_results = reddit_search(query, sort="new", limit=max_results) + for r in reddit_results: + articles.append({ + "title": r["title"], + "url": r["url"] if not r["is_self"] else f"https://reddit.com{r.get('permalink', '')}", + "source": f"r/{r['subreddit']}", + "score": r["score"], + }) + if articles: + return articles[:max_results] + except Exception as e: + logger.info(f"Reddit search failed, falling back: {e}") + + # ── Tier 3: Major RSS outlets ────────────────────────────────── + fallback_feeds = [ + "https://feeds.bbci.co.uk/news/world/rss.xml", + "https://rss.nytimes.com/services/xml/rss/nyt/World.xml", + ] + for feed_url in fallback_feeds: + try: + parsed = parse_rss(feed_url, max_entries=max_results) + for entry in parsed["entries"]: + articles.append({ + "title": entry.get("title"), + "url": entry.get("url"), + "source": parsed.get("title", feed_url), + "published": entry.get("published"), + }) + if articles: + return articles[:max_results] + except Exception: + continue + + logger.warning("All news sources failed.") + return [] +``` + +### 8.3 付费墙降级链 + +当逐级尝试付费墙绕行时,每层失败应优雅地继续,而非崩溃: + +```python +def extract_with_graceful_degradation(url): + """Extract article content with graceful degradation. + + Tries: full text → archive text → summary → meta → None. + Always returns a dict with "level" indicating what was obtained. + """ + # Level 0: Direct extraction + try: + import trafilatura + html = trafilatura.fetch_url(url) + if html: + text = trafilatura.extract(html) + if text and len(text) > 200: + return {"level": "full", "method": "trafilatura", "content": text} + except Exception: + pass + + # Level 1: Jina Reader + try: + r = requests.get(f"https://r.jina.ai/{url}", timeout=30) + if r.status_code == 200 and len(r.text) > 200: + return {"level": "full", "method": "jina", "content": r.text} + except Exception: + pass + + # Level 2: Wayback Machine + try: + wb = requests.get( + f"https://archive.org/wayback/available?url={url}", timeout=15 + ) + closest = wb.json().get("archived_snapshots", {}).get("closest") + if closest and closest.get("available"): + ar = requests.get(closest["url"], timeout=15) + text = trafilatura.extract(ar.text) + if text and len(text) > 200: + return { + "level": "archived", + "method": "wayback", + "timestamp": closest["timestamp"], + "content": text, + } + except Exception: + pass + + # Level 3: Meta summary only + try: + from bs4 import BeautifulSoup + r = requests.get(url, timeout=15, headers={"User-Agent": "Mozilla/5.0"}) + soup = BeautifulSoup(r.text, "lxml") + og_desc = soup.find("meta", property="og:description") + title = soup.find("title") + summary = og_desc.get("content") if og_desc else None + if summary: + return { + "level": "summary", + "title": title.get_text() if title else None, + "content": summary, + } + except Exception: + pass + + # Level 4: Nothing available + return {"level": "unavailable", "url": url} +``` + +### 8.4 各平台退避参数 + +| 平台 | 首次失败延迟 | 最大重试 | 退避因子 | 429 特殊处理 | +|------|-------------|----------|---------|-------------| +| Google News RSS | 2s | 3 | 2x | 无 Retry-After header,用指数退避 | +| Reddit | 2s | 3 | 3x | 遵守 `Retry-After` header(秒数) | +| Wayback Machine | 5s | 2 | 2x | 保存操作 timeout 设 60s | +| RSS Feed | 3s | 2 | 2x | 解析失败不重试,换 feed | + +--- + +## 9. 依赖安装 + +```bash +pip install requests feedparser beautifulsoup4 lxml trafilatura +``` + +| 包 | 用途 | +|---|------| +| `requests` | HTTP 请求 | +| `feedparser` | RSS/Atom 解析 | +| `beautifulsoup4` | HTML 解析与清理 | +| `lxml` | XML/HTML 解析后端 | +| `trafilatura` | 网页正文提取 | + +### 日志配置 + +**缓存策略:** 新闻与 RSS 数据有天然时效性,但短时间内重复请求同一 URL 纯属浪费且易触发封禁。建议使用共器 `cached_get`: + +```python +import sys +sys.path.insert(0, "/scripts") +from cached_get import cached_get + +# 用法与 requests.get 一致,多一个 ttl 参数和 cache_name 参数 +r = cached_get("https://news.google.com/rss/search", + params={"q": "AI"}, ttl=300, cache_name="news") +feed = feedparser.parse(r.text) +``` + +共器位于 `/scripts/cached_get.py`,特性: +- 原子写入(先书临时,成而后正名,断电不生残卷) +- 自动淘汰陈腐(>1 天自动删除,总量上限 200 条) +- 按 `cache_name` 隔离不同技能的缓存目录 + +**建议 TTL:** + +| 数据源 | TTL | 理由 | +|--------|-----|------| +| Google News RSS | 300s (5min) | 新闻更新快,但 5 分钟内不会大变 | +| Reddit 帖子列表 | 120s (2min) | 热帖排序变化较快 | +| RSS Feed | 600s (10min) | 多数站点更新频率较低 | +| Wayback Machine | 86400s (1day) | 归档数据不会变 | +| 付费墙尝试 | 3600s (1hr) | 短时间重复尝试无意义 | + +本技能代码中使用 `logging` 模块记录请求成功、降级触发与异常信息。调用前须先配置: + +```python +import logging + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(name)s: %(message)s", + datefmt="%H:%M:%S", +) +# 或为特定模块设更细粒度: +logging.getLogger("news_and_rss").setLevel(logging.DEBUG) +``` + +**日志级别约定:** + +| 级别 | 用途 | +|------|------| +| `DEBUG` | 完整请求 URL、响应状态码、解析字段详情 | +| `INFO` | 降级触发、数据源切换、结果数量 | +| `WARNING` | 重试中、速率限制命中、付费墙检测 | +| `ERROR` | 全部数据源失败、无法恢复的异常 | + +--- + +## 10. 与主技能的关系 + +本技能是 **web-browsing-manual v3.0** 的子技能(sub-skill)。父技能提供了完整的网页浏览策略(七层渐进策略、学术 API、搜索引擎、反检测技术等),本技能专注于其中的新闻获取领域。 + +- 需要通用网页抓取 → 参见 `web-browsing-manual` +- 需要社交媒体数据提取 → 参见 `social-media-extraction` +- 需要学术论文搜索 → 参见 `web-browsing-manual` 中的学术 API 章节 diff --git a/src/lingtai/tools/web_search/manual/reference/realtime-data.md b/src/lingtai/tools/web_search/manual/reference/realtime-data.md new file mode 100644 index 000000000..26ad22bfc --- /dev/null +++ b/src/lingtai/tools/web_search/manual/reference/realtime-data.md @@ -0,0 +1,548 @@ +# Real-Time Data + +> Part of the [web-browsing](../SKILL.md) skill. +> Financial data, weather, system status, Stack Exchange, Wikipedia. + +# realtime-data + +> **Real-time data from free APIs — stocks, weather, Q&A, encyclopedia, and more.** +> Part of the `web-browsing-manual` v3.0 skill family. + +--- + +## Decision Tree + +``` +Real-time data need ────────┐ + │ + ┌─────────┬──────────────┬───────────┬────────┐ + │ │ │ │ │ + Finance Weather Tech Q&A Facts/ News/ + /Stock /Climate Encyclo Status + │ │ │ │ │ + ▼ ▼ ▼ ▼ ▼ + yfinance Open-Meteo Stack Wikipedia Google + (free) (free) Exchange REST API News RSS + + others +``` + +--- + +## 1. Financial Data (yfinance) + +**When to use:** Stock prices, company info, historical data, financial news. +**Free:** ✅ | **Key:** Not needed | **Speed:** ~1-2s + +### Stock Information + +```python +import yfinance as yf + +def stock_info(ticker_symbol): + """Get comprehensive stock information. + + Args: + ticker_symbol: e.g. "AAPL", "GOOGL", "TSLA", "^GSPC" + Returns: dict with name, price, market cap, etc. + """ + ticker = yf.Ticker(ticker_symbol) + info = ticker.info + return { + "name": info.get("longName"), + "symbol": ticker_symbol, + "price": info.get("currentPrice"), + "previous_close": info.get("previousClose"), + "change_pct": info.get("regularMarketChangePercent"), + "market_cap": info.get("marketCap"), + "pe_ratio": info.get("trailingPE"), + "forward_pe": info.get("forwardPE"), + "52week_high": info.get("fiftyTwoWeekHigh"), + "52week_low": info.get("fiftyTwoWeekLow"), + "volume": info.get("volume"), + "avg_volume": info.get("averageVolume"), + "dividend_yield": info.get("dividendYield"), + "sector": info.get("sector"), + "industry": info.get("industry"), + "description": info.get("longBusinessSummary"), + "website": info.get("website"), + "employees": info.get("fullTimeEmployees"), + } +``` + +### Historical Data + +```python +def stock_history(ticker_symbol, period="1mo", interval="1d"): + """Get OHLCV historical data. + + period: 1d, 5d, 1mo, 3mo, 6mo, 1y, 2y, 5y, ytd, max + interval: 1m, 2m, 5m, 15m, 30m, 60m, 1h, 1d, 5d, 1wk, 1mo, 3mo + Note: 1m data only available for last 7 days + """ + ticker = yf.Ticker(ticker_symbol) + hist = ticker.history(period=period, interval=interval) + # Returns DataFrame with columns: Open, High, Low, Close, Volume + return hist + +def stock_history_csv(ticker_symbol, period="1y", filename=None): + """Get history as CSV-friendly records.""" + hist = stock_history(ticker_symbol, period=period) + return hist.reset_index().to_dict("records") +``` + +### Stock News & Recommendations + +```python +def stock_news(ticker_symbol, count=10): + """Get news related to a stock.""" + ticker = yf.Ticker(ticker_symbol) + return ticker.news[:count] + +def stock_recommendations(ticker_symbol): + """Get analyst recommendations.""" + ticker = yf.Ticker(ticker_symbol) + recs = ticker.recommendations + return recs.to_dict("records") if recs is not None else [] + +def stock_earnings(ticker_symbol): + """Get earnings data (quarterly).""" + ticker = yf.Ticker(ticker_symbol) + dates = ticker.earnings_dates + return dates.to_dict("records") if dates is not None else [] +``` + +### Multi-Stock Comparison + +```python +def compare_stocks(symbols, period="3mo"): + """Compare multiple stocks' performance over a period.""" + data = {} + for sym in symbols: + try: + ticker = yf.Ticker(sym) + hist = ticker.history(period=period) + if len(hist) > 0: + start_price = hist["Close"].iloc[0] + end_price = hist["Close"].iloc[-1] + data[sym] = { + "start": round(start_price, 2), + "end": round(end_price, 2), + "change_pct": round(((end_price / start_price) - 1) * 100, 2), + "high": round(hist["High"].max(), 2), + "low": round(hist["Low"].min(), 2), + } + except Exception: + data[sym] = {"error": "Failed to fetch"} + return data +``` + +**When NOT to use:** Need real-time tick data, options chains, or crypto (use `yf.Ticker("BTC-USD")` for crypto). +**Gotcha:** yfinance scrapes Yahoo Finance — may break if Yahoo changes their page structure. + +--- + +## 2. Weather (Open-Meteo) + +**When to use:** Weather forecasts, historical weather, climate data. +**Free:** ✅ | **Key:** Not needed | **Speed:** ~0.5s + +### Current Weather + Forecast + +```python +import requests + +def get_weather(latitude, longitude, days=7): + """Get weather forecast from Open-Meteo. + + Free, no API key, no registration. + Returns current weather + daily forecast. + """ + url = "https://api.open-meteo.com/v1/forecast" + params = { + "latitude": latitude, + "longitude": longitude, + "current_weather": True, + "daily": "temperature_2m_max,temperature_2m_min,precipitation_sum," + "windspeed_10m_max,weathercode", + "timezone": "auto", + "forecast_days": days, + } + r = requests.get(url, params=params, timeout=15) + r.raise_for_status() + data = r.json() + return { + "current": data.get("current_weather"), + "daily": data.get("daily"), + "timezone": data.get("timezone"), + } + +def get_weather_by_city(city_name, days=7): + """Convenience: city name → weather forecast.""" + coords = geocode(city_name) + if not coords: + return None + return get_weather(coords["lat"], coords["lon"], days=days) +``` + +### Geocoding (City → Coordinates) + +```python +def geocode(city_name): + """Convert city name to lat/lon via Open-Meteo geocoding.""" + url = "https://geocoding-api.open-meteo.com/v1/search" + params = {"name": city_name, "count": 1, "language": "en", "format": "json"} + r = requests.get(url, params=params, timeout=10) + results = r.json().get("results") + if results: + res = results[0] + return { + "lat": res["latitude"], + "lon": res["longitude"], + "name": res["name"], + "country": res.get("country"), + "admin1": res.get("admin1"), # State/province + } + return None +``` + +### Historical Weather + +```python +def historical_weather(latitude, longitude, start_date, end_date): + """Get historical weather data. + + Dates in YYYY-MM-DD format. Max range: past 80 years. + """ + url = "https://archive-api.open-meteo.com/v1/archive" + params = { + "latitude": latitude, + "longitude": longitude, + "start_date": start_date, + "end_date": end_date, + "daily": "temperature_2m_max,temperature_2m_min,precipitation_sum," + "windspeed_10m_max", + "timezone": "auto", + } + r = requests.get(url, params=params, timeout=15) + r.raise_for_status() + return r.json() +``` + +**When NOT to use:** Need severe weather alerts, radar data, or very specific hourly forecasts beyond what Open-Meteo provides. + +--- + +## 3. Stack Exchange API + +**When to use:** Technical Q&A, programming help, domain-specific knowledge. +**Free:** ✅ | **Key:** Not needed (300/day with key: 10000/day) | **Speed:** ~0.5s + +```python +def stackexchange_search(query, site="stackoverflow", limit=10, sort="relevance"): + """Search Stack Exchange sites. + + site: stackoverflow, serverfault, askubuntu, superuser, math, etc. + Full list: https://stackexchange.com/sites + sort: relevance, votes, newest, activity + """ + url = "https://api.stackexchange.com/2.3/search" + params = { + "intitle": query, + "site": site, + "pagesize": limit, + "sort": sort, + "order": "desc", + } + r = requests.get(url, params=params, timeout=15) + r.raise_for_status() + items = r.json().get("items", []) + return [{ + "title": i.get("title"), + "url": i.get("link"), + "score": i.get("score"), + "answers": i.get("answer_count"), + "is_answered": i.get("is_answered"), + "tags": i.get("tags", []), + "views": i.get("view_count"), + "created": i.get("creation_date"), + } for i in items] + +def stackexchange_answers(question_id, site="stackoverflow"): + """Get answers for a specific question (with body content).""" + url = f"https://api.stackexchange.com/2.3/questions/{question_id}/answers" + params = { + "site": site, + "order": "desc", + "sort": "votes", + "filter": "withbody", # Include full HTML body + } + r = requests.get(url, params=params, timeout=15) + items = r.json().get("items", []) + return [{ + "body_html": i.get("body"), + "score": i.get("score"), + "is_accepted": i.get("is_accepted"), + "author": i.get("owner", {}).get("display_name"), + } for i in items] + +def stackexchange_advanced_search(query, site="stackoverflow", tagged=None, + accepted=True, limit=10): + """Advanced search with filters. + + tagged: comma-separated tags, e.g. "python,pandas" + accepted: only questions with accepted answers + """ + url = "https://api.stackexchange.com/2.3/search/advanced" + params = { + "q": query, + "site": site, + "pagesize": limit, + "order": "desc", + "sort": "relevance", + "answers": 1 if accepted else 0, + } + if tagged: + params["tagged"] = tagged + r = requests.get(url, params=params, timeout=15) + return r.json().get("items", []) +``` + +**When NOT to use:** Need content from non-SE sites, or need real-time streaming answers. + +--- + +## 4. Wikipedia REST API + +**When to use:** Encyclopedia facts, definitions, summaries, images. +**Free:** ✅ | **Key:** Not needed | **Speed:** ~0.3s + +```python +def wikipedia_summary(title, lang="en"): + """Get a summary of a Wikipedia article. + + Returns: title, extract, thumbnail, coordinates (if applicable). + """ + url = f"https://{lang}.wikipedia.org/api/rest_v1/page/summary/{title}" + r = requests.get(url, timeout=15, + headers={"User-Agent": "LingTai/3.0"}) + if r.status_code == 404: + return None + r.raise_for_status() + return r.json() + +def wikipedia_full_text(title, lang="en"): + """Get full plain text of a Wikipedia article.""" + url = f"https://{lang}.wikipedia.org/api/rest_v1/page/plain/{title}" + r = requests.get(url, timeout=15, + headers={"User-Agent": "LingTai/3.0"}) + if r.status_code == 404: + return None + r.raise_for_status() + return r.json() + +def wikipedia_search(query, lang="en", limit=10): + """Search Wikipedia for articles matching a query.""" + url = f"https://{lang}.wikipedia.org/w/api.php" + params = { + "action": "query", + "list": "search", + "srsearch": query, + "format": "json", + "srlimit": limit, + } + r = requests.get(url, params=params, timeout=15) + return r.json()["query"]["search"] + +def wikipedia_related(title, lang="en", limit=10): + """Get related articles using morelike: search. + + NOTE: The REST API /page/related endpoint was decommissioned by Wikimedia + in Feb 2025 (Phabricator T376297). This uses MediaWiki Action API's + morelike: search as the recommended replacement. + """ + url = f"https://{lang}.wikipedia.org/w/api.php" + params = { + "action": "query", + "list": "search", + "srsearch": f"morelike:{title}", + "srnamespace": "0", + "srlimit": limit, + "format": "json", + } + r = requests.get(url, params=params, timeout=15, + headers={"User-Agent": "LingTai/3.0"}) + r.raise_for_status() + results = r.json().get("query", {}).get("search", []) + return [ + { + "title": item.get("title"), + "pageid": item.get("pageid"), + "snippet": item.get("snippet", "").replace( + '', "").replace("", ""), + } + for item in results + ] +``` + +**Multi-language support:** Change `lang` parameter: "en", "zh", "ja", "de", "fr", "es", "ru", etc. + +--- + +## 5. System Status Pages + +**When to use:** Check if a service is down, get incident reports. +**Speed:** ~1-5s + +```python +def check_statuspage(domain): + """Check Statuspage.io-powered status pages. + + Many SaaS companies use Statuspage.io with a standard JSON API. + E.g., github.statuspage.io, api.openai.com, etc. + """ + # Try common patterns + candidates = [ + f"https://{domain}.statuspage.io/api/v2/status.json", + f"https://status.{domain}/api/v2/status.json", + ] + for url in candidates: + try: + r = requests.get(url, timeout=10) + if r.status_code == 200: + data = r.json() + return { + "page": data.get("page", {}).get("name"), + "status": data.get("status", {}).get("indicator"), + "description": data.get("status", {}).get("description"), + } + except Exception: + continue + return {"status": "unknown", "error": "Could not find status page"} + +def check_statuspage_incidents(domain): + """Get active incidents from a Statuspage.io page.""" + candidates = [ + f"https://{domain}.statuspage.io/api/v2/incidents.json", + f"https://status.{domain}/api/v2/incidents.json", + ] + for url in candidates: + try: + r = requests.get(url, timeout=10) + if r.status_code == 200: + return r.json().get("incidents", []) + except Exception: + continue + return [] +``` + +--- + +## 6. News & Social Feeds (Quick Reference) + +For detailed usage, see the `news-and-rss` and `social-media-extraction` sub-skills. + +```python +# Google News RSS (free, no key) +def quick_news(query): + import feedparser + url = f"https://news.google.com/rss/search?q={query}&hl=en" + r = requests.get(url, timeout=15, headers={"User-Agent": "Mozilla/5.0"}) + feed = feedparser.parse(r.text) + return [{"title": e.title, "url": e.link, "date": e.get("published")} + for e in feed.entries[:10]] + +# Hacker News top stories +def hn_top(limit=10): + r = requests.get("https://hacker-news.firebaseio.com/v0/topstories.json", timeout=10) + ids = r.json()[:limit] + stories = [] + for id_ in ids: + item = requests.get(f"https://hacker-news.firebaseio.com/v0/item/{id_}.json", + timeout=10).json() + if item: + stories.append({"title": item.get("title"), "url": item.get("url"), + "score": item.get("score")}) + return stories + +# GitHub trending (via search API) +def github_trending(language=None, since="daily"): + url = "https://api.github.com/search/repositories" + query = "stars:>100" + if language: + query += f" language:{language}" + query += f" pushed:>{_since_date(since)}" + params = {"q": query, "sort": "stars", "order": "desc", "per_page": 10} + r = requests.get(url, params=params, timeout=15) + return r.json().get("items", []) + +def _since_date(since): + from datetime import datetime, timedelta + days = {"daily": 1, "weekly": 7, "monthly": 30} + return (datetime.now() - timedelta(days=days.get(since, 1))).strftime("%Y-%m-%d") +``` + +--- + +## Caching Strategy + +Real-time data changes frequently but doesn't need to be fetched every second. + +```python +import time, functools + +# Simple TTL cache +_cache = {} +_cache_ttl = {} + +def cached_fetch(key, fetch_fn, ttl_seconds=300): + """Fetch with TTL-based cache. + + ttl_seconds: 300 = 5min (good for stock prices) + 3600 = 1hr (good for weather) + 86400 = 1day (good for Wikipedia) + """ + now = time.time() + if key in _cache and (now - _cache_ttl.get(key, 0)) < ttl_seconds: + return _cache[key] + + result = fetch_fn() + _cache[key] = result + _cache_ttl[key] = now + return result + +# Usage +def get_stock_price_cached(symbol): + return cached_fetch(f"stock:{symbol}", lambda: stock_info(symbol), ttl_seconds=300) + +def get_weather_cached(city): + return cached_fetch(f"weather:{city}", lambda: get_weather_by_city(city), ttl_seconds=3600) +``` + +--- + +## Failure Modes & Fallback Table + +| Failure | Cause | Fallback | +|---------|-------|----------| +| yfinance returns empty | Yahoo Finance changed structure | Retry once, check ticker symbol | +| yfinance KeyError on info | Not all fields available for all stocks | Use `.get()` with defaults | +| Open-Meteo timeout | Network issues | Retry once, use cached data | +| Stack Exchange 429 | Rate limit exceeded (300/day without key) | Register for free key (10K/day) | +| Wikipedia 404 | Article doesn't exist | Try `wikipedia_search` to find correct title | +| Status page not found | Not using Statuspage.io | Try scraping the status page HTML | +| All data sources fail | Network outage | Return last cached result if available | + +--- + +## Dependencies + +```bash +pip install yfinance # Financial data +pip install requests # All HTTP APIs +pip install feedparser # RSS/Atom feeds +pip install beautifulsoup4 # HTML parsing (status pages) +``` + +--- + +*This sub-skill is part of `web-browsing-manual` v3.0. For news-specific workflows, see `news-and-rss`. For social media, see `social-media-extraction`.* diff --git a/src/lingtai/tools/web_search/manual/reference/routing-and-sites/SKILL.md b/src/lingtai/tools/web_search/manual/reference/routing-and-sites/SKILL.md new file mode 100644 index 000000000..fd425d511 --- /dev/null +++ b/src/lingtai/tools/web_search/manual/reference/routing-and-sites/SKILL.md @@ -0,0 +1,95 @@ +--- +name: web-browsing-routing-and-sites +description: > + Nested web-browsing reference for auto-tier decisions, per-site tier + recommendations, known limitations/gotchas, and real-time data endpoints. +version: 1.0.0 +last_changed_at: "2026-06-01T01:47:09-07:00" +maintenance: "If you find stale or incorrect information here, use the lingtai-issue-report skill to assemble evidence and obtain per-issue human consent before filing an issue. Never include secrets, credentials, tokens, or private paths." +--- + +# Web Browsing Routing and Site Reference + +Nested web-browsing reference. Open this when the auto-tier extractor misroutes a +site, when you know the site class, or when you need real-time data endpoints. + +## Auto-Tier Decision Tree + +The bundled `extract_page.py` script auto-selects the cheapest viable tier.完整逻辑见 `scripts/extract_page.py` 的 `auto_tier()` 函数。 + +**分层规则概要:** + +| URL 特征 | 分配 Tier | +|----------|----------| +| PDF (.pdf 后缀 或 /pdf/ 路径) | Tier 0 | +| 学术 API (arXiv, CrossRef, DOI, PubMed, etc.) | Tier 1 | +| 静态内容站 (Wikipedia, BBC, Stack Overflow, etc.) | Tier 1.5 | +| 需结构化提取 (GitHub, Reddit, Nature) | Tier 2 | +| 需 JS 渲染/反爬 (Scholar, Springer, Medium, Reuters) | Tier 3 | +| 其他 → 默认 Tier 1.5 (trafilatura) | Tier 1.5 | + +--- + +## Per-Site Tier Recommendations + +| Site | Recommended tier | Success rate | Notes | +|------|------------------|--------------|-------| +| arXiv abstract | Tier 1 | high | API call via `requests` | +| arXiv PDF | Tier 0 | high | curl -L + fitz | +| OpenAlex / CrossRef | Tier 1 | high | Fully free, most reliable | +| Unpaywall | Tier 1 | high | Finds OA PDF for any DOI | +| DBLP | Tier 1 | high | CS papers, conference proceedings | +| CORE | Tier 1 | high | OA full text (30M+ papers) | +| Europe PMC | Tier 1 | high | Biomedical + PMC full text | +| Papers With Code | Tier 1 | high | ML papers with code | +| Google Scholar list | Tier 2 | medium-high | curl + BS, needs clean IP | +| Nature.com | Tier 2/3 | medium | og meta cheap; full body needs JS | +| Springer paywalled | Tier 3 | low | Needs cookies/session | +| Medium / Substack | Tier 1.5 | high | trafilatura extracts clean text | +| Reddit | Tier 2 | high | `.json` API or old.reddit.com + BS | +| GitHub | Tier 2 | high | API or BS | +| Wikipedia | Tier 1 | high | REST API `/page/summary/{title}` | +| Hacker News | Tier 1 | high | Firebase API, fully free | +| Google News | Tier 2 | high | RSS feed, free, no key | +| Twitter/X | Tier 3 | low | Aggressive bot detection | +| LinkedIn | Tier 3 | low | Requires login + stealth | +| Any generic article | Tier 1.5 | high | trafilatura — your default | + +--- + +## Known Limitations & Gotchas + +Read this before fighting a site — most of these are paired with the per-site table above. + +1. **Major publishers (Wiley / Science / PNAS / Elsevier):** almost always return 403; APIs are the only practical route. Use Unpaywall to find OA versions. +2. **Nature.com:** do NOT use `networkidle` with Playwright — it will time out. Use `domcontentloaded`. +3. **Google Scholar:** rapid requests get IP-blocked; pace with `time.sleep(2)`. Better: use SerpAPI/Serper. +4. **Semantic Scholar API:** needs free API key for usable rate limits (otherwise 100 req/5min). +5. **PDF links on arXiv:** the abstract page does NOT contain a direct PDF link. Derive: `/pdf/{ID}.pdf`. +6. **Jina Reader:** 20 req/min free tier. For heavy use, get an API key. +7. **Reddit:** must include a descriptive User-Agent header. Rate limit: ~60 req/min. +8. **Medium paywall:** trafilatura often extracts full text even from paywalled articles. If not, try Jina Reader. +9. **DuckDuckGo search:** no API key needed but rate-limited. Use responsibly. +10. **CORE API:** requires free API key from https://core.ac.uk/services/api for reasonable limits. + +--- + +## Real-Time Data Quick Reference + +| Source | Method | Free? | Endpoint / Pattern | +|--------|--------|-------|-------------------| +| **Google News** | RSS | ✅ | `https://news.google.com/rss/search?q={query}` | +| **Reddit** | JSON API | ✅ | Append `.json` to any URL + User-Agent header | +| **Hacker News** | Firebase API | ✅ | `https://hacker-news.firebaseio.com/v0/topstories.json` | +| **GitHub** | REST API | ✅* | `https://api.github.com/search/repositories?q={q}&sort=stars` | +| **Stack Exchange** | API | ✅ | `https://api.stackexchange.com/2.3/search?intitle={q}&site=stackoverflow` | +| **Wikipedia** | REST API | ✅ | `https://en.wikipedia.org/api/rest_v1/page/summary/{title}` | +| **Wayback Machine** | API | ✅ | `https://archive.org/wayback/available?url={url}` | +| **Stock data** | yfinance | ✅ | `yf.Ticker("AAPL").history(period="1mo")` | +| **Weather** | Open-Meteo | ✅ | `https://api.open-meteo.com/v1/forecast?...` | + +→ Deep-dive: [reference/realtime-data.md](../realtime-data.md) +→ News/RSS guide: [reference/news-and-rss.md](../news-and-rss.md) +→ Social media: [reference/social-media.md](../social-media.md) + +--- diff --git a/src/lingtai/tools/web_search/manual/reference/search-strategies.md b/src/lingtai/tools/web_search/manual/reference/search-strategies.md new file mode 100644 index 000000000..65f077aa4 --- /dev/null +++ b/src/lingtai/tools/web_search/manual/reference/search-strategies.md @@ -0,0 +1,436 @@ +# Search Strategies + +> Part of the [web-browsing](../SKILL.md) skill. +> Decision trees, query optimization, engine comparison, and search+extract workflows. + +# search-strategies + +> **Choosing the right search engine and crafting the right query.** +> Part of the `web-browsing-manual` v3.0 skill family. + +--- + +## Decision Tree: Academic vs General Search + +``` +Search need arrives ────────┐ + │ + ┌────────────────────────┼────────────────────────────┐ + │ │ │ + Academic General Real-time + intent intent intent + │ │ │ + ▼ ▼ ▼ + Is it a known What quality? What source? + identifier? │ │ + │ ┌───────┼───────┐ ┌────┼────┐ + ▼ │ │ │ │ │ │ + DOI/arXiv Google Semantic Free & News Social Reddit + → Tier 1 quality meaning no-setup RSS media JSON + API route │ │ │ │ │ │ + Serper Exa Duck Google HN Reddit + /SerperAPI DuckGo News API JSON + /GoogleCS RSS + │ │ │ │ │ │ │ + └──────────┴───────┴───────┴────────┴────┴────┘ + │ + Return results + + optionally extract + content from top hits +``` + +### Quick Selection + +| Need | Engine | Why | +|------|--------|-----| +| Quick free search, no setup | DuckDuckGo | Zero config, decent results | +| AI agent workflow (search+answer) | Tavily | Returns answer + raw content | +| Find by meaning, not keywords | Exa | Neural/semantic search | +| Google-quality results | Serper | Google proxy, simple API | +| Privacy/independent index | Brave | Not Google/Bing | +| Unlimited self-hosted | SearXNG | Aggregates all engines | +| Academic papers | OpenAlex > Semantic Scholar > SerpAPI Scholar | See `academic-search-pipeline` | + +--- + +## Engine Reference: Complete Code + Params + +### 1. DuckDuckGo (Free, No Key) + +**Best for:** Quick searches, no setup needed. Good for general queries. +**Rate limit:** Implicit (be reasonable, add delays). **Key:** Not needed. + +```python +from duckduckgo_search import DDGS + +def search_ddg(query, max_results=10): + """Standard web search via DuckDuckGo.""" + with DDGS() as ddgs: + results = list(ddgs.text(query, max_results=max_results)) + return [{"title": r["title"], "url": r["href"], + "snippet": r["body"]} for r in results] + +def search_ddg_news(query, max_results=10): + """News search via DuckDuckGo.""" + with DDGS() as ddgs: + results = list(ddgs.news(query, max_results=max_results)) + return results + +def search_ddg_images(query, max_results=10): + """Image search via DuckDuckGo.""" + with DDGS() as ddgs: + results = list(ddgs.images(query, max_results=max_results)) + return results + +# Usage +results = search_ddg("transformer attention mechanism") +for r in results[:3]: + print(f"- {r['title']}: {r['url']}") +``` + +**When NOT to use:** Need Google-quality results, semantic search, or structured answers. + +### 2. Tavily (AI-Native Search) + +**Best for:** AI agent workflows — search + extract + answer in one call. +**Rate limit:** 1000/month free. **Key:** Required (`TAVILY_API_KEY`). + +```python +import os, requests + +def search_tavily(query, api_key=None, max_results=5, include_answer=True, + search_depth="advanced"): + """Tavily search — returns AI-generated answer + raw content. + + search_depth: "basic" (fast) or "advanced" (thorough) + include_raw_content: True to get full page text (expensive!) + """ + api_key = api_key or os.environ.get("TAVILY_API_KEY") + if not api_key: + raise ValueError("Need TAVILY_API_KEY") + + r = requests.post("https://api.tavily.com/search", json={ + "api_key": api_key, + "query": query, + "search_depth": search_depth, + "include_answer": include_answer, + "include_raw_content": True, # Get full page content + "max_results": max_results, + }, timeout=30) + r.raise_for_status() + data = r.json() + return { + "answer": data.get("answer"), # AI-generated summary + "results": data.get("results", []), + "count": len(data.get("results", [])), + } + +# Usage +result = search_tavily("what is retrieval-augmented generation?") +print(f"Answer: {result['answer']}") +``` + +**When NOT to use:** No API key available, or need pure keyword search without AI overhead. + +### 3. Exa (Neural / Semantic Search) + +**Best for:** Finding content by meaning. "Find papers about using transformers for code generation" works better than exact keyword matching. +**Rate limit:** 1000/month free. **Key:** Required (`EXA_API_KEY`). + +```python +def search_exa(query, api_key=None, num_results=5, search_type="auto", + include_domains=None, exclude_domains=None): + """Exa neural search — understands meaning, not just keywords. + + search_type: "neural" (meaning), "keyword" (exact), "auto" + include_domains/exclude_domains: filter by site + """ + api_key = api_key or os.environ.get("EXA_API_KEY") + if not api_key: + raise ValueError("Need EXA_API_KEY") + + payload = { + "query": query, + "type": search_type, + "numResults": num_results, + "contents": {"text": {"maxCharacters": 3000}}, # Include content! + } + if include_domains: + payload["includeDomains"] = include_domains + if exclude_domains: + payload["excludeDomains"] = exclude_domains + + r = requests.post("https://api.exa.ai/search", + headers={"x-api-key": api_key}, + json=payload, timeout=30) + r.raise_for_status() + data = r.json() + return { + "results": data.get("results", []), + "count": len(data.get("results", [])), + } + +# Usage: find similar pages by meaning +results = search_exa("papers on scaling laws for language models", + search_type="neural") +``` + +**When NOT to use:** Need exact keyword matching, no API key. + +### 4. Serper (Google Search Proxy) + +**Best for:** Google-quality results with a simple API. +**Rate limit:** 2500 free (one-time). **Key:** Required (`SERPER_API_KEY`). + +```python +def search_serper(query, api_key=None, gl="us", hl="en", num=10): + """Google search via Serper API.""" + api_key = api_key or os.environ.get("SERPER_API_KEY") + r = requests.post("https://google.serper.dev/search", + headers={"X-API-KEY": api_key}, + json={"q": query, "gl": gl, "hl": hl, "num": num}, + timeout=15) + r.raise_for_status() + return r.json() + # Returns: organic[], knowledgeGraph, answerBox, etc. + +# Scholar search +def search_serper_scholar(query, api_key=None): + """Google Scholar via Serper.""" + r = requests.post("https://google.serper.dev/scholar", + headers={"X-API-KEY": api_key or os.environ.get("SERPER_API_KEY")}, + json={"q": query}, timeout=15) + r.raise_for_status() + return r.json() +``` + +### 5. Brave Search + +**Best for:** Independent search index, privacy-focused. +**Rate limit:** 2000/month free. **Key:** Required (`BRAVE_API_KEY`). + +```python +def search_brave(query, api_key=None): + """Brave Search — independent index.""" + api_key = api_key or os.environ.get("BRAVE_API_KEY") + r = requests.get("https://api.search.brave.com/res/v1/web/search", + headers={"X-Subscription-Token": api_key}, + params={"q": query}, timeout=15) + r.raise_for_status() + return r.json() +``` + +### 6. SearXNG (Self-Hosted Meta-Search) + +**Best for:** Unlimited searches, privacy, aggregating multiple engines. +**Rate limit:** None (self-hosted). **Key:** Not needed. + +```bash +# Start SearXNG instance +docker run -d -p 8888:8080 searxng/searxng +``` + +```python +def search_searxng(query, instance="http://localhost:8888", categories="general"): + """SearXNG meta-search — aggregates Google, Bing, DDG, etc.""" + r = requests.get(f"{instance}/search", + params={"q": query, "format": "json", "categories": categories}, + timeout=15) + r.raise_for_status() + return r.json() +``` + +--- + +## Query Optimization + +### Operators (work on most engines) + +| Operator | Example | Effect | +|----------|---------|--------| +| `site:` | `"attention" site:arxiv.org` | Restrict to specific domain | +| `filetype:` | `"deep learning" filetype:pdf` | Find specific file types | +| `""` (quotes) | `"retrieval-augmented generation"` | Exact phrase match | +| `OR` | `"transformer" OR "attention mechanism"` | Either term | +| `-` (minus) | `"apple" -fruit` | Exclude term | +| `intitle:` | `intitle:"large language model"` | Must appear in title | +| `inurl:` | `inurl:blog "machine learning"` | Must appear in URL | +| `after:/before:` | `"AI" after:2024-01-01 before:2024-12-31` | Date range | + +### Query Reformulation Strategy + +```python +def reformulate_query(original_query, strategy="broader"): + """Generate alternative queries for better results.""" + queries = [original_query] + + if strategy == "broader": + # Remove specific terms, use synonyms + # e.g., "RoPE attention mechanism" → "rotary position embedding" + pass + elif strategy == "narrower": + # Add domain-specific terms + # e.g., "transformers" → "transformers NLP attention" + pass + elif strategy == "alternative": + # Use different phrasing + # e.g., "how to train LLMs" → "large language model training methods" + pass + + return queries +``` + +--- + +## Pagination Pattern + +```python +import time + +def paginated_search(search_fn, query, max_pages=3, per_page=10, delay=1.0): + """Generic pagination wrapper for any search API. + + Works with DDG, Tavily, Exa, etc. — any function that accepts + (query, offset, limit) or equivalent. + """ + all_results = [] + for page in range(max_pages): + results = search_fn(query, offset=page * per_page, limit=per_page) + if not results: + break + all_results.extend(results) + if len(results) < per_page: + break # No more results + if page < max_pages - 1: + time.sleep(delay) # Rate limit + return all_results +``` + +--- + +## Search + Extract Unified Workflow + +### Pattern 1: Tavily One-Shot + +Tavily can search AND extract content in a single API call: + +```python +def search_and_extract_tavily(query, api_key, max_results=5): + """Search + extract content in one call via Tavily.""" + r = requests.post("https://api.tavily.com/search", json={ + "api_key": api_key, + "query": query, + "search_depth": "advanced", + "include_answer": True, + "include_raw_content": True, + "max_results": max_results, + }, timeout=60) + data = r.json() + + return { + "answer": data.get("answer"), + "sources": [{ + "title": r.get("title"), + "url": r.get("url"), + "content": r.get("raw_content", "")[:3000], + "score": r.get("score"), + } for r in data.get("results", [])], + } +``` + +### Pattern 2: Exa with Content + +```python +def search_and_extract_exa(query, api_key, num_results=5): + """Search + get full text via Exa.""" + r = requests.post("https://api.exa.ai/search", + headers={"x-api-key": api_key}, + json={ + "query": query, + "type": "auto", + "numResults": num_results, + "contents": {"text": {"maxCharacters": 5000}}, + }, timeout=30) + data = r.json() + return data.get("results", []) +``` + +### Pattern 3: Search Then Extract + +```python +def search_then_extract(query, search_engine="ddg", max_extract=3): + """Search with any engine, then extract content from top results.""" + # Step 1: Search + if search_engine == "ddg": + results = search_ddg(query, max_results=max_extract) + else: + raise ValueError(f"Unsupported engine: {search_engine}") + + # Step 2: Extract content from top hits + import trafilatura + enriched = [] + for r in results: + try: + html = trafilatura.fetch_url(r["url"]) + if html: + text = trafilatura.extract(html) + r["extracted_text"] = text[:3000] if text else None + except Exception: + r["extracted_text"] = None + enriched.append(r) + time.sleep(0.5) + + return enriched +``` + +--- + +## Failure Modes & Fallback Table + +| Failure | Cause | Fallback | +|---------|-------|----------| +| DuckDuckGo 429 (rate limit) | Too many requests | Wait 60s, or switch to Tavily | +| Tavily quota exhausted | Monthly limit reached | Switch to DuckDuckGo | +| Exa returns empty | Too specific query | Switch to `type: "keyword"` mode | +| Serper 401 | Invalid/expired key | Check key, fallback to DDG | +| All search engines fail | Network/API outage | Use built-in `web_search` tool | +| No relevant results | Bad query | Reformulate query, try broader terms | +| Search results are spam | Low-quality sources | Add `site:` operator, use Exa for quality | + +--- + +## Engine Comparison Matrix + +| Feature | DDG | Tavily | Exa | Serper | Brave | SearXNG | +|---------|-----|--------|-----|--------|-------|---------| +| **Free tier** | ✅∞ | 1K/mo | 1K/mo | 2.5K once | 2K/mo | ✅∞ | +| **No key needed** | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | +| **Search quality** | Good | Good | Semantic | Google | Good | Best* | +| **Content extraction** | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | +| **AI answer** | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | +| **Self-hostable** | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| **Scholarly search** | ❌ | ❌ | Partial | ✅ | ❌ | ✅ | + +*SearXNG quality depends on aggregated engines. + +--- + +## Dependencies + +```bash +pip install duckduckgo-search # DuckDuckGo +pip install requests trafilatura # Search + extract pattern +pip install feedparser # RSS search (optional) +``` + +Tavily, Exa, Serper, Brave require API keys set as environment variables: +```bash +export TAVILY_API_KEY=tvly-... +export EXA_API_KEY=... +export SERPER_API_KEY=... +export BRAVE_API_KEY=... +``` + +--- + +*This sub-skill is part of `web-browsing-manual` v3.0. For academic search specifically, see `academic-search-pipeline`. For general web content extraction, see the parent skill.* diff --git a/src/lingtai/tools/web_search/manual/reference/social-media.md b/src/lingtai/tools/web_search/manual/reference/social-media.md new file mode 100644 index 000000000..7ff4b3054 --- /dev/null +++ b/src/lingtai/tools/web_search/manual/reference/social-media.md @@ -0,0 +1,1315 @@ +# Social Media Extraction + +> Part of the [web-browsing](../SKILL.md) skill. +> Reddit, Hacker News, Mastodon, X/Twitter, GitHub data extraction. + + +本技能是 `web-browsing-manual` 的子技能,专注于从社交媒体平台提取公开数据。涵盖 Reddit、Hacker News、Mastodon、X/Twitter 和 GitHub 的公开 API 与页面提取方法,包括速率限制、认证说明、完整代码片段、失败模式与伦理边界。 + +--- + +## 1. 社交媒体决策树 + +面对"我需要从社交媒体获取数据"这个需求时,按下图选择路径: + +``` +社交媒体数据 + │ + ├─ Reddit? ────────────→ JSON API(§2) + │ (帖子、评论、搜索) + │ 方法: URL 追加 .json + │ + ├─ Hacker News? ───────→ Firebase API(§3) + │ (技术讨论、Show HN) + │ 方法: firebaseio.com/v0/ + │ + ├─ Mastodon? ──────────→ 公开 API(§4) + │ (联邦宇宙帖子、搜索) + │ 方法: /api/v1/ 或 /api/v2/ + │ + ├─ X/Twitter? ─────────→ 公开页面提取(§5,受限) + │ (推文、用户信息) + │ 方法: Nitter 实例(不稳定) + │ 边界: 不绕过登录 + │ + └─ GitHub? ────────────→ REST/GraphQL API(§6) + (仓库、用户、代码搜索) + 方法: api.github.com +``` + +**快速选择指南:** + +| 平台 | 免费额度 | 认证需求 | 稳定性 | 数据丰富度 | +|------|----------|----------|--------|-----------| +| Reddit | 60 req/min | 无需(设 User-Agent 即可) | 高 | 高 | +| Hacker News | 无限制 | 无需 | 极高 | 中 | +| Mastodon | 因实例而异 | 无需(公开端点) | 高 | 高 | +| X/Twitter | 极有限 | 官方 API 需付费 | 低 | 低 | +| GitHub | 60/hr (无 token) | 无需(有 token 则 5000/hr) | 极高 | 极高 | + +--- + +## 2. Reddit JSON API 详细用法 + +Reddit 提供的 JSON API 是最易用的社交媒体接口之一——只需在标准 URL 后追加 `.json`。 + +### 2.1 子版块帖子 + +```python +import requests +import time + +# Rate limiting +_last_request_time = 0 +_MIN_INTERVAL = 2.0 + + +def _reddit_get(url, params=None): + """Rate-limited GET for Reddit API.""" + global _last_request_time + elapsed = time.time() - _last_request_time + if elapsed < _MIN_INTERVAL: + time.sleep(_MIN_INTERVAL - elapsed) + _last_request_time = time.time() + headers = {"User-Agent": "LingTai/1.0 (Social Media Extractor)"} + return requests.get(url, headers=headers, params=params, timeout=15) + + +def reddit_subreddit(subreddit, sort="hot", limit=25, timeframe="day"): + """Get posts from a subreddit. + + Args: + subreddit: Name without r/ prefix (e.g. "python"). + sort: "hot", "new", "top", "rising", "controversial". + limit: Max 100 posts per request. + timeframe: For "top" sort — "hour", "day", "week", "month", "year", "all". + + Returns: + List of post dicts. + """ + url = f"https://www.reddit.com/r/{subreddit}/{sort}.json" + params = {"limit": limit} + if sort in ("top", "controversial"): + params["t"] = timeframe + + r = _reddit_get(url, params=params) + r.raise_for_status() + + posts = [] + for child in r.json()["data"]["children"]: + d = child["data"] + posts.append({ + "id": d.get("name"), # fullname: t3_{id} + "title": d.get("title"), + "url": d.get("url"), + "score": d.get("score"), + "upvote_ratio": d.get("upvote_ratio"), + "num_comments": d.get("num_comments"), + "author": d.get("author"), + "created_utc": d.get("created_utc"), + "selftext": d.get("selftext", "")[:1000], + "is_self": d.get("is_self", False), + "link_flair_text": d.get("link_flair_text"), + "over_18": d.get("over_18", False), + "subreddit": d.get("subreddit"), + "permalink": d.get("permalink"), + }) + return posts + + +def reddit_subreddit_top_week(subreddit, limit=25): + """Shortcut: top posts this week.""" + return reddit_subreddit(subreddit, sort="top", limit=limit, timeframe="week") +``` + +### 2.2 帖子评论 + +```python +def reddit_comments(post_id, sort="best", limit=25, depth=None): + """Get comments for a Reddit post. + + Args: + post_id: Post ID (e.g. "abc123") or fullname (e.g. "t3_abc123"). + sort: "best", "top", "new", "controversial", "old", "qa". + limit: Max comments to return. + depth: Maximum comment nesting depth (optional). + + Returns: + List of comment dicts. + """ + # Strip fullname prefix if present + if post_id.startswith("t3_"): + post_id = post_id[3:] + + url = f"https://www.reddit.com/comments/{post_id}.json" + params = {"sort": sort, "limit": limit} + if depth is not None: + params["depth"] = depth + + r = _reddit_get(url, params=params) + r.raise_for_status() + + # Response is a 2-element array: [post_listing, comments_listing] + data = r.json() + comments = [] + + def extract_comments(children, level=0): + """Recursively extract comments.""" + for child in children: + if child["kind"] == "t1": # comment + d = child["data"] + comments.append({ + "id": d.get("id"), + "author": d.get("author"), + "body": d.get("body", "")[:2000], + "score": d.get("score"), + "created_utc": d.get("created_utc"), + "level": level, + "is_submitter": d.get("is_submitter", False), + }) + # Recurse into replies + replies = d.get("replies") + if isinstance(replies, dict): + extract_comments(replies["data"]["children"], level + 1) + + if len(data) > 1: + extract_comments(data[1]["data"]["children"]) + + return comments +``` + +### 2.3 搜索 + +```python +def reddit_search(query, sort="relevance", limit=25, subreddit=None, timeframe="all"): + """Search Reddit posts. + + Args: + query: Search string. Supports AND/OR/NOT and field search. + sort: "relevance", "hot", "top", "new", "comments". + limit: Max results (100 per request). + subreddit: Restrict to subreddit (optional). + timeframe: "hour", "day", "week", "month", "year", "all". + """ + if subreddit: + url = f"https://www.reddit.com/r/{subreddit}/search.json" + params = {"q": query, "sort": sort, "limit": limit, + "t": timeframe, "restrict_sr": "on"} + else: + url = "https://www.reddit.com/search.json" + params = {"q": query, "sort": sort, "limit": limit, "t": timeframe} + + r = _reddit_get(url, params=params) + r.raise_for_status() + + return [ + { + "title": c["data"]["title"], + "url": c["data"]["url"], + "score": c["data"]["score"], + "subreddit": c["data"].get("subreddit"), + "created_utc": c["data"].get("created_utc"), + "selftext": c["data"].get("selftext", "")[:500], + } + for c in r.json()["data"]["children"] + ] +``` + +### 2.4 用户信息 + +```python +def reddit_user(username, limit=25): + """Get a user's posts and comments. + + Args: + username: Reddit username (without u/ prefix). + limit: Max items to return. + """ + url = f"https://www.reddit.com/user/{username}.json" + params = {"limit": limit, "sort": "new"} + r = _reddit_get(url, params=params) + r.raise_for_status() + + items = [] + for child in r.json()["data"]["children"]: + d = child["data"] + items.append({ + "type": "post" if child["kind"] == "t3" else "comment", + "subreddit": d.get("subreddit"), + "title": d.get("title"), + "body": d.get("selftext") or d.get("body", "")[:500], + "score": d.get("score"), + "created_utc": d.get("created_utc"), + }) + return items +``` + +### 2.5 Reddit API 规则与边界 + +| 规则 | 详情 | +|------|------| +| 速率限制 | ~60 req/min(无 OAuth);如遇 429 则降速 | +| User-Agent | **必须设置**,否则可能被封禁。推荐格式: `App/Version (Purpose)` | +| 请求间隔 | 建议每次 ≥ 2 秒 | +| 分页 | 用 `after` / `before` 参数,值取自上次返回的 `data.after` | +| 登录 | **不要模拟登录或绕过认证**。如需更高限额,使用官方 OAuth 流程。 | +| 数据范围 | 只能获取公开子版块和数据。私有/quarantined 子版块无法访问。 | +| NSFW | `over_18: True` 的帖子在应用中应妥善处理。 | + +--- + +## 3. Hacker News Firebase API + +Hacker News (HN) 提供完全免费、无速率限制的 Firebase API。这是获取技术社区讨论的最佳数据源。 + +### 3.1 完整代码 + +```python +import requests + +HN_API = "https://hacker-news.firebaseio.com/v0" + + +def hn_get_item(item_id): + """Get a single item (story, comment, job, poll) by ID.""" + r = requests.get(f"{HN_API}/item/{item_id}.json", timeout=10) + r.raise_for_status() + return r.json() + + +def hn_top_stories(limit=30): + """Get current top stories.""" + r = requests.get(f"{HN_API}/topstories.json", timeout=15) + ids = r.json()[:limit] + return _hn_fetch_items(ids) + + +def hn_new_stories(limit=30): + """Get newest stories.""" + r = requests.get(f"{HN_API}/newstories.json", timeout=15) + ids = r.json()[:limit] + return _hn_fetch_items(ids) + + +def hn_best_stories(limit=30): + """Get best stories (highest scoring).""" + r = requests.get(f"{HN_API}/beststories.json", timeout=15) + ids = r.json()[:limit] + return _hn_fetch_items(ids) + + +def hn_ask_stories(limit=30): + """Get Ask HN stories.""" + r = requests.get(f"{HN_API}/askstories.json", timeout=15) + ids = r.json()[:limit] + return _hn_fetch_items(ids) + + +def hn_show_stories(limit=30): + """Get Show HN stories.""" + r = requests.get(f"{HN_API}/showstories.json", timeout=15) + ids = r.json()[:limit] + return _hn_fetch_items(ids) + + +def hn_job_stories(limit=30): + """Get job postings.""" + r = requests.get(f"{HN_API}/jobstories.json", timeout=15) + ids = r.json()[:limit] + return _hn_fetch_items(ids) + + +def _hn_fetch_items(ids): + """Fetch multiple items by ID list.""" + stories = [] + for id_ in ids: + item = hn_get_item(id_) + if item: + stories.append({ + "id": item.get("id"), + "type": item.get("type"), # story, comment, job, poll + "title": item.get("title"), + "url": item.get("url"), + "text": item.get("text"), # HTML for self-posts + "score": item.get("score"), + "by": item.get("by"), + "time": item.get("time"), # Unix timestamp + "descendants": item.get("descendants", 0), # comment count + "kids": item.get("kids", []), # direct comment IDs + }) + return stories +``` + +### 3.2 评论递归提取 + +```python +def hn_comments(story_id, max_depth=3, max_comments=50): + """Recursively fetch comments for a story. + + Args: + story_id: The story/item ID. + max_depth: Maximum nesting depth to follow. + max_comments: Total comments to collect (safety limit). + + Returns: + List of comment dicts with "level" field. + """ + collected = [] + + def _recurse(item_ids, depth=0): + if depth > max_depth or len(collected) >= max_comments: + return + for kid_id in item_ids: + if len(collected) >= max_comments: + return + item = hn_get_item(kid_id) + if not item or item.get("deleted") or item.get("dead"): + continue + if item.get("type") == "comment": + collected.append({ + "id": item["id"], + "by": item.get("by"), + "text": item.get("text", ""), # HTML + "time": item.get("time"), + "level": depth, + "parent": item.get("parent"), + }) + if item.get("kids"): + _recurse(item["kids"], depth + 1) + + story = hn_get_item(story_id) + if story and story.get("kids"): + _recurse(story["kids"]) + + return collected +``` + +### 3.3 用户信息 + +```python +def hn_user(username): + """Get Hacker News user profile.""" + r = requests.get(f"{HN_API}/user/{username}.json", timeout=10) + r.raise_for_status() + data = r.json() + if not data: + return None + return { + "id": data.get("id"), + "karma": data.get("karma"), + "about": data.get("about"), # HTML + "created": data.get("created"), # Unix timestamp + "submitted": data.get("submitted", [])[:100], # item IDs + } +``` + +### 3.4 HN API 特性 + +| 特性 | 详情 | +|------|------| +| 完全免费 | 无需 API Key | +| 无速率限制 | 但应合理使用,避免并发大量请求 | +| 数据格式 | JSON | +| 时间格式 | Unix 时间戳 | +| 评论 HTML | `text` 字段包含 HTML,需要自行清理 | +| 故事类型 | `story`, `comment`, `job`, `poll`, `pollopt` | +| 删除/死亡 | `deleted: true` 或 `dead: true` 表示已被删除/屏蔽 | + +--- + +## 4. Mastodon 公开 API + +Mastodon 是去中心化的社交网络,每个实例(instance)都有独立的 API。公开端点无需认证。 + +### 4.1 完整代码 + +```python +import requests + + +def mastodon_trends(instance="mastodon.social", limit=20): + """Get trending statuses from a Mastodon instance. + + Args: + instance: Domain of the Mastodon instance. + limit: Number of trending posts. + + Returns: + List of status dicts. + """ + url = f"https://{instance}/api/v1/trends/statuses" + params = {"limit": limit} + r = requests.get(url, params=params, timeout=15) + r.raise_for_status() + + statuses = [] + for s in r.json(): + statuses.append(_parse_mastodon_status(s)) + return statuses + + +def mastodon_trends_links(instance="mastodon.social", limit=20): + """Get trending links from a Mastodon instance.""" + url = f"https://{instance}/api/v1/trends/links" + params = {"limit": limit} + r = requests.get(url, params=params, timeout=15) + r.raise_for_status() + return r.json() + + +def mastodon_search(instance, query, search_type="statuses", limit=20): + """Search public content on a Mastodon instance. + + Args: + instance: Domain (e.g. "mastodon.social"). + query: Search query string. + search_type: "statuses", "accounts", or "hashtags". + limit: Max results. + """ + url = f"https://{instance}/api/v2/search" + params = {"q": query, "type": search_type, "limit": limit} + r = requests.get(url, params=params, timeout=15) + r.raise_for_status() + + data = r.json() + results = data.get(search_type, []) + if search_type == "statuses": + return [_parse_mastodon_status(s) for s in results] + return results + + +def mastodon_account_statuses(instance, account_id, limit=20): + """Get public statuses from a specific account. + + Args: + instance: Domain. + account_id: Numeric account ID (not username). + limit: Max statuses. + """ + url = f"https://{instance}/api/v1/accounts/{account_id}/statuses" + params = {"limit": limit, "exclude_replies": "true"} + r = requests.get(url, params=params, timeout=15) + r.raise_for_status() + return [_parse_mastodon_status(s) for s in r.json()] + + +def mastodon_hashtag(instance, tag, limit=20): + """Get recent statuses with a hashtag.""" + url = f"https://{instance}/api/v1/timelines/tag/{tag}" + params = {"limit": limit} + r = requests.get(url, params=params, timeout=15) + r.raise_for_status() + return [_parse_mastodon_status(s) for s in r.json()] + + +def mastodon_instance_info(instance): + """Get information about a Mastodon instance.""" + url = f"https://{instance}/api/v1/instance" + r = requests.get(url, timeout=15) + r.raise_for_status() + return r.json() + + +def _parse_mastodon_status(s): + """Parse a Mastodon status object into a cleaner dict.""" + account = s.get("account", {}) + return { + "id": s.get("id"), + "url": s.get("url"), + "content": s.get("content"), # HTML + "created_at": s.get("created_at"), + "reblogs_count": s.get("reblogs_count"), + "favourites_count": s.get("favourites_count"), + "replies_count": s.get("replies_count"), + "account": { + "username": account.get("username"), + "display_name": account.get("display_name"), + "url": account.get("url"), + "followers_count": account.get("followers_count"), + }, + "tags": [t.get("name") for t in s.get("tags", [])], + "sensitive": s.get("sensitive", False), + "media_attachments": [ + {"type": m.get("type"), "url": m.get("url")} + for m in s.get("media_attachments", []) + ], + } +``` + +### 4.2 常用实例 + +| 实例 | 特色 | +|------|------| +| mastodon.social | 最大的通用实例 | +| fosstodon.org | 开源技术社区 | +| hachyderm.io | 技术专业人士 | +| mas.to | 通用实例 | +| mstdn.jp | 日语社区 | + +### 4.3 Mastodon API 特性 + +| 特性 | 详情 | +|------|------| +| 去中心化 | 每个实例有独立 API,数据不互通 | +| 公开端点 | 无需认证即可访问公开内容 | +| 速率限制 | 因实例而异,通常 300 req/5min | +| 内容格式 | HTML(`content` 字段需自行清理标签) | +| 分页 | 使用 `Link` 响应头中的 `next` / `prev` URL | + +### 4.4 HTML 内容清理 + +```python +from bs4 import BeautifulSoup + + +def clean_mastodon_html(html_content): + """Remove HTML tags from Mastodon content.""" + if not html_content: + return "" + soup = BeautifulSoup(html_content, "lxml") + return soup.get_text(separator=" ", strip=True) +``` + +--- + +## 5. X/Twitter 公开页面(边界与限制) + +### 5.1 现状与限制 + +X(原 Twitter)的数据获取面临严重限制: + +| 方式 | 可行性 | 说明 | +|------|--------|------| +| 官方 API | 需付费 | Free tier 仅能发推,不能读取 | +| 公开页面抓取 | 极不稳定 | 页面严重依赖 JS 渲染,反爬严格 | +| Nitter 实例 | 不稳定 | 2024 年后大量实例关闭 | +| 第三方服务 | 可行但需评估 | 如 SocialData, Twitter API Pro | + +### 5.2 伦理边界 + +**严格遵守以下原则:** + +- ✅ 使用官方 API(如有付费订阅) +- ✅ 使用 Nitter 等公开替代前端(如可用) +- ✅ 提取 OG meta 标签(页面提供的公开元数据) +- ❌ 不尝试绕过登录 +- ❌ 不使用抓取工具破解 JS 渲染后的登录墙 +- ❌ 不使用被盗或泄露的 API 凭证 + +### 5.3 基本提取(如果页面可访问) + +```python +import requests +from bs4 import BeautifulSoup + + +def twitter_meta_extract(url): + """Extract metadata from a public Twitter/X page. + + This only works if the page is accessible without login. + Primarily extracts OpenGraph and Twitter Card meta tags. + + Args: + url: Twitter/X post URL. + + Returns: + Dict with available metadata, or None. + """ + headers = { + "User-Agent": "Mozilla/5.0 (compatible; LingTai/1.0)", + } + try: + r = requests.get(url, headers=headers, timeout=15) + if r.status_code != 200: + return None + + soup = BeautifulSoup(r.text, "lxml") + + # Extract OG meta tags + og = {} + for tag in soup.find_all("meta"): + prop = tag.get("property", "") + if prop.startswith("og:"): + og[prop[3:]] = tag.get("content", "") + + # Extract Twitter Card meta tags + twitter = {} + for tag in soup.find_all("meta"): + name = tag.get("name", "") + if name.startswith("twitter:"): + twitter[name[8:]] = tag.get("content", "") + + if not og and not twitter: + return None + + return { + "title": og.get("title") or twitter.get("title"), + "description": og.get("description") or twitter.get("description"), + "image": og.get("image"), + "url": og.get("url"), + "site_name": og.get("site_name"), + "card_type": twitter.get("card"), + } + except Exception: + return None + + +def nitter_extract(nitter_url): + """Extract content from a Nitter instance (if available). + + Nitter instances provide a lightweight, JS-free view of tweets. + However, many instances have shut down since 2024. + + Args: + nitter_url: Full Nitter URL (e.g. "https://nitter.net/user/status/123"). + + Returns: + Extracted text content, or None. + """ + headers = {"User-Agent": "Mozilla/5.0 (compatible; LingTai/1.0)"} + try: + r = requests.get(nitter_url, headers=headers, timeout=15) + if r.status_code != 200: + return None + + soup = BeautifulSoup(r.text, "lxml") + # Nitter uses specific CSS classes + tweet_content = soup.find("div", class_="tweet-content") + if tweet_content: + return tweet_content.get_text(strip=True) + + # Fallback: try the main content area + return soup.get_text(strip=True)[:1000] + except Exception: + return None +``` + +### 5.4 实用建议 + +```python +# Practical approach: use Jina Reader as fallback +def twitter_via_jina(url): + """Try to get tweet content via Jina Reader.""" + jina_url = f"https://r.jina.ai/{url}" + try: + r = requests.get(jina_url, timeout=30) + if r.status_code == 200 and len(r.text) > 50: + return r.text + except Exception: + pass + return None +``` + +> **底线:** 如果以上方法都无法获取内容,说明 X/Twitter 的数据确实不可公开获取。接受这个限制,不要进一步尝试。 + +--- + +## 6. GitHub 公开数据 + +GitHub 提供 REST API v3 和 GraphQL API v4,是获取代码仓库、用户、issue、PR 等数据的最佳途径。 + +### 6.1 仓库搜索 + +```python +import requests +import time + +# GitHub rate limiter +_gh_last_request = 0 +_GH_MIN_INTERVAL = 1.0 + + +def _github_get(url, params=None, token=None): + """Rate-limited GET for GitHub API.""" + global _gh_last_request + elapsed = time.time() - _gh_last_request + if elapsed < _GH_MIN_INTERVAL: + time.sleep(_GH_MIN_INTERVAL - elapsed) + _gh_last_request = time.time() + + headers = { + "Accept": "application/vnd.github.v3+json", + "User-Agent": "LingTai/1.0", + } + if token: + headers["Authorization"] = f"token {token}" + + r = requests.get(url, headers=headers, params=params, timeout=15) + r.raise_for_status() + return r + + +def github_search_repos(query, sort="stars", order="desc", per_page=10, token=None): + """Search GitHub repositories. + + Args: + query: Search query. Supports qualifiers like "language:python", "stars:>1000". + sort: "stars", "forks", "help-wanted-issues", "updated". + order: "desc" or "asc". + per_page: Max 100. + token: Optional GitHub personal access token. + """ + url = "https://api.github.com/search/repositories" + params = {"q": query, "sort": sort, "order": order, "per_page": per_page} + r = _github_get(url, params=params, token=token) + + repos = [] + for item in r.json().get("items", []): + repos.append({ + "full_name": item.get("full_name"), + "description": item.get("description"), + "url": item.get("html_url"), + "stars": item.get("stargazers_count"), + "forks": item.get("forks_count"), + "language": item.get("language"), + "topics": item.get("topics", []), + "created_at": item.get("created_at"), + "updated_at": item.get("updated_at"), + "license": item.get("license", {}).get("spdx_id") if item.get("license") else None, + "open_issues": item.get("open_issues_count"), + }) + return repos + + +def github_search_code(query, per_page=10, token=None): + """Search code on GitHub. + + Note: Code search requires authentication (token required). + """ + url = "https://api.github.com/search/code" + params = {"q": query, "per_page": per_page} + r = _github_get(url, params=params, token=token) + return r.json().get("items", []) +``` + +### 6.2 仓库详情与 README + +```python +def github_repo_info(owner, repo, token=None): + """Get detailed repository information.""" + url = f"https://api.github.com/repos/{owner}/{repo}" + r = _github_get(url, token=token) + data = r.json() + return { + "full_name": data.get("full_name"), + "description": data.get("description"), + "url": data.get("html_url"), + "stars": data.get("stargazers_count"), + "forks": data.get("forks_count"), + "watchers": data.get("watchers_count"), + "language": data.get("language"), + "topics": data.get("topics", []), + "license": data.get("license", {}).get("spdx_id") if data.get("license") else None, + "default_branch": data.get("default_branch"), + "created_at": data.get("created_at"), + "updated_at": data.get("updated_at"), + "pushed_at": data.get("pushed_at"), + "size": data.get("size"), # KB + "open_issues": data.get("open_issues_count"), + "network_count": data.get("network_count"), + "subscribers_count": data.get("subscribers_count"), + } + + +def github_readme(owner, repo, token=None): + """Get repository README content. + + Returns raw text of the README file (README.md, README.rst, etc.) + """ + url = f"https://api.github.com/repos/{owner}/{repo}/readme" + headers = { + "Accept": "application/vnd.github.raw", + "User-Agent": "LingTai/1.0", + } + if token: + headers["Authorization"] = f"token {token}" + + r = requests.get(url, headers=headers, timeout=15) + if r.status_code == 200: + return r.text + return None + + +def github_repo_contents(owner, repo, path="", token=None): + """List repository directory contents.""" + url = f"https://api.github.com/repos/{owner}/{repo}/contents/{path}" + r = _github_get(url, token=token) + return [ + { + "name": item.get("name"), + "path": item.get("path"), + "type": item.get("type"), # "file", "dir", "symlink" + "size": item.get("size"), + "url": item.get("html_url"), + } + for item in r.json() + ] +``` + +### 6.3 Issue 和 Pull Request + +```python +def github_issues(owner, repo, state="open", sort="created", direction="desc", + per_page=20, token=None): + """List repository issues. + + Args: + state: "open", "closed", "all". + sort: "created", "updated", "comments". + direction: "desc" or "asc". + """ + url = f"https://api.github.com/repos/{owner}/{repo}/issues" + params = {"state": state, "sort": sort, "direction": direction, "per_page": per_page} + r = _github_get(url, params=params, token=token) + + issues = [] + for item in r.json(): + # GitHub API returns PRs in the issues endpoint too + is_pr = "pull_request" in item + issues.append({ + "number": item.get("number"), + "title": item.get("title"), + "state": item.get("state"), + "is_pr": is_pr, + "user": item.get("user", {}).get("login"), + "labels": [l.get("name") for l in item.get("labels", [])], + "created_at": item.get("created_at"), + "updated_at": item.get("updated_at"), + "comments": item.get("comments"), + "body": (item.get("body") or "")[:1000], + "url": item.get("html_url"), + }) + return issues + + +def github_pull_requests(owner, repo, state="open", sort="created", direction="desc", + per_page=20, token=None): + """List repository pull requests.""" + url = f"https://api.github.com/repos/{owner}/{repo}/pulls" + params = {"state": state, "sort": sort, "direction": direction, "per_page": per_page} + r = _github_get(url, params=params, token=token) + + return [ + { + "number": pr.get("number"), + "title": pr.get("title"), + "state": pr.get("state"), + "user": pr.get("user", {}).get("login"), + "merged": pr.get("merged", False), + "created_at": pr.get("created_at"), + "updated_at": pr.get("updated_at"), + "additions": pr.get("additions"), + "deletions": pr.get("deletions"), + "changed_files": pr.get("changed_files"), + "url": pr.get("html_url"), + } + for pr in r.json() + ] +``` + +### 6.4 用户信息 + +```python +def github_user(username, token=None): + """Get GitHub user profile.""" + url = f"https://api.github.com/users/{username}" + r = _github_get(url, token=token) + data = r.json() + return { + "login": data.get("login"), + "name": data.get("name"), + "company": data.get("company"), + "blog": data.get("blog"), + "location": data.get("location"), + "bio": data.get("bio"), + "public_repos": data.get("public_repos"), + "public_gists": data.get("public_gists"), + "followers": data.get("followers"), + "following": data.get("following"), + "created_at": data.get("created_at"), + "url": data.get("html_url"), + } +``` + +### 6.5 GitHub API 速率限制 + +| 认证状态 | 搜索 API | 普通 API | +|----------|----------|----------| +| 无 Token | 10 req/min | 60 req/hr | +| 有 Token | 30 req/min | 5000 req/hr | + +**检查剩余配额:** + +```python +def github_rate_limit(token=None): + """Check current GitHub API rate limit status.""" + url = "https://api.github.com/rate_limit" + headers = {"User-Agent": "LingTai/1.0"} + if token: + headers["Authorization"] = f"token {token}" + r = requests.get(url, headers=headers, timeout=15) + data = r.json() + return { + "core": data["resources"]["core"], + "search": data["resources"]["search"], + } +``` + +**关键 Accept Header:** + +``` +application/vnd.github.v3+json # 标准 JSON 响应 +application/vnd.github.raw # README 原始内容 +application/vnd.github.html # HTML 渲染的 Markdown +application/vnd.github.diff # diff 格式 +application/vnd.github.patch # patch 格式 +``` + +--- + +## 7. 失败模式与平台特殊注意 + +### 7.1 Reddit + +| 症状 | 原因 | 解决方案 | +|------|------|----------| +| HTTP 429 | 速率限制 | 等待 > 2s,确认 User-Agent 已设 | +| HTTP 403 | 子版块私有/封禁 | 确认子版块公开存在 | +| 返回 HTML 而非 JSON | URL 格式错误 | 确认 URL 以 `.json` 结尾 | +| 数据不完整 | Reddit 截断长内容 | 使用 `after` 分页获取更多 | +| HTTP 200 但 `children` 为空 | 子版块不存在或无帖子 | Reddit 对不存在的子版块也返回 200+空列表;检查 `data.dist` 为 0 则可能是无效子版块 | +| 评论区为空 | 帖子无评论或被锁 | 检查 `locked` 字段 | + +**特殊注意:** +- 子版块名称大小写不敏感,但建议用小写。 +- `selftext` 可能包含 Markdown,需要渲染或转义。 +- `over_18` 标记的帖子应做适当处理。 + +### 7.2 Hacker News + +| 症状 | 原因 | 解决方案 | +|------|------|----------| +| 返回 `null` | item ID 无效或已删除 | 跳过 `null` 结果 | +| 评论数过多 | 递归过深 | 设 `max_depth` 和 `max_comments` | +| 网络超时 | Firebase 响应慢 | 增大 timeout,重试 | +| 重复获取同一 item | 列表中有重复 ID | 去重处理 | + +**特殊注意:** +- `text` 字段是 HTML 格式,需要清理。 +- `url` 为 `None` 表示自帖(Ask HN 等),内容在 `text` 中。 +- 时间是 Unix 时间戳,需要转换。 +- `deleted` 和 `dead` 字段标记已删除或被屏蔽的内容。 + +### 7.3 Mastodon + +| 症状 | 原因 | 解决方案 | +|------|------|----------| +| HTTP 404 | 实例不存在或 API 版本不同 | 尝试其他实例 | +| HTTP 429 | 实例速率限制 | 降低请求频率,换实例 | +| 返回空列表 | 搜索无结果 | 扩大搜索范围,换关键词 | +| HTML 标签未清理 | `content` 是 HTML | 使用 BeautifulSoup 清理 | +| 跨实例内容不可见 | 去中心化限制 | 尝试目标用户所在的实例 | + +**特殊注意:** +- 不同实例的速率限制不同,`/api/v1/instance` 可查看实例规则。 +- 帖子 `content` 是 HTML,始终需要清理。 +- `account_id` 是数字,不是用户名(格式 `@user@instance`)。 +- 搜索端点 `/api/v2/search` 的结果取决于实例已知的联邦内容。 + +### 7.4 X/Twitter + +| 症状 | 原因 | 解决方案 | +|------|------|----------| +| 页面重定向到登录 | 需要登录 | 接受限制,尝试 meta 提取 | +| Nitter 502/503 | 实例已关闭 | 尝试其他实例或放弃 | +| Jina Reader 返回登录页 | 被检测 | 放弃,接受不可获取 | +| OG meta 为空 | 页面未正确渲染 | 无法获取 | + +**特殊注意:** +- **X/Twitter 数据获取是目前最困难的。** 官方 API 需付费(最低 $100/月)。 +- Nitter 实例大量关闭,不可依赖。 +- 唯一可靠的免费方式是通过搜索引擎(Google `site:twitter.com`)获取缓存片段。 +- **伦理底线:绝不尝试绕过登录验证。** + +### 7.5 GitHub + +| 症状 | 原因 | 解决方案 | +|------|------|----------| +| HTTP 403 + rate limit | 配额用尽 | 等待重置或使用 Token | +| HTTP 404 | 仓库不存在或私有 | 确认仓库名拼写 | +| 搜索无结果 | 查询语法错误 | 检查 qualifier 格式 | +| README 返回 404 | 仓库无 README | 返回 None,正常处理 | +| 大仓库内容截断 | API 分页 | 使用 page 参数 | + +**特殊注意:** +- 搜索 API 限额(10/min 无 Token)远低于普通 API(60/hr 无 Token)。 +- `Accept` header 控制返回格式,获取 README 用 `application/vnd.github.raw`。 +- 仓库的 `size` 单位是 KB。 +- Issue API 也包含 PR,用 `pull_request` 字段区分。 + +--- + +## 8. 重试退避与降级策略 + +外域接口诡变无常——速率越限、连接超时、令牌见夺皆为常态。本节提供系统性的重试、退避与降级方案。 + +### 8.1 通用重试装饰器 + +```python +import requests +import time +import logging + +logger = logging.getLogger(__name__) + + +def retry_with_backoff(max_retries=3, base_delay=2.0, backoff_factor=2.0, + retryable_statuses=(429, 500, 502, 503, 504)): + """Decorator: retry a function with exponential backoff. + + Args: + max_retries: Maximum retry attempts (0 = no retry). + base_delay: Initial delay in seconds. + backoff_factor: Multiplier for each subsequent delay. + retryable_statuses: HTTP status codes that trigger a retry. + """ + def decorator(func): + def wrapper(*args, **kwargs): + last_exception = None + for attempt in range(max_retries + 1): + try: + result = func(*args, **kwargs) + if isinstance(result, requests.Response): + if result.status_code in retryable_statuses: + retry_after = result.headers.get("Retry-After") + delay = float(retry_after) if retry_after else base_delay * (backoff_factor ** attempt) + logger.warning( + f"HTTP {result.status_code} on attempt {attempt + 1}/{max_retries + 1}, " + f"retrying in {delay:.1f}s" + ) + time.sleep(delay) + continue + result.raise_for_status() + return result + except (requests.Timeout, requests.ConnectionError) as e: + last_exception = e + if attempt < max_retries: + delay = base_delay * (backoff_factor ** attempt) + logger.warning( + f"Network error on attempt {attempt + 1}/{max_retries + 1}: {e}, " + f"retrying in {delay:.1f}s" + ) + time.sleep(delay) + else: + raise + raise last_exception + return wrapper + return decorator +``` + +### 8.2 社交媒体降级链 + +当一个平台失败时,自动降级到备选平台获取相似数据: + +```python +def fetch_tech_discussion(query, max_results=10): + """Fetch tech community discussion with fallback across platforms. + + Priority chain: + 1. Reddit (r/technology + search) + 2. Hacker News (search via Algolia) + 3. Mastodon (search on mastodon.social) + + Returns: + List of dicts with title, url, score, source_platform. + """ + results = [] + + # ── Tier 1: Reddit ─────────────────────────────────────────── + try: + posts = reddit_search(query, sort="new", limit=max_results) + for p in posts: + results.append({ + "title": p["title"], + "url": p["url"], + "score": p.get("score"), + "comments": p.get("num_comments"), + "source_platform": "reddit", + "subreddit": p.get("subreddit"), + }) + if results: + return results[:max_results] + except Exception as e: + logger.info(f"Reddit failed, falling back: {e}") + + # ── Tier 2: Hacker News (via Algolia search API) ───────────── + try: + algolia_url = "https://hn.algolia.com/api/v1/search" + params = {"query": query, "tags": "story", "hitsPerPage": max_results} + r = requests.get(algolia_url, params=params, timeout=15) + r.raise_for_status() + for hit in r.json().get("hits", []): + results.append({ + "title": hit.get("title"), + "url": hit.get("url") or f"https://news.ycombinator.com/item?id={hit['objectID']}", + "score": hit.get("points"), + "comments": hit.get("num_comments"), + "source_platform": "hackernews", + }) + if results: + return results[:max_results] + except Exception as e: + logger.info(f"Hacker News failed, falling back: {e}") + + # ── Tier 3: Mastodon ───────────────────────────────────────── + try: + statuses = mastodon_search("mastodon.social", query, + search_type="statuses", limit=max_results) + for s in statuses: + from bs4 import BeautifulSoup + text = BeautifulSoup(s.get("content", ""), "lxml").get_text(strip=True) + results.append({ + "title": text[:100], + "url": s.get("url"), + "score": s.get("favourites_count", 0), + "source_platform": "mastodon", + }) + if results: + return results[:max_results] + except Exception as e: + logger.info(f"Mastodon failed, all sources exhausted: {e}") + + return [] +``` + +### 8.3 各平台退避参数 + +| 平台 | 首次失败延迟 | 最大重试 | 退避因子 | 429 特殊处理 | +|------|-------------|----------|---------|-------------| +| Reddit | 2s | 3 | 3x | 遵守 `Retry-After` header | +| Hacker News | 1s | 3 | 2x | 无速率限制,但 Firebase 偶尔超时 | +| Mastodon | 2s | 2 | 2x | 因实例而异,检查 `X-RateLimit-Remaining` | +| GitHub | 2s | 3 | 2x | 检查 `X-RateLimit-Remaining` header | +| X/Twitter | 5s | 1 | — | 基本不重试,直接降级 | + +### 8.4 GitHub 配额耗尽降级 + +```python +def github_with_fallback(query, per_page=10, token=None): + """Search GitHub with graceful degradation on rate limit. + + If rate limited on search API, falls back to: + 1. Google site:github.com search + 2. Return cached partial results + """ + # Check remaining quota first + limits = github_rate_limit(token) + search_remaining = limits["search"].get("remaining", 0) + + if search_remaining > 0: + try: + return github_search_repos(query, per_page=per_page, token=token) + except requests.HTTPError: + pass + + # Fallback: DuckDuckGo site search + logger.info("GitHub search rate limited, falling back to web search") + try: + ddg_url = "https://html.duckduckgo.com/html/" + params = {"q": f"{query} site:github.com"} + r = requests.get(ddg_url, params=params, timeout=15, + headers={"User-Agent": "Mozilla/5.0"}) + from bs4 import BeautifulSoup + soup = BeautifulSoup(r.text, "lxml") + results = [] + for item in soup.find_all("div", class_="result"): + title_el = item.find("a", class_="result__a") + if title_el: + results.append({ + "title": title_el.get_text(strip=True), + "url": title_el.get("href"), + "source": "duckduckgo_fallback", + }) + return results[:per_page] + except Exception: + return [] +``` + +--- + +## 9. 依赖安装 + +```bash +pip install requests beautifulsoup4 lxml +``` + +| 包 | 用途 | +|---|------| +| `requests` | HTTP 请求(所有平台共用) | +| `beautifulsoup4` | HTML 解析(Mastodon、Twitter、HN 评论) | +| `lxml` | 解析后端 | + +### 日志配置 + +**缓存策略:** 社交媒体 API 调用频繁极易触发封禁。建议使用共器 `cached_get`: + +```python +import sys +sys.path.insert(0, "/scripts") +from cached_get import cached_get + +# 用法与 requests.get 一致,多一个 ttl 参数和 cache_name 参数 +r = cached_get("https://www.reddit.com/r/python/hot.json", + headers={"User-Agent": "LingTai/1.0"}, ttl=120, cache_name="social") +posts = r.json()["data"]["children"] +``` + +共器位于 `/scripts/cached_get.py`,特性: +- 原子写入(先书临时,成而后正名,断电不生残卷) +- 自动淘汰陈腐(>1 天自动删除,总量上限 200 条) +- 按 `cache_name` 隔离不同技能的缓存目录 + +**建议 TTL:** + +| 平台 | TTL | 理由 | +|------|-----|------| +| Reddit 帖子列表 | 120s | 排序变化快 | +| Reddit 搜索 | 300s | 搜索结果相对稳定 | +| Hacker News | 60s | 实时讨论,变化极快 | +| Mastodon | 300s | 趋势和搜索结果较稳定 | +| GitHub 仓库搜索 | 3600s | 仓库数据变化慢 | +| GitHub Rate Limit | 60s | 仅用于检查配额 | + +本技能代码中使用 `logging` 模块记录请求成败、降级触发与异常信息。调用前须先配置: + +```python +import logging + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(name)s: %(message)s", + datefmt="%H:%M:%S", +) +# 或为特定模块设更细粒度: +logging.getLogger("social_media").setLevel(logging.DEBUG) +``` + +**日志级别约定:** + +| 级别 | 用途 | +|------|------| +| `DEBUG` | 完整请求 URL、响应状态码、解析字段详情 | +| `INFO` | 降级触发、平台切换、结果数量 | +| `WARNING` | 重试中、速率限制命中(含剩余配额) | +| `ERROR` | 全部平台失败、GitHub 配额耗尽、无法恢复的异常 | + +--- + +## 10. 与主技能的关系 + +本技能是 **web-browsing-manual v3.0** 的子技能(sub-skill)。父技能提供了完整的网页浏览策略(七层渐进策略、学术 API、搜索引擎、反检测技术等),本技能专注于其中的社交媒体数据提取领域。 + +- 需要通用网页抓取 → 参见 `web-browsing-manual` +- 需要新闻获取和 RSS 处理 → 参见 `news-and-rss` +- 需要学术论文搜索 → 参见 `web-browsing-manual` 中的学术 API 章节 +- 需要反检测和代理策略 → 参见 `web-browsing-manual` 中的反检测章节 diff --git a/src/lingtai/tools/web_search/manual/reference/stealth.md b/src/lingtai/tools/web_search/manual/reference/stealth.md new file mode 100644 index 000000000..3113679e8 --- /dev/null +++ b/src/lingtai/tools/web_search/manual/reference/stealth.md @@ -0,0 +1,608 @@ +# Stealth Browsing & Anti-Detection + +> Part of the [web-browsing](../SKILL.md) skill. +> Browser fingerprinting, User-Agent rotation, proxy strategies, CAPTCHA handling. + +# stealth-browsing + +> **Evading detection while browsing — from simple UA rotation to full stealth mode.** +> Part of the `web-browsing-manual` v3.0 skill family. + +--- + +## How Websites Detect Bots + +Understanding detection methods helps you choose the right countermeasure. + +### 1. Browser Fingerprinting + +Websites build a unique fingerprint from: +- **Canvas/WebGL:** GPU rendering differences create unique hashes +- **Audio context:** Audio processing differences +- **Font enumeration:** Installed fonts list +- **Screen:** Resolution + color depth +- **Hardware:** CPU cores, device memory +- **Platform:** OS + browser version consistency + +### 2. WebDriver Detection + +Automated browsers leak signals: +- `navigator.webdriver === true` (Playwright/Selenium) +- Missing `window.chrome` object in headless +- CDP (Chrome DevTools Protocol) detection +- `$cdc_` variable injection by ChromeDriver +- Missing plugins/mimeTypes + +### 3. TLS/JA3 Fingerprinting + +- Python `requests` has a distinctive TLS fingerprint +- Headless browsers may differ from real ones + +### 4. Behavioral Analysis + +- Mouse movement patterns (too perfect = bot) +- No scroll behavior +- Click timing (instant = bot) +- Request frequency patterns + +### 5. IP/Network Level + +- Datacenter IP detection +- Proxy detection headers +- Rate limiting per IP +- Geo-inconsistency + +--- + +## playwright-stealth: Primary Defense + +**When to use:** Default for Tier 3 (JS-rendered pages). Patches most detection vectors. +**Installed:** ✅ in LingTai environment (`playwright-stealth` 2.0.3) + +> **NOTE:** `playwright-stealth` v2.0.3+ removed `stealth_sync()`. Use `Stealth().use_sync(page)` instead. All code blocks below use a compatibility shim (`_apply_stealth`) that works with both v1 and v2. + +### Basic Usage + +```python +from playwright.sync_api import sync_playwright + +# playwright-stealth v2.0.3+ compatibility +try: + from playwright_stealth import Stealth + _apply_stealth = lambda page: Stealth().use_sync(page) +except ImportError: + from playwright_stealth import stealth_sync + _apply_stealth = lambda page: stealth_sync(page) + +def stealth_fetch(url, wait_until="domcontentloaded", timeout=30000, wait_seconds=3): + """Fetch a page with full stealth patches applied. + + _apply_stealth patches: navigator.webdriver, chrome object, permissions, + plugins, languages, WebGL vendor, and more. + """ + with sync_playwright() as p: + browser = p.chromium.launch(headless=True) + page = browser.new_page() + _apply_stealth(page) # Apply all stealth patches + + page.goto(url, wait_until=wait_until, timeout=timeout) + page.wait_for_timeout(wait_seconds * 1000) + + content = page.inner_text("body") + html = page.content() + title = page.title() + final_url = page.url + + browser.close() + return { + "url": final_url, "title": title, + "body": content, "html": html, + } +``` + +### ⚠️ Nature/Springer Gotcha + +**Do NOT use `wait_until="networkidle"` for Nature or Springer pages.** They have long-running connections that cause timeouts. Use `"domcontentloaded"` instead. + +--- + +## Manual Stealth Overrides + +For sites that detect `playwright-stealth`, apply additional overrides: + +### Override navigator.webdriver + +```python +page.add_init_script(""" + Object.defineProperty(navigator, 'webdriver', { + get: () => undefined + }); +""") +``` + +### Override chrome runtime + +```python +page.add_init_script(""" + window.chrome = { + runtime: {}, + loadTimes: function() {}, + csi: function() {}, + app: {} + }; +""") +``` + +### Override permissions API + +```python +page.add_init_script(""" + const originalQuery = window.navigator.permissions.query; + window.navigator.permissions.query = (parameters) => + parameters.name === 'notifications' + ? Promise.resolve({ state: Notification.permission }) + : originalQuery(parameters); +""") +``` + +### Override plugins + +```python +page.add_init_script(""" + Object.defineProperty(navigator, 'plugins', { + get: () => [1, 2, 3, 4, 5], + }); +""") +``` + +### Override languages + +```python +page.add_init_script(""" + Object.defineProperty(navigator, 'languages', { + get: () => ['en-US', 'en'], + }); +""") +``` + +### Full Stealth Page (All Overrides) + +```python +def create_stealth_page(browser): + """Create a fully stealthed page with all overrides.""" + page = browser.new_page() + _apply_stealth(page) + + # Additional overrides beyond stealth_sync + page.add_init_script(""" + // WebGL vendor/renderer override + const getParameter = WebGLRenderingContext.prototype.getParameter; + WebGLRenderingContext.prototype.getParameter = function(param) { + if (param === 37445) return 'Intel Inc.'; + if (param === 37446) return 'Intel Iris OpenGL Engine'; + return getParameter.call(this, param); + }; + + // Hardware concurrency override + Object.defineProperty(navigator, 'hardwareConcurrency', { + get: () => 8 + }); + + // Device memory override + Object.defineProperty(navigator, 'deviceMemory', { + get: () => 8 + }); + """) + + return page +``` + +--- + +## Resource Interception (Speed + Stealth) + +Block unnecessary resources to load faster AND look less like a data-heavy bot: + +```python +def block_resources(route): + """Block images, CSS, fonts, media — only allow documents and scripts.""" + if route.request.resource_type in ["image", "stylesheet", "font", "media"]: + route.abort() + else: + route.continue_() + +# Apply to page +page.route("**/*", block_resources) +``` + +**When to use:** Always unless you need images or visual rendering. +**Speed improvement:** ~2-5× faster page loads. + +--- + +## User-Agent Rotation + +```python +import random + +USER_AGENTS = [ + # Chrome on Windows + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36", + # Chrome on Mac + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36", + # Chrome on Linux + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36", + # Firefox on Windows + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:126.0) Gecko/20100101 Firefox/126.0", + # Firefox on Mac + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:126.0) Gecko/20100101 Firefox/126.0", + # Safari on Mac + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Safari/605.1.15", + # Edge on Windows + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 Edg/125.0.0.0", +] + +def random_ua(): + """Return a random User-Agent string.""" + return random.choice(USER_AGENTS) + +# Usage with requests +r = requests.get(url, headers={"User-Agent": random_ua()}) + +# Usage with Playwright +page = browser.new_page(user_agent=random_ua()) +``` + +**When to rotate:** Every few requests to the same domain. Don't rotate per-request (suspicious). + +--- + +## Proxy Strategies + +### HTTP/HTTPS Proxy + +```python +# With requests +proxies = {"http": "http://user:pass@proxy:8080", + "https": "http://user:pass@proxy:8080"} +r = requests.get(url, proxies=proxies) + +# With Playwright +context = browser.new_context(proxy={"server": "http://proxy:8080", + "username": "user", + "password": "pass"}) +``` + +### SOCKS5 Proxy + +```python +# With requests (requires pip install requests[socks]) +proxies = {"http": "socks5://user:pass@proxy:1080", + "https": "socks5://user:pass@proxy:1080"} + +# With Playwright +context = browser.new_context(proxy={"server": "socks5://proxy:1080"}) +``` + +### Proxy Pool Rotation + +```python +from itertools import cycle + +proxies = [ + "http://user:pass@proxy1:8080", + "http://user:pass@proxy2:8080", + "http://user:pass@proxy3:8080", +] +proxy_pool = cycle(proxies) + +def request_with_rotation(url): + """Make a request with rotating proxy.""" + proxy = next(proxy_pool) + return requests.get(url, proxies={"http": proxy, "https": proxy}, + headers={"User-Agent": random_ua()}) +``` + +**When to use proxies:** Getting IP-blocked, need geo-diversity, heavy scraping. +**When NOT to:** Simple one-off requests, free API queries. + +--- + +## Session Management (Cookie Persistence) + +For sites that require login or maintain session state: + +```python +from playwright.sync_api import sync_playwright + +# playwright-stealth v2.0.3+ compatibility +try: + from playwright_stealth import Stealth + _apply_stealth = lambda page: Stealth().use_sync(page) +except ImportError: + from playwright_stealth import stealth_sync + _apply_stealth = lambda page: stealth_sync(page) + +def persistent_session(url, user_data_dir="./browser_session", save_cookies=True): + """Use persistent browser context to maintain cookies/sessions. + + user_data_dir: Directory to store browser profile (cookies, localStorage, etc.) + """ + with sync_playwright() as p: + context = p.chromium.launch_persistent_context( + user_data_dir=user_data_dir, + headless=True, + user_agent=random_ua(), + ) + page = context.new_page() + _apply_stealth(page) + + page.goto(url, wait_until="domcontentloaded", timeout=30000) + page.wait_for_timeout(3000) + + content = page.inner_text("body") + + if save_cookies: + context.storage_state(path=f"{user_data_dir}/storage_state.json") + + context.close() + return content + +def load_session(url, storage_state_path="./browser_session/storage_state.json"): + """Load a previously saved session.""" + with sync_playwright() as p: + browser = p.chromium.launch(headless=True) + context = browser.new_context(storage_state=storage_state_path) + page = context.new_page() + _apply_stealth(page) + + page.goto(url, wait_until="domcontentloaded") + content = page.inner_text("body") + + browser.close() + return content +``` + +**When to use:** Sites requiring login (LinkedIn, some forums), sites with CSRF tokens. + +--- + +## CAPTCHA Decision Tree + +``` +CAPTCHA encountered ─────────────┐ + │ + ┌────────────────────────────┼────────────────────────────┐ + │ │ │ + reCAPTCHA/ Simple/ Cloudflare + hCaptcha Image CAPTCHA Challenge + │ │ │ + ▼ ▼ ▼ + STOP. Try refreshing Try nodriver + Do NOT attempt the page. If (if installed). + to solve. persistent, If still blocked, + Switch to lower request STOP. + another frequency. Switch to API + data source. or different + (API, RSS, data source. + different site) +``` + +### Key Rules + +1. **Never try to solve reCAPTCHA/hCaptcha programmatically** — it's unethical and usually futile +2. **Cloudflare challenge** → try `nodriver` library or switch to API +3. **Simple image CAPTCHA** → refresh the page, try again with different fingerprint +4. **Rate-triggered CAPTCHA** → slow down! You're going too fast + +--- + +## Rate Limiting + +### RateLimiter Class + +```python +import time, random +from collections import defaultdict + +class RateLimiter: + """Per-domain rate limiter with jitter. + + Usage: + limiter = RateLimiter(min_interval=2.0) + limiter.wait("example.com") + r = requests.get(url) + """ + def __init__(self, min_interval=2.0): + self.last_request = defaultdict(float) + self.min_interval = min_interval + + def wait(self, domain): + """Wait if needed before making a request to this domain.""" + elapsed = time.time() - self.last_request[domain] + if elapsed < self.min_interval: + wait_time = self.min_interval - elapsed + random.random() + time.sleep(wait_time) + self.last_request[domain] = time.time() + + def set_interval(self, domain, interval): + """Override interval for a specific domain.""" + self.min_interval = interval + +# Global instance +rate_limiter = RateLimiter(min_interval=2.0) +``` + +### Smart Delay + +```python +def smart_delay(base=2.0, jitter=1.0): + """Random delay to avoid pattern detection. + + base: minimum delay in seconds + jitter: additional random delay (0 to jitter) + """ + time.sleep(base + random.random() * jitter) + +def exponential_backoff(attempt, base=1.0, max_delay=60.0): + """Exponential backoff with jitter for retries. + + Use when getting 429 or 5xx errors. + """ + delay = min(base * (2 ** attempt) + random.random(), max_delay) + time.sleep(delay) + return delay +``` + +### Site-Specific Rate Limits + +| Site | Max Rate | Notes | +|------|----------|-------| +| Google Scholar | ~10 req/IP then CAPTCHA | Use SerpAPI instead | +| Reddit | 60 req/min | Must set User-Agent | +| Wikipedia | Reasonable use | Be nice, cache results | +| GitHub | 60/hr (5000/hr with token) | Use token for heavy use | +| arXiv | Be reasonable | No explicit limit | +| Nature/Springer | Aggressive blocking | Use API when possible | +| Twitter/X | Very aggressive | Requires login | + +--- + +## Site-Specific Stealth Notes + +### Google Scholar +- Max ~10 requests from same IP before CAPTCHA +- Playwright stealth works briefly but not reliably +- **Better approach:** Use SerpAPI/Serper for Scholar results +- If you must scrape: residential proxy + long delays (10s+) + +### Twitter/X +- Requires login for most content now +- API requires paid tier +- **Alternative:** Nitter instances (if available): `https://nitter.net/user` +- Do NOT attempt to bypass login wall + +### LinkedIn +- Most aggressive bot detection of any major site +- Requires persistent session + residential proxy +- **Better approach:** Use LinkedIn API if available +- Do NOT scrape LinkedIn profiles at scale + +### Reddit +- `old.reddit.com` is easier to scrape than new UI +- JSON API: append `.json` to any URL + User-Agent header +- Rate limit: 60 req/min +- **Best approach:** Use JSON API, not HTML scraping + +### Medium +- Client-side paywall (trafilatura often gets full text) +- `?source=friends_link` sometimes bypasses paywall +- Jina Reader can often extract full articles +- **Try:** trafilatura → Jina Reader → give up + +### News Sites +- Most use client-side paywalls (JS-based) +- trafilatura often extracts before paywall triggers +- RSS feeds give headlines + summaries for free +- **Strategy:** RSS first → trafilatura → Jina Reader → Wayback Machine + +--- + +## nodriver: Maximum Stealth (Alternative to Playwright) + +For sites that detect Playwright specifically: + +```python +# nodriver (formerly undetected-chromedriver) +# pip install nodriver + +import nodriver as uc + +async def nodriver_fetch(url): + """Fetch using nodriver — most stealthy Python option. + + nodriver patches Chrome at a lower level than playwright-stealth, + making it harder to detect. Slower startup but more reliable + against advanced bot detection. + """ + browser = await uc.start() + page = await browser.get(url) + await page.sleep(3) # Wait for JS + content = await page.get_content() + return content +``` + +**When to use:** Playwright stealth is detected (Cloudflare, some banking sites). +**When NOT to:** Standard scraping — Playwright is faster and more reliable. + +--- + +## Complete Stealth Workflow + +```python +def stealth_workflow(url, max_retries=2): + """Complete stealth browsing workflow with fallback. + + 1. Try requests + random UA (lightest) + 2. Try trafilatura (fast extraction) + 3. Try Playwright stealth (JS rendering) + 4. Try Jina Reader (cloud fallback) + """ + rate_limiter.wait(urlparse(url).netloc) + + # Step 1: Simple request + try: + r = requests.get(url, headers={"User-Agent": random_ua()}, timeout=15) + if r.status_code == 200 and len(r.text) > 500: + return {"method": "requests", "content": r.text[:5000]} + except Exception: + pass + + # Step 2: trafilatura + try: + import trafilatura + html = trafilatura.fetch_url(url) + if html: + text = trafilatura.extract(html) + if text and len(text) > 200: + return {"method": "trafilatura", "content": text[:5000]} + except Exception: + pass + + # Step 3: Playwright stealth + try: + result = stealth_fetch(url) + if result.get("body") and len(result["body"]) > 200: + return {"method": "playwright-stealth", "content": result["body"][:5000]} + except Exception: + pass + + # Step 4: Jina Reader + try: + r = requests.get(f"https://r.jina.ai/{url}", timeout=45, + headers={"Accept": "text/markdown"}) + if r.status_code == 200 and len(r.text) > 200: + return {"method": "jina-reader", "content": r.text[:5000]} + except Exception: + pass + + return {"method": "failed", "error": "All stealth methods failed"} +``` + +--- + +## Dependencies + +```bash +# Core (already installed in LingTai) +pip install playwright playwright-stealth +playwright install chromium + +# Optional +pip install nodriver # Maximum stealth alternative +pip install requests[socks] # SOCKS5 proxy support +``` + +--- + +*This sub-skill is part of `web-browsing-manual` v3.0. For the main extraction pipeline, see the parent skill. For academic-specific browsing, see `academic-search-pipeline`.* diff --git a/src/lingtai/tools/web_search/manual/reference/tier-0-pdf.md b/src/lingtai/tools/web_search/manual/reference/tier-0-pdf.md new file mode 100644 index 000000000..fc699e25e --- /dev/null +++ b/src/lingtai/tools/web_search/manual/reference/tier-0-pdf.md @@ -0,0 +1,24 @@ +# Tier 0 — PDF Direct Download + +> Part of the [web-browsing](../SKILL.md) skill. + +**When it applies:** PDF direct links, DOI strings, arXiv IDs. +**Tools:** `curl` + `fitz` (no script needed). +**Speed:** ~1s. + +```bash +# Direct PDF link +curl -L "https://arxiv.org/pdf/1706.03762.pdf" -o paper.pdf + +# arXiv ID → derive the PDF path +curl -L "https://arxiv.org/pdf/$(echo "2401.12345" | sed 's/\.//').pdf" -o paper.pdf +``` + +**Python (extract PDF text):** +```python +import fitz # pip install pymupdf +doc = fitz.open("paper.pdf") +print(doc[0].get_text()[:500]) # first-page preview +``` + +**Use when:** the URL contains `.pdf`, or you already have a DOI / arXiv ID. diff --git a/src/lingtai/tools/web_search/manual/reference/tier-1-5-trafilatura.md b/src/lingtai/tools/web_search/manual/reference/tier-1-5-trafilatura.md new file mode 100644 index 000000000..44a04c727 --- /dev/null +++ b/src/lingtai/tools/web_search/manual/reference/tier-1-5-trafilatura.md @@ -0,0 +1,45 @@ +# Tier 1.5 — Trafilatura Fast Extraction + +> Part of the [web-browsing](../SKILL.md) skill. + +**When it applies:** Any static HTML page — articles, blogs, news, documentation. +**Tools:** `trafilatura` (**already installed** — no setup needed). +**Speed:** ~0.05s per page (10-50× faster than BeautifulSoup, 200× faster than browser). + +```python +import trafilatura + +def tier1_5(url): + """Fast article extraction — no browser, no JS, blazing speed.""" + html = trafilatura.fetch_url(url) + if not html: + return None + text = trafilatura.extract(html) # Main content as plain text + meta = trafilatura.bare_extraction(html) # Returns Document, NOT dict + + # bare_extraction() returns a Document object (has .title, .author etc.) + # Convert to dict for safe .get() access — do NOT call meta.get() directly! + if hasattr(meta, '__dict__'): + meta = {k: v for k, v in meta.__dict__.items() if not k.startswith('_')} + elif meta is None: + meta = {} + + return { + "url": url, + "method": "tier1.5-trafilatura", + "title": meta.get("title"), + "author": meta.get("author"), + "date": meta.get("date"), + "description": meta.get("description"), + "categories": meta.get("categories"), + "tags": meta.get("tags"), + "text": text, + "text_len": len(text) if text else 0, + } +``` + +**Also outputs:** Markdown (`output_format="markdown"`), JSON, XML. +**Batch mode:** `trafilatura.spider` for sitemap-based crawling. +**Feed support:** `trafilatura.feed` for RSS/Atom feed extraction. + +**Use when:** the page is a static article/blog/news page and you just need the text content. This is your **default first attempt** for any non-API, non-PDF URL. diff --git a/src/lingtai/tools/web_search/manual/reference/tier-1-apis.md b/src/lingtai/tools/web_search/manual/reference/tier-1-apis.md new file mode 100644 index 000000000..ba5366622 --- /dev/null +++ b/src/lingtai/tools/web_search/manual/reference/tier-1-apis.md @@ -0,0 +1,102 @@ +# Tier 1 — API Metadata Queries + +> Part of the [web-browsing](../SKILL.md) skill. + +**When it applies:** Known academic IDs (DOI, arXiv, PMID, PMC), or sites with free APIs. +**Tools:** `requests` (HTTP) — call APIs directly from Python. +**Speed:** ~0.5s. + +### Academic APIs + +| API | Endpoint | Free? | Best for | +|-----|----------|-------|----------| +| **arXiv** | `GET https://export.arxiv.org/api/query?id_list={ID}` | ✅ | CS/Physics/Math papers | +| **OpenAlex** | `GET https://api.openalex.org/works/https://doi.org/{DOI}` | ✅ | Any DOI → full metadata + citations | +| **CrossRef** | `GET https://api.crossref.org/works/{DOI}` | ✅ | DOI → metadata (title, authors, journal) | +| **Semantic Scholar** | `GET https://api.semanticscholar.org/graph/v1/paper/{DOI}?fields=...` | ✅* | AI/ML papers, citation graphs | +| **PubMed E-utilities** | `GET https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=pubmed&id={PMID}` | ✅ | Biomedical literature | +| **CORE** | `GET https://api.core.ac.uk/v3/search/works?q={query}` | ✅† | Open access full text (30M+ papers) | +| **Unpaywall** | `GET https://api.unpaywall.org/v2/{DOI}?email=lingtai@users.noreply.github.com` | ✅ | Find free PDF for any paper | +| **Europe PMC** | `GET https://www.ebi.ac.uk/europepmc/webservices/rest/search?query={q}&format=json` | ✅ | Biomedical + PMC full text | +| **DBLP** | `GET https://dblp.org/search/publ/api?q={query}&format=json&h=10` | ✅ | Computer science conference papers | +| **Papers With Code** | `GET https://paperswithcode.com/api/v1/search/?q={query}` | ✅ | ML/AI papers with code + benchmarks | +| **DOAJ** | `GET https://doaj.org/api/search/articles/{query}` | ✅ | Open access journal articles | +| **Zenodo** | `GET https://zenodo.org/api/records?q={query}` | ✅ | Research data, software, datasets | +| **NASA ADS** | `GET https://ui.adsabs.harvard.edu/abs/{arxiv_id}/bibtex` | ✅ | Astrophysics/astronomy | + +\* Semantic Scholar: 100 req/5min without key, 10k/day with key. +† CORE: free API key at https://core.ac.uk/services/api, 1000/day. + +### Quick Examples + +```python +import requests + +# OpenAlex — most powerful, completely free +r = requests.get("https://api.openalex.org/works/https://doi.org/10.1038/s41586-023-05995-9") +data = r.json() +# Returns: title, authors, abstract, citation_count, topics, pdf_link + +# Unpaywall — find free PDF for any DOI +r = requests.get(f"https://api.unpaywall.org/v2/{doi}?email=lingtai@users.noreply.github.com") +oa = r.json() +if oa.get("best_oa_location"): + pdf_url = oa["best_oa_location"].get("url_for_pdf") + +# DBLP — computer science papers +r = requests.get("https://dblp.org/search/publ/api?q=transformer+attention&format=json&h=5") +# Returns: title, authors, venue, year, DOI, URL + +# Papers With Code — ML papers with implementations +r = requests.get("https://paperswithcode.com/api/v1/search/?q=vision+transformer") +# Returns: papers linked to GitHub repos + benchmark results + +# Europe PMC — biomedical, includes PMC full text +r = requests.get("https://www.ebi.ac.uk/europepmc/webservices/rest/search?query=malaria+ vaccine&format=json&pageSize=5") +``` + +### Academic Search Pipeline: Find → Enrich → Get PDF + +```python +def academic_search_pipeline(query, max_results=10): + """Complete pipeline: search → enrich with metadata → find OA PDF.""" + # Step 1: Search via OpenAlex (best general academic search) + r = requests.get("https://api.openalex.org/works", + params={"search": query, "per_page": max_results, + "sort": "cited_by_count:desc"}) + papers = [] + for result in r.json()["results"]: + paper = { + "title": result["title"], + "doi": result.get("doi", "").replace("https://doi.org/", ""), + "year": result.get("publication_year"), + "citations": result.get("cited_by_count", 0), + "authors": [a["author"]["display_name"] for a in result.get("authorships", [])], + } + # Step 2: Find OA PDF via Unpaywall + if paper["doi"]: + try: + oa = requests.get( + f"https://api.unpaywall.org/v2/{paper['doi']}?email=lingtai@users.noreply.github.com", + timeout=5).json() + if oa.get("best_oa_location"): + paper["pdf_url"] = oa["best_oa_location"].get("url_for_pdf") + paper["oa_url"] = oa["best_oa_location"].get("url") + except Exception: + pass + papers.append(paper) + return papers +``` + +### ID Resolution Chain + +``` +Given any identifier: + DOI? → CrossRef / OpenAlex / Unpaywall + arXiv ID? → arXiv API + PMID? → PubMed E-utilities / Europe PMC + Title only? → OpenAlex search / Semantic Scholar / DBLP (CS) + Need PDF? → Unpaywall → arXiv (if CS) → CORE → Europe PMC (biomedical) +``` + +**Use when:** you have a DOI, arXiv ID, PMID, or just need metadata + abstract quickly. diff --git a/src/lingtai/tools/web_search/manual/reference/tier-2-beautifulsoup.md b/src/lingtai/tools/web_search/manual/reference/tier-2-beautifulsoup.md new file mode 100644 index 000000000..c7aecc850 --- /dev/null +++ b/src/lingtai/tools/web_search/manual/reference/tier-2-beautifulsoup.md @@ -0,0 +1,146 @@ +# Tier 2 — BeautifulSoup Structured Extraction + +> Part of the [web-browsing](../SKILL.md) skill. + +**When it applies:** Pages needing structured data extraction (lists, tables, multi-element scraping). +**Tools:** `requests` + `beautifulsoup4` + `lxml` (all installed). +**Speed:** ~0.15s per page. + +```python +import requests, re +from bs4 import BeautifulSoup +from urllib.parse import urljoin + +def tier2(url): + """Structured extraction with site-specific selectors.""" + r = requests.get(url, headers={"User-Agent": "Mozilla/5.0"}, timeout=10) + soup = BeautifulSoup(r.text, "lxml") + title = soup.find("title").get_text(strip=True) if soup.find("title") else None + result = {"url": url, "method": "tier2-bs", "title": title} + + # Google Scholar (search results page) + if "scholar.google" in url: + papers = [] + for card in soup.select("div.gs_ri"): + title_el = card.select_one("h3.gs_rt") + abstract_el = card.select_one("div.gs_rs") + link_el = card.select_one("h3.gs_rt a") + papers.append({ + "title": title_el.get_text(strip=True) if title_el else None, + "link": link_el["href"] if link_el else None, + "abstract": abstract_el.get_text(strip=True) if abstract_el else None, + }) + result["papers"] = papers[:10] + + # Reddit (old.reddit.com or .json) + elif "reddit.com" in url: + for post in soup.select("div.thing.linkflair"): + result.setdefault("posts", []).append({ + "title": post.select_one("a.title").get_text(strip=True) if post.select_one("a.title") else None, + "score": post.select_one("div.score").get_text(strip=True) if post.select_one("div.score") else None, + "comments": post.select_one("a.comments").get_text(strip=True) if post.select_one("a.comments") else None, + }) + + # Nature.com (og meta) + elif "nature.com" in url: + og_title = soup.find("meta", property="og:title") + og_desc = soup.find("meta", property="og:description") + citation_doi = soup.find("meta", attrs={"name": "citation_doi"}) + result.update({ + "og_title": og_title["content"] if og_title else None, + "og_description": og_desc["content"] if og_desc else None, + "doi": citation_doi["content"] if citation_doi else None, + }) + + # arXiv (structured metadata) + elif "arxiv.org" in url: + abstract_el = soup.find("blockquote", class_="abstract") + pdf_links = re.findall(r'href="(/pdf/[^"]+\.pdf)"', r.text) + result.update({ + "abstract": abstract_el.get_text(strip=True) if abstract_el else None, + "pdf_links": [urljoin(url, p) for p in pdf_links[:3]], + }) + + # Generic: extract JSON-LD structured data + jsonld_scripts = soup.find_all("script", type="application/ld+json") + if jsonld_scripts: + import json + result["jsonld"] = [json.loads(s.string) for s in jsonld_scripts if s.string] + + # Generic: extract OpenGraph + Twitter Cards + og_data = {} + for tag in soup.find_all("meta", attrs={"property": True}): + if tag["property"].startswith("og:"): + og_data[tag["property"]] = tag.get("content", "") + for tag in soup.find_all("meta", attrs={"name": True}): + if tag["name"].startswith("twitter:"): + og_data[tag["name"]] = tag.get("content", "") + if og_data: + result["og"] = og_data + + return result +``` + +### Quick Structured Extraction Patterns + +**Reddit JSON API** (append `.json` to any Reddit URL): +```python +import requests +r = requests.get("https://www.reddit.com/r/programming/hot.json?limit=25", + headers={"User-Agent": "LingTai/1.0"}, timeout=10) +posts = r.json()["data"]["children"] +for post in posts: + print(post["data"]["title"], post["data"]["score"]) +``` + +**Hacker News API** (completely free, no key): +```python +import requests +ids = requests.get("https://hacker-news.firebaseio.com/v0/topstories.json").json()[:10] +for id in ids: + item = requests.get(f"https://hacker-news.firebaseio.com/v0/item/{id}.json").json() + print(item["title"], item.get("url")) +``` + +**Google News RSS** (free, no key): +```python +import requests, xml.etree.ElementTree as ET +r = requests.get("https://news.google.com/rss/search?q=AI+safety&hl=en&gl=US&ceid=US:en") +root = ET.fromstring(r.text) +for item in root.findall(".//item")[:5]: + print(item.find("title").text, item.find("link").text) +``` + +**GitHub Search** (60/hr without token, 5000/hr with token): +```python +r = requests.get("https://api.github.com/search/repositories?q=web+scraping&sort=stars") +for repo in r.json()["items"][:5]: + print(repo["full_name"], repo["stargazers_count"], repo["html_url"]) +``` + +**Wikipedia Summary** (free, no key): +```python +r = requests.get("https://en.wikipedia.org/api/rest_v1/page/summary/Web_scraping") +data = r.json() +print(data["title"], data["extract"]) +``` + +### Key CSS Selectors + +| Site | Selector | Extracts | +|------|----------|----------| +| Google Scholar | `div.gs_ri` | One paper card | +| Google Scholar | `h3.gs_rt` | Paper title | +| Google Scholar | `div.gs_rs` | Abstract / snippet | +| arXiv | `h1.title` | Title | +| arXiv | `blockquote.abstract` | Abstract | +| arXiv | `a[href*="/pdf/"]` | PDF link | +| Nature.com | `meta[property="og:title"]` | Title | +| Nature.com | `meta[name="citation_doi"]` | DOI | +| Springer | `meta[name="citation_doi"]` | DOI | +| Reddit (old) | `div.thing` | Post card | +| Reddit (old) | `a.title` | Post title | +| Medium | `article` | Article body | +| Generic | `article` / `[itemprop="articleBody"]` | Main content | + +**Use when:** you need structured data from a page (lists, tables, metadata), not just raw text. diff --git a/src/lingtai/tools/web_search/manual/reference/tier-3-playwright.md b/src/lingtai/tools/web_search/manual/reference/tier-3-playwright.md new file mode 100644 index 000000000..133e170df --- /dev/null +++ b/src/lingtai/tools/web_search/manual/reference/tier-3-playwright.md @@ -0,0 +1,520 @@ +# Tier 3 — Playwright Stealth + +> Part of the [web-browsing](../SKILL.md) skill. +> See also: [stealth.md](./stealth.md) for comprehensive anti-detection techniques. + +**When it applies:** JS-rendered pages, login-gated content, sites blocking simple requests. +**Tools:** `playwright` + `playwright-stealth` (**already installed**). +**Speed:** ~3-5s per page. +**⚠️ CRITICAL:** For Nature / Springer, use `domcontentloaded`, NOT `networkidle` (it hangs forever). + +```python +from playwright.sync_api import sync_playwright + +# playwright-stealth v2.0.3+ compatibility +try: + from playwright_stealth import Stealth + _apply_stealth = lambda page: Stealth().use_sync(page) +except ImportError: + from playwright_stealth import stealth_sync + _apply_stealth = lambda page: stealth_sync(page) + +def tier3(url, wait_time=3): + """Playwright stealth — JS-rendered or protected pages.""" + with sync_playwright() as p: + browser = p.chromium.launch(headless=True) + page = browser.new_page() + _apply_stealth(page) + + # Block images/styles/fonts for speed + def block_resources(route): + if route.request.resource_type in ["image", "stylesheet", "font", "media"]: + route.abort() + else: + route.continue_() + page.route("**/*", block_resources) + + # CRITICAL: do NOT use networkidle (Nature / Springer hang forever) + page.goto(url, wait_until="domcontentloaded", timeout=30000) + page.wait_for_timeout(wait_time * 1000) + + content = page.inner_text("body") + html = page.content() + title = page.title() + browser.close() + + return { + "url": page.url, + "method": "tier3-playwright-stealth", + "title": title, + "body_preview": content[:5000], + "html_len": len(html), + } +``` + +### Advanced Stealth Techniques + +**Custom init scripts** (override fingerprinting): +```python +# Override navigator.webdriver +page.add_init_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})") + +# Override chrome runtime +page.add_init_script(""" +window.chrome = { runtime: {}, loadTimes: function(){}, csi: function(){}, app: {} }; +""") + +# Override languages +page.add_init_script(""" +Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] }); +""") +``` + +**Persistent sessions** (for login-gated sites): +```python +context = p.chromium.launch_persistent_context( + user_data_dir="./browser_data", + headless=True, +) +# ... work ... +context.storage_state(path="cookies.json") # Save session +``` + +**Smart rate limiting:** +```python +import time, random +def smart_delay(base=2.0, jitter=1.0): + """Random delay to avoid pattern detection.""" + time.sleep(base + random.random() * jitter) +``` + +**User-Agent rotation:** +```python +import random +USER_AGENTS = [ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120.0.0.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/120.0.0.0", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/120.0.0.0", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 Safari/17.2", +] +``` + +**Use when:** Tier 1 / 1.5 / 2 fail, or the page genuinely requires JS rendering or bypasses anti-bot. + +--- + +## Extended Stealth Techniques (from stealth-browsing sub-skill) + +### How Websites Detect Bots + +Understanding detection methods helps you choose the right countermeasure. + +#### 1. Browser Fingerprinting + +Websites build a unique fingerprint from: +- **Canvas/WebGL:** GPU rendering differences create unique hashes +- **Audio context:** Audio processing differences +- **Font enumeration:** Installed fonts list +- **Screen:** Resolution + color depth +- **Hardware:** CPU cores, device memory +- **Platform:** OS + browser version consistency + +#### 2. WebDriver Detection + +Automated browsers leak signals: +- `navigator.webdriver === true` (Playwright/Selenium) +- Missing `window.chrome` object in headless +- CDP (Chrome DevTools Protocol) detection +- `$cdc_` variable injection by ChromeDriver +- Missing plugins/mimeTypes + +#### 3. TLS/JA3 Fingerprinting + +- Python `requests` has a distinctive TLS fingerprint +- Headless browsers may differ from real ones + +#### 4. Behavioral Analysis + +- Mouse movement patterns (too perfect = bot) +- No scroll behavior +- Click timing (instant = bot) +- Request frequency patterns + +#### 5. IP/Network Level + +- Datacenter IP detection +- Proxy detection headers +- Rate limiting per IP +- Geo-inconsistency + +### Additional Manual Stealth Overrides + +For sites that detect `playwright-stealth`, apply additional overrides: + +**Override permissions API:** +```python +page.add_init_script(""" + const originalQuery = window.navigator.permissions.query; + window.navigator.permissions.query = (parameters) => + parameters.name === 'notifications' + ? Promise.resolve({ state: Notification.permission }) + : originalQuery(parameters); +""") +``` + +**Override plugins:** +```python +page.add_init_script(""" + Object.defineProperty(navigator, 'plugins', { + get: () => [1, 2, 3, 4, 5], + }); +""") +``` + +**Full Stealth Page (All Overrides):** +```python +def create_stealth_page(browser): + """Create a fully stealthed page with all overrides.""" + page = browser.new_page() + _apply_stealth(page) + + # Additional overrides beyond stealth_sync + page.add_init_script(""" + // WebGL vendor/renderer override + const getParameter = WebGLRenderingContext.prototype.getParameter; + WebGLRenderingContext.prototype.getParameter = function(param) { + if (param === 37445) return 'Intel Inc.'; + if (param === 37446) return 'Intel Iris OpenGL Engine'; + return getParameter.call(this, param); + }; + + // Hardware concurrency override + Object.defineProperty(navigator, 'hardwareConcurrency', { + get: () => 8 + }); + + // Device memory override + Object.defineProperty(navigator, 'deviceMemory', { + get: () => 8 + }); + """) + + return page +``` + +### Proxy Strategies + +**HTTP/HTTPS Proxy:** +```python +# With requests +proxies = {"http": "http://user:pass@proxy:8080", + "https": "http://user:pass@proxy:8080"} +r = requests.get(url, proxies=proxies) + +# With Playwright +context = browser.new_context(proxy={"server": "http://proxy:8080", + "username": "user", + "password": "pass"}) +``` + +**SOCKS5 Proxy:** +```python +# With requests (requires pip install requests[socks]) +proxies = {"http": "socks5://user:pass@proxy:1080", + "https": "socks5://user:pass@proxy:1080"} + +# With Playwright +context = browser.new_context(proxy={"server": "socks5://proxy:1080"}) +``` + +**Proxy Pool Rotation:** +```python +from itertools import cycle + +proxies = [ + "http://user:pass@proxy1:8080", + "http://user:pass@proxy2:8080", + "http://user:pass@proxy3:8080", +] +proxy_pool = cycle(proxies) + +def request_with_rotation(url): + """Make a request with rotating proxy.""" + proxy = next(proxy_pool) + return requests.get(url, proxies={"http": proxy, "https": proxy}, + headers={"User-Agent": random_ua()}) +``` + +**When to use proxies:** Getting IP-blocked, need geo-diversity, heavy scraping. +**When NOT to:** Simple one-off requests, free API queries. + +### Session Management (Cookie Persistence) + +For sites that require login or maintain session state: + +```python +from playwright.sync_api import sync_playwright + +# playwright-stealth v2.0.3+ compatibility +try: + from playwright_stealth import Stealth + _apply_stealth = lambda page: Stealth().use_sync(page) +except ImportError: + from playwright_stealth import stealth_sync + _apply_stealth = lambda page: stealth_sync(page) + +def persistent_session(url, user_data_dir="./browser_session", save_cookies=True): + """Use persistent browser context to maintain cookies/sessions.""" + with sync_playwright() as p: + context = p.chromium.launch_persistent_context( + user_data_dir=user_data_dir, + headless=True, + user_agent=random_ua(), + ) + page = context.new_page() + _apply_stealth(page) + + page.goto(url, wait_until="domcontentloaded", timeout=30000) + page.wait_for_timeout(3000) + + content = page.inner_text("body") + + if save_cookies: + context.storage_state(path=f"{user_data_dir}/storage_state.json") + + context.close() + return content + +def load_session(url, storage_state_path="./browser_session/storage_state.json"): + """Load a previously saved session.""" + with sync_playwright() as p: + browser = p.chromium.launch(headless=True) + context = browser.new_context(storage_state=storage_state_path) + page = context.new_page() + _apply_stealth(page) + + page.goto(url, wait_until="domcontentloaded") + content = page.inner_text("body") + + browser.close() + return content +``` + +**When to use:** Sites requiring login (LinkedIn, some forums), sites with CSRF tokens. + +### CAPTCHA Decision Tree + +``` +CAPTCHA encountered ──────┐ + │ + ┌─────────────────────┼─────────────────────┐ + │ │ │ + reCAPTCHA/ Simple/ Cloudflare + hCaptcha Image CAPTCHA Challenge + │ │ │ + ▼ ▼ ▼ + STOP. Try refreshing Try nodriver + Do NOT attempt the page. If (if installed). + to solve. persistent, If still blocked, + Switch to lower request STOP. + another frequency. Switch to API + data source. or different + (API, RSS, data source. + different site) +``` + +**Key Rules:** + +1. **Never try to solve reCAPTCHA/hCaptcha programmatically** — it's unethical and usually futile +2. **Cloudflare challenge** → try `nodriver` library or switch to API +3. **Simple image CAPTCHA** → refresh the page, try again with different fingerprint +4. **Rate-triggered CAPTCHA** → slow down! You're going too fast + +### Rate Limiting + +**RateLimiter Class:** +```python +import time, random +from collections import defaultdict + +class RateLimiter: + """Per-domain rate limiter with jitter. + + Usage: + limiter = RateLimiter(min_interval=2.0) + limiter.wait("example.com") + r = requests.get(url) + """ + def __init__(self, min_interval=2.0): + self.last_request = defaultdict(float) + self.min_interval = min_interval + + def wait(self, domain): + """Wait if needed before making a request to this domain.""" + elapsed = time.time() - self.last_request[domain] + if elapsed < self.min_interval: + wait_time = self.min_interval - elapsed + random.random() + time.sleep(wait_time) + self.last_request[domain] = time.time() + + def set_interval(self, domain, interval): + """Override interval for a specific domain.""" + self.min_interval = interval + +# Global instance +rate_limiter = RateLimiter(min_interval=2.0) +``` + +**Exponential Backoff:** +```python +def exponential_backoff(attempt, base=1.0, max_delay=60.0): + """Exponential backoff with jitter for retries. + + Use when getting 429 or 5xx errors. + """ + delay = min(base * (2 ** attempt) + random.random(), max_delay) + time.sleep(delay) + return delay +``` + +**Site-Specific Rate Limits:** + +| Site | Max Rate | Notes | +|------|----------|-------| +| Google Scholar | ~10 req/IP then CAPTCHA | Use SerpAPI instead | +| Reddit | 60 req/min | Must set User-Agent | +| Wikipedia | Reasonable use | Be nice, cache results | +| GitHub | 60/hr (5000/hr with token) | Use token for heavy use | +| arXiv | Be reasonable | No explicit limit | +| Nature/Springer | Aggressive blocking | Use API when possible | +| Twitter/X | Very aggressive | Requires login | + +### Site-Specific Stealth Notes + +**Google Scholar:** +- Max ~10 requests from same IP before CAPTCHA +- Playwright stealth works briefly but not reliably +- **Better approach:** Use SerpAPI/Serper for Scholar results +- If you must scrape: residential proxy + long delays (10s+) + +**Twitter/X:** +- Requires login for most content now +- API requires paid tier +- **Alternative:** Nitter instances (if available): `https://nitter.net/user` +- Do NOT attempt to bypass login wall + +**LinkedIn:** +- Most aggressive bot detection of any major site +- Requires persistent session + residential proxy +- **Better approach:** Use LinkedIn API if available +- Do NOT scrape LinkedIn profiles at scale + +**Reddit:** +- `old.reddit.com` is easier to scrape than new UI +- JSON API: append `.json` to any URL + User-Agent header +- Rate limit: 60 req/min +- **Best approach:** Use JSON API, not HTML scraping + +**Medium:** +- Client-side paywall (trafilatura often gets full text) +- `?source=friends_link` sometimes bypasses paywall +- Jina Reader can often extract full articles +- **Try:** trafilatura → Jina Reader → give up + +**News Sites:** +- Most use client-side paywalls (JS-based) +- trafilatura often extracts before paywall triggers +- RSS feeds give headlines + summaries for free +- **Strategy:** RSS first → trafilatura → Jina Reader → Wayback Machine + +### nodriver: Maximum Stealth (Alternative to Playwright) + +For sites that detect Playwright specifically: + +```python +# nodriver (formerly undetected-chromedriver) +# pip install nodriver + +import nodriver as uc + +async def nodriver_fetch(url): + """Fetch using nodriver — most stealthy Python option. + + nodriver patches Chrome at a lower level than playwright-stealth, + making it harder to detect. Slower startup but more reliable + against advanced bot detection. + """ + browser = await uc.start() + page = await browser.get(url) + await page.sleep(3) # Wait for JS + content = await page.get_content() + return content +``` + +**When to use:** Playwright stealth is detected (Cloudflare, some banking sites). +**When NOT to:** Standard scraping — Playwright is faster and more reliable. + +### Complete Stealth Workflow + +```python +def stealth_workflow(url, max_retries=2): + """Complete stealth browsing workflow with fallback. + + 1. Try requests + random UA (lightest) + 2. Try trafilatura (fast extraction) + 3. Try Playwright stealth (JS rendering) + 4. Try Jina Reader (cloud fallback) + """ + rate_limiter.wait(urlparse(url).netloc) + + # Step 1: Simple request + try: + r = requests.get(url, headers={"User-Agent": random_ua()}, timeout=15) + if r.status_code == 200 and len(r.text) > 500: + return {"method": "requests", "content": r.text[:5000]} + except Exception: + pass + + # Step 2: trafilatura + try: + import trafilatura + html = trafilatura.fetch_url(url) + if html: + text = trafilatura.extract(html) + if text and len(text) > 200: + return {"method": "trafilatura", "content": text[:5000]} + except Exception: + pass + + # Step 3: Playwright stealth + try: + result = stealth_fetch(url) + if result.get("body") and len(result["body"]) > 200: + return {"method": "playwright-stealth", "content": result["body"][:5000]} + except Exception: + pass + + # Step 4: Jina Reader + try: + r = requests.get(f"https://r.jina.ai/{url}", timeout=45, + headers={"Accept": "text/markdown"}) + if r.status_code == 200 and len(r.text) > 200: + return {"method": "jina-reader", "content": r.text[:5000]} + except Exception: + pass + + return {"method": "failed", "error": "All stealth methods failed"} +``` + +### Dependencies + +```bash +# Core (already installed in LingTai) +pip install playwright playwright-stealth +playwright install chromium + +# Optional +pip install nodriver # Maximum stealth alternative +pip install requests[socks] # SOCKS5 proxy support +``` diff --git a/src/lingtai/tools/web_search/manual/reference/tier-4-jina-firecrawl.md b/src/lingtai/tools/web_search/manual/reference/tier-4-jina-firecrawl.md new file mode 100644 index 000000000..e96b6a179 --- /dev/null +++ b/src/lingtai/tools/web_search/manual/reference/tier-4-jina-firecrawl.md @@ -0,0 +1,46 @@ +# Tier 4 — API Fallback (Jina Reader / Firecrawl) + +> Part of the [web-browsing](../SKILL.md) skill. + +**When it applies:** Everything else fails. Jina Reader is the universal fallback — it renders JS server-side and returns clean markdown. +**Tools:** `requests` (Jina Reader) or `firecrawl-py` (Firecrawl). +**Speed:** ~2-5s (server-side rendering). +**Cost:** Jina Reader free (20 req/min), Firecrawl free (500 credits/month). + +### Jina Reader — Universal Page-to-Markdown (FREE, no key) + +```python +import requests + +def jina_extract(url, fmt="markdown"): + """Convert any URL to clean markdown via Jina Reader API.""" + r = requests.get( + f"https://r.jina.ai/{url}", + headers={ + "Accept": f"text/{fmt}", + "X-Return-Format": fmt, + }, + timeout=30, + ) + r.raise_for_status() + return r.text + +# Also supports: +# X-With-Links: true → include link references +# X-With-Images: true → include image descriptions +# Batch: https://r.jina.ai/{url1},{url2} +``` + +**One-liner:** `curl -s "https://r.jina.ai/$URL"` + +### Firecrawl — Production-Grade Scraping (Freemium) + +```python +# pip install firecrawl-py +from firecrawl import FirecrawlApp + +app = FirecrawlApp(api_key="your-key") +result = app.scrape_url(url, params={"formats": ["markdown", "html"]}) +``` + +**Use when:** All local methods fail (403, CAPTCHA, heavy JS, etc.). Jina Reader should be your first fallback — it's free, fast, and handles most cases. diff --git a/src/lingtai/tools/web_search/manual/reference/tier-5-ai-search.md b/src/lingtai/tools/web_search/manual/reference/tier-5-ai-search.md new file mode 100644 index 000000000..b6b2aedcd --- /dev/null +++ b/src/lingtai/tools/web_search/manual/reference/tier-5-ai-search.md @@ -0,0 +1,72 @@ +# Tier 5 — AI-Native Search (Tavily / Exa) + +> Part of the [web-browsing](../SKILL.md) skill. +> See also: [search-strategies.md](./search-strategies.md) for comprehensive search strategy guidance. + +**When it applies:** You need to *discover* content, not extract a known URL. AI-native search returns clean content with the results. +**Tools:** `requests` (Tavily/Exa APIs). +**Speed:** ~3-5s. +**Cost:** Tavily 1000 req/month free, Exa 1000 req/month free. + +### Tavily — Search + Extract + Answer in One Call + +```python +import requests + +def tavily_search(query, api_key, max_results=5, include_answer=True): + """AI-native search: returns search results + AI-generated answer + page content.""" + r = requests.post("https://api.tavily.com/search", json={ + "api_key": api_key, + "query": query, + "search_depth": "advanced", # "basic" or "advanced" + "include_answer": include_answer, + "include_raw_content": True, # Full page content + "max_results": max_results, + }) + data = r.json() + return { + "answer": data.get("answer"), # AI-generated answer + "results": data.get("results", []), # Each: title, url, content, raw_content, score + } +``` + +### Exa — Neural/Semantic Search + +```python +def exa_search(query, api_key, num_results=5): + """Neural search — finds content by meaning, not keywords.""" + r = requests.post("https://api.exa.ai/search", + headers={"x-api-key": api_key}, + json={ + "query": query, + "type": "auto", # "neural", "keyword", or "auto" + "numResults": num_results, + "contents": {"text": {"maxCharacters": 3000}}, # Extract full content + }) + return r.json() +``` + +### DuckDuckGo — Free, No-Key Search (Python library) + +```python +# pip install duckduckgo-search (already available) +from duckduckgo_search import DDGS + +with DDGS() as ddgs: + results = list(ddgs.text("machine learning frameworks 2025", max_results=10)) + # Also: ddgs.news(), ddgs.images(), ddgs.videos() + for r in results: + print(r["title"], r["href"], r["body"]) +``` + +**Search strategy decision tree:** +``` +Need search → Free & no setup? → DuckDuckGo (ddgs) + → AI agent workflow? → Tavily (search + answer + content) + → Semantic/meaning search? → Exa + → Google quality? → Serper or Google Custom Search + → Academic? → OpenAlex > Semantic Scholar > DBLP (CS) + → News? → Google News RSS (free) or ddgs.news() +``` + +**Use when:** you need to search the web and get content, not just links. Tavily and Exa combine search + extraction in a single call. diff --git a/src/lingtai/tools/web_search/manual/reference/tier-quick-refs/SKILL.md b/src/lingtai/tools/web_search/manual/reference/tier-quick-refs/SKILL.md new file mode 100644 index 000000000..1774dadb1 --- /dev/null +++ b/src/lingtai/tools/web_search/manual/reference/tier-quick-refs/SKILL.md @@ -0,0 +1,189 @@ +--- +name: web-browsing-tier-quick-refs +description: > + Nested web-browsing reference for tier quick-reference commands: Tier 0 PDF, + Tier 1 APIs, Tier 1.5 Trafilatura, Tier 2 BeautifulSoup, Tier 3 Playwright + stealth, Tier 4 Jina/Firecrawl, and Tier 5 AI-native search. +version: 1.0.0 +last_changed_at: "2026-06-01T01:47:09-07:00" +maintenance: "If you find stale or incorrect information here, use the lingtai-issue-report skill to assemble evidence and obtain per-issue human consent before filing an issue. Never include secrets, credentials, tokens, or private paths." +--- + +# Web Browsing Tier Quick References + +Nested web-browsing reference. Open this after the top-level router when you need +manual commands for a specific extraction tier instead of the auto-tier script. + +## Tier 0 — PDF Direct Download (Quick Reference) + +```bash +# Direct PDF link +curl -L "https://arxiv.org/pdf/1706.03762.pdf" -o paper.pdf + +# arXiv ID → derive the PDF path +curl -L "https://arxiv.org/pdf/2401.12345.pdf" -o paper.pdf +``` + +```python +import fitz # pip install pymupdf +doc = fitz.open("paper.pdf") +text = doc[0].get_text() # first page +``` + +→ Full details: [reference/tier-0-pdf.md](../tier-0-pdf.md) + +--- + +## Tier 1 — API Metadata Queries (Quick Reference) + +### Academic APIs at a Glance + +| API | Best for | Free? | +|-----|----------|-------| +| **OpenAlex** | Any DOI → metadata + citations | ✅ | +| **CrossRef** | DOI → title, authors, journal | ✅ | +| **Semantic Scholar** | AI/ML papers, citation graphs | ✅* | +| **arXiv** | CS/Physics/Math papers | ✅ | +| **Unpaywall** | Find free PDF for any DOI | ✅ | +| **PubMed E-utilities** | Biomedical literature | ✅ | +| **CORE** | Open access full text (30M+) | ✅† | +| **Europe PMC** | Biomedical + PMC full text | ✅ | +| **DBLP** | CS conference papers | ✅ | +| **Papers With Code** | ML papers with code + benchmarks | ✅ | +| **DOAJ** | Open access journal articles | ✅ | +| **Zenodo** | Research data, datasets | ✅ | +| **NASA ADS** | Astrophysics/astronomy | ✅ | + +```python +import requests + +# OpenAlex — most powerful, completely free +r = requests.get("https://api.openalex.org/works/https://doi.org/10.1038/s41586-023-05995-9") +data = r.json() + +# Unpaywall — find free PDF +r = requests.get(f"https://api.unpaywall.org/v2/{doi}?email=lingtai@users.noreply.github.com") +if r.json().get("best_oa_location"): + pdf_url = r.json()["best_oa_location"].get("url_for_pdf") +``` + +→ Full details: [reference/tier-1-apis.md](../tier-1-apis.md) +→ Academic search pipeline: [reference/academic-pipeline.md](../academic-pipeline.md) + +--- + +## Tier 1.5 — Trafilatura (Quick Reference) + +```python +import trafilatura + +# Fetch + extract in one call +downloaded = trafilatura.fetch_url("https://example.com/article") +text = trafilatura.extract(downloaded) + +# With metadata +metadata = trafilatura.extract(downloaded, output_format="json", include_metadata=True) + +# Batch extraction +results = [trafilatura.extract(trafilatura.fetch_url(url)) for url in urls] +``` + +→ Full details: [reference/tier-1-5-trafilatura.md](../tier-1-5-trafilatura.md) + +--- + +## Tier 2 — BeautifulSoup (Quick Reference) + +```python +import requests +from bs4 import BeautifulSoup + +resp = requests.get(url, headers={"User-Agent": "Mozilla/5.0..."}) +soup = BeautifulSoup(resp.text, "html.parser") + +# Common patterns +title = soup.find("h1").get_text(strip=True) +links = [a["href"] for a in soup.select("a[href]") if a["href"].startswith("http")] +article = "\n".join(p.get_text() for p in soup.select("article p")) +meta_desc = soup.find("meta", attrs={"name": "description"})["content"] +``` + +→ Full details: [reference/tier-2-beautifulsoup.md](../tier-2-beautifulsoup.md) + +--- + +## Tier 3 — Playwright Stealth (Quick Reference) + +```python +from playwright.sync_api import sync_playwright +try: + from playwright_stealth import Stealth + _apply_stealth = lambda page: Stealth().use_sync(page) +except ImportError: + from playwright_stealth import stealth_sync + _apply_stealth = lambda page: stealth_sync(page) + +with sync_playwright() as p: + browser = p.chromium.launch(headless=True) + ctx = browser.new_context( + user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) ...", + viewport={"width": 1920, "height": 1080}, + ) + page = ctx.new_page() + _apply_stealth(page) # ← 关键:绕过反爬虫检测 + # Block heavy resources + page.route("**/*.{png,jpg,jpeg,gif,svg,woff,woff2}", lambda r: r.abort()) + page.goto(url, wait_until="domcontentloaded") # NOT networkidle! + page.wait_for_timeout(2000) + content = page.content() + browser.close() +``` + +→ Full details: [reference/tier-3-playwright.md](../tier-3-playwright.md) +→ Anti-detection deep-dive: [reference/stealth.md](../stealth.md) + +--- + +## Tier 4 — Jina Reader / Firecrawl (Quick Reference) + +```python +import requests + +# Jina Reader — FREE, no key needed +resp = requests.get(f"https://r.jina.ai/{url}", + headers={"Accept": "text/markdown"}) +markdown = resp.text + +# Firecrawl — production-grade (needs API key) +resp = requests.post("https://api.firecrawl.dev/v1/scrape", + json={"url": url}, + headers={"Authorization": "Bearer YOUR_KEY"}) +``` + +→ Full details: [reference/tier-4-jina-firecrawl.md](../tier-4-jina-firecrawl.md) + +--- + +## Tier 5 — AI-Native Search (Quick Reference) + +```python +# DuckDuckGo — free, no key +from ddgs import DDGS +with DDGS() as ddgs: + results = list(ddgs.text("attention is all you need", max_results=5)) + +# Tavily — search + extract + answer (needs key) +import requests +r = requests.post("https://api.tavily.com/search", + json={"api_key": "tvly-xxx", "query": "transformer paper", "max_results": 5}) + +# Exa — neural/semantic search (needs key) +r = requests.post("https://api.exa.ai/search", + json={"query": "transformer architecture", "numResults": 5}, + headers={"x-api-key": "xxx"}) +``` + +→ Full details: [reference/tier-5-ai-search.md](../tier-5-ai-search.md) +→ Search strategy guide: [reference/search-strategies.md](../search-strategies.md) + +--- diff --git a/src/lingtai/tools/web_search/manual/scripts/cached_get.py b/src/lingtai/tools/web_search/manual/scripts/cached_get.py new file mode 100644 index 000000000..9d918de16 --- /dev/null +++ b/src/lingtai/tools/web_search/manual/scripts/cached_get.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 +""" +cached_get.py — Shared caching utility for web-browsing skill. + +Provides a simple file-based HTTP cache with TTL support. +Use this to avoid hammering APIs and to speed up repeated requests. + +Usage: + from cached_get import cached_get + + # Simple GET with 1-hour cache + response = cached_get("https://api.example.com/data") + + # Custom TTL (seconds) + response = cached_get("https://api.example.com/data", ttl=3600) + + # Force refresh + response = cached_get("https://api.example.com/data", refresh=True) + + # With custom headers + response = cached_get("https://api.example.com/data", + headers={"Accept": "application/json"}) + + # POST request (not cached by default) + response = cached_get("https://api.example.com/submit", + method="POST", json={"key": "value"}) + +Cache location: /tmp/web-browsing-cache/ +""" + +import hashlib +import json +import os +import time +from pathlib import Path + +import requests + +# --- Configuration --- +CACHE_DIR = Path(os.environ.get("WEB_BROWSING_CACHE_DIR", "/tmp/web-browsing-cache")) +DEFAULT_TTL = int(os.environ.get("WEB_BROWSING_CACHE_TTL", "3600")) # 1 hour +USER_AGENT = ( + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" +) + + +def _cache_key(url: str, method: str = "GET") -> str: + """Generate a filesystem-safe cache key from URL and method.""" + h = hashlib.sha256(f"{method.upper()}:{url}".encode()).hexdigest()[:16] + return f"{method.lower()}_{h}.json" + + +def _is_expired(cache_path: Path, ttl: int) -> bool: + """Check if a cached response has expired.""" + if not cache_path.exists(): + return True + mtime = cache_path.stat().st_mtime + return (time.time() - mtime) > ttl + + +def cached_get( + url: str, + *, + method: str = "GET", + headers: dict | None = None, + params: dict | None = None, + json: dict | None = None, + data: dict | None = None, + ttl: int = DEFAULT_TTL, + refresh: bool = False, + timeout: int = 30, + allow_codes: tuple[int, ...] = (200,), + **kwargs, +) -> requests.Response: + """ + HTTP GET/POST with file-based caching. + + Only GET requests with successful status codes are cached. + POST/PUT/DELETE requests are never cached. + + Args: + url: Target URL. + method: HTTP method (default GET). + headers: Custom headers (User-Agent added if not present). + params: Query parameters. + json: JSON body (for POST/PUT). + data: Form data body. + ttl: Cache TTL in seconds (default 1 hour). + refresh: Force cache bypass. + timeout: Request timeout in seconds. + allow_codes: Status codes considered cacheable. + **kwargs: Additional arguments passed to requests.request(). + + Returns: + requests.Response object. + + Raises: + requests.HTTPError: If the response status code is not in allow_codes + and not cached. + """ + # Only cache GET requests + cacheable = method.upper() == "GET" + + if cacheable and not refresh: + CACHE_DIR.mkdir(parents=True, exist_ok=True) + cache_path = CACHE_DIR / _cache_key(url, method) + + if not _is_expired(cache_path, ttl): + try: + with open(cache_path) as f: + cached = json.load(f) + # Reconstruct a Response-like object + resp = requests.Response() + resp.status_code = cached["status_code"] + resp.headers.update(cached["headers"]) + resp._content = cached["body"].encode("utf-8") + resp.encoding = "utf-8" + resp.url = url + resp.reason = "OK" if resp.status_code == 200 else "Cached" + return resp + except (json.JSONDecodeError, KeyError, OSError): + pass # Cache corrupt, re-fetch + + # Build headers + req_headers = {"User-Agent": USER_AGENT} + if headers: + req_headers.update(headers) + + # Make the request + response = requests.request( + method=method.upper(), + url=url, + headers=req_headers, + params=params, + json=json, + data=data, + timeout=timeout, + **kwargs, + ) + + # Cache successful GET responses + if cacheable and response.status_code in allow_codes: + try: + CACHE_DIR.mkdir(parents=True, exist_ok=True) + cache_path = CACHE_DIR / _cache_key(url, method) + with open(cache_path, "w") as f: + json.dump( + { + "status_code": response.status_code, + "headers": dict(response.headers), + "body": response.text, + "url": url, + "cached_at": time.time(), + }, + f, + ensure_ascii=False, + ) + except OSError: + pass # Cache write failure is non-fatal + + return response + + +def clear_cache(url: str | None = None) -> int: + """ + Clear the cache. If url is given, clear only that entry. + Returns the number of entries removed. + """ + if not CACHE_DIR.exists(): + return 0 + + if url: + cache_path = CACHE_DIR / _cache_key(url) + if cache_path.exists(): + cache_path.unlink() + return 1 + return 0 + + # Clear all + count = 0 + for f in CACHE_DIR.glob("*.json"): + f.unlink() + count += 1 + return count + + +if __name__ == "__main__": + import sys + + if len(sys.argv) < 2: + print("Usage: cached_get.py [--refresh] [--ttl SECONDS]") + sys.exit(1) + + url = sys.argv[1] + opts = sys.argv[2:] + + refresh = "--refresh" in opts + ttl = DEFAULT_TTL + if "--ttl" in opts: + idx = opts.index("--ttl") + ttl = int(opts[idx + 1]) + + resp = cached_get(url, refresh=refresh, ttl=ttl) + print(f"Status: {resp.status_code}") + print(f"Cached: {'no' if refresh else 'check /tmp/web-browsing-cache/'}") + print(f"Content length: {len(resp.text)} chars") + print("---") + print(resp.text[:2000]) diff --git a/src/lingtai/tools/web_search/manual/scripts/extract_page.py b/src/lingtai/tools/web_search/manual/scripts/extract_page.py new file mode 100644 index 000000000..7789c3828 --- /dev/null +++ b/src/lingtai/tools/web_search/manual/scripts/extract_page.py @@ -0,0 +1,1159 @@ +#!/usr/bin/env python3 +""" +web-content-extractor / 万能网页内容提取脚本 v3.0 +七层渐进式提取:Tier 0-5 + auto + search + fallback + +用法: + # 自动分层提取 + python3 extract_page.py "https://arxiv.org/abs/1706.03762" + + # 强制指定层 + python3 extract_page.py "https://example.com" --tier 2 + python3 extract_page.py "https://example.com" --tier 1.5 + + # 自动降级链(失败时自动尝试下一层) + python3 extract_page.py "https://example.com" --fallback + + # 搜索模式 + python3 extract_page.py "quantum computing" --search + python3 extract_page.py "quantum computing" --search --search-provider tavily + + # 保存结果 + python3 extract_page.py "https://example.com" --json result.json + +Tier 层级说明: + 0 PDF 直接下载 (curl + fitz) + 1 API 元数据查询 (CrossRef, arXiv, OpenAlex 等) + 1.5 trafilatura 快速提取 (~0.05s/page) + 2 BeautifulSoup 结构化提取 + 3 Playwright stealth JS 渲染 (~3-10s) + 4 Jina Reader API fallback (云端转换) + 5 AI 搜索 (Tavily / Exa) +""" + +import argparse +import json +import os +import re +import sys +import time +import random +import requests +from bs4 import BeautifulSoup +from urllib.parse import urljoin, urlparse, quote_plus + + +# ────────────────────────────────────────────────────────── +# Utility helpers +# ────────────────────────────────────────────────────────── + +def _safe_get(url, headers=None, timeout=15, **kwargs): + """requests.get with default User-Agent and error handling.""" + hdrs = {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36"} + if headers: + hdrs.update(headers) + return requests.get(url, headers=hdrs, timeout=timeout, **kwargs) + + +def _extract_title(soup): + """Extract page title from various meta sources.""" + og = soup.find("meta", property="og:title") + if og and og.get("content"): + return og["content"] + dc = soup.find("meta", attrs={"name": "citation_title"}) + if dc and dc.get("content"): + return dc["content"] + title = soup.find("title") + return title.get_text(strip=True) if title else None + + +def _is_pdf_response(r, url): + """Check if response is a PDF (by Content-Type, .pdf suffix, or /pdf/ path).""" + ct = r.headers.get("Content-Type", "").lower() + u = url.lower() + return "pdf" in ct or u.endswith(".pdf") or "/pdf/" in u + + +# ────────────────────────────────────────────────────────── +# Tier 0: PDF Direct Download +# ────────────────────────────────────────────────────────── + +def tier0(url, save_pdf=None): + """Tier 0: PDF 直链下载 + fitz 文本提取""" + print(f"[Tier 0] PDF 下载: {url}") + try: + r = _safe_get(url, stream=True, timeout=30) + r.raise_for_status() + + if not _is_pdf_response(r, url): + print("[Tier 0] 警告: 目标可能不是 PDF") + + if save_pdf: + with open(save_pdf, "wb") as f: + for chunk in r.iter_content(chunk_size=8192): + f.write(chunk) + print(f"[Tier 0] 已保存到 {save_pdf}") + + # Try fitz for text extraction + try: + import fitz + from io import BytesIO + doc = fitz.open(stream=r.content, filetype="pdf") + n_pages = len(doc) + text = "\n".join(page.get_text() for page in doc) + doc.close() + return {"url": url, "method": "tier0-pdf+fitz", + "text_preview": text[:2000], "pages": n_pages, + "text_length": len(text)} + except ImportError: + return {"url": url, "method": "tier0-curl", + "saved_to": save_pdf, "size": len(r.content)} + + except Exception as e: + return {"url": url, "method": "tier0", "error": str(e)} + + +# ────────────────────────────────────────────────────────── +# Tier 1: API Metadata Queries +# ────────────────────────────────────────────────────────── + +def tier1(url): + """Tier 1: API 元数据查询 (arXiv, CrossRef, OpenAlex, PubMed, Unpaywall)""" + print(f"[Tier 1] API 查询: {url}") + parsed = urlparse(url) + host = parsed.netloc.lower() + u = url.lower() + + # --- arXiv --- + arxiv_match = re.search(r"arxiv\.org/(?:abs|pdf|html)/(\d{4}\.\d{4,5}(?:v\d+)?)", url, re.I) + if arxiv_match or "export.arxiv" in u: + arxiv_id = arxiv_match.group(1) if arxiv_match else re.search(r"id_list=([^\s&]+)", url) + if arxiv_id: + if isinstance(arxiv_id, re.Match): + arxiv_id = arxiv_id.group(1) + api_url = f"https://export.arxiv.org/api/query?id_list={arxiv_id}" + try: + r = _safe_get(api_url, timeout=15) + import xml.etree.ElementTree as ET + root = ET.fromstring(r.text) + ns = {"atom": "http://www.w3.org/2005/Atom"} + entry = root.find("atom:entry", ns) + if entry is not None: + title = entry.find("atom:title", ns) + summary = entry.find("atom:summary", ns) + authors = [a.find("atom:name", ns).text + for a in entry.findall("atom:author", ns)] + pdf_link = None + for link in entry.findall("atom:link", ns): + if link.get("title") == "pdf": + pdf_link = link.get("href") + return { + "url": url, "method": "tier1-arxiv-api", + "title": title.text.strip() if title is not None else None, + "abstract": summary.text.strip()[:1000] if summary is not None else None, + "authors": authors, + "pdf_url": pdf_link, + "arxiv_id": arxiv_id, + } + except Exception as e: + return {"url": url, "method": "tier1-arxiv-api", "error": str(e)} + + # --- DOI → CrossRef --- + doi = _extract_doi(url) + if doi: + # Try CrossRef + result = _crossref_lookup(doi, url) + if result and "error" not in result: + return result + + # --- PubMed --- + pmid_match = re.search(r"pubmed\.ncbi\.nlm\.nih\.gov/(\d+)", url, re.I) + if pmid_match: + pmid = pmid_match.group(1) + try: + api_url = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi" + r = _safe_get(api_url, params={"db": "pubmed", "id": pmid, + "rettype": "abstract", "retmode": "xml"}, + timeout=15) + import xml.etree.ElementTree as ET + root = ET.fromstring(r.text) + article = root.find(".//PubmedArticle/MedlineCitation/Article") + if article is not None: + title_el = article.find("ArticleTitle") + abstract_el = article.find("Abstract/AbstractText") + return { + "url": url, "method": "tier1-pubmed-api", + "pmid": pmid, + "title": title_el.text if title_el is not None else None, + "abstract": abstract_el.text[:1000] if abstract_el is not None else None, + } + except Exception as e: + return {"url": url, "method": "tier1-pubmed-api", "error": str(e)} + + # --- Semantic Scholar --- + if doi: + result = _semantic_scholar_lookup(doi, url) + if result and "error" not in result: + return result + + # --- Fallback: simple request --- + try: + r = _safe_get(url, timeout=10) + soup = BeautifulSoup(r.text, "lxml") + return { + "url": url, "method": "tier1-requests", + "title": _extract_title(soup), + "status": r.status_code, + } + except Exception as e: + return {"url": url, "method": "tier1", "error": str(e)} + + +def _extract_doi(url): + """Extract DOI from URL or page content.""" + # Direct DOI in URL + doi_match = re.search(r"(?:doi\.org/|doi[:/])\s*(10\.\d{4,}/[^\s\"'<>)]+)", url, re.I) + if doi_match: + return doi_match.group(1).rstrip("/") + + # DOI in URL path + doi_match = re.search(r"(10\.\d{4,}/[^\s\"'<>)]+)", url) + if doi_match: + return doi_match.group(1).rstrip("/") + + # Try to fetch and extract from page + try: + r = _safe_get(url, timeout=10) + soup = BeautifulSoup(r.text, "lxml") + for tag in [soup.find("meta", attrs={"name": "citation_doi"}), + soup.find("meta", attrs={"name": "DC.identifier"}), + soup.find("meta", attrs={"name": "dc.identifier"})]: + if tag and tag.get("content") and "10." in tag.get("content", ""): + doi_match = re.search(r"(10\.\d{4,}/[^\s\"'<>)]+)", tag["content"]) + if doi_match: + return doi_match.group(1).rstrip("/") + except Exception: + pass + return None + + +def _crossref_lookup(doi, original_url): + """Look up DOI via CrossRef API.""" + try: + api_url = f"https://api.crossref.org/works/{doi}" + r = _safe_get(api_url, + headers={"User-Agent": "LingTai/3.0 (mailto:lingtai@users.noreply.github.com)"}, + timeout=15) + if r.status_code == 404: + return None + r.raise_for_status() + data = r.json().get("message", {}) + return { + "url": original_url, "method": "tier1-crossref-api", + "doi": doi, + "title": data.get("title", [""])[0], + "authors": [f"{a.get('given', '')} {a.get('family', '')}".strip() + for a in data.get("author", [])], + "journal": data.get("container-title", [""])[0], + "year": str(data.get("published-print", data.get("published-online", {})) + .get("date-parts", [[None]])[0][0]), + "type": data.get("type"), + "abstract": data.get("abstract", "")[:500] if data.get("abstract") else None, + "references_count": len(data.get("reference", [])), + "is_oa": data.get("is-referenced-by-count"), + } + except Exception as e: + return {"url": original_url, "method": "tier1-crossref-api", "error": str(e)} + + +def _semantic_scholar_lookup(doi, original_url): + """Look up DOI via Semantic Scholar API.""" + try: + api_url = f"https://api.semanticscholar.org/graph/v1/paper/DOI:{doi}" + r = _safe_get(api_url, + params={"fields": "title,authors,abstract,citationCount,year,openAccessPdf,tldr"}, + timeout=15) + if r.status_code == 404: + return None + r.raise_for_status() + p = r.json() + return { + "url": original_url, "method": "tier1-semantic-scholar", + "doi": doi, + "title": p.get("title"), + "authors": [a.get("name") for a in p.get("authors", [])], + "abstract": p.get("abstract", "")[:1000], + "citations": p.get("citationCount"), + "year": p.get("year"), + "pdf": (p.get("openAccessPdf") or {}).get("url"), + "tldr": (p.get("tldr") or {}).get("text"), + } + except Exception: + return None + + +# ────────────────────────────────────────────────────────── +# Tier 1.5: trafilatura Fast Extraction +# ────────────────────────────────────────────────────────── + +def tier1_5(url): + """Tier 1.5: trafilatura 快速正文提取 (~0.05s/page)""" + print(f"[Tier 1.5] trafilatura: {url}") + try: + import trafilatura + except ImportError: + return {"url": url, "method": "tier1.5", "error": + "trafilatura 未安装: pip install trafilatura"} + + try: + html = trafilatura.fetch_url(url) + if not html: + return {"url": url, "method": "tier1.5", "error": "trafilatura 返回空内容"} + + # Extract text + text = trafilatura.extract(html, include_comments=False, favor_precision=True) + if not text or len(text.strip()) < 50: + return {"url": url, "method": "tier1.5", + "error": f"trafilatura 提取内容过短 ({len(text or '')} chars)"} + + # Extract metadata — bare_extraction returns a Document, not a dict + doc = trafilatura.bare_extraction(html, include_comments=False) + + # Convert Document to dict if needed (trafilatura may return Document object) + if doc is not None and not isinstance(doc, dict): + try: + metadata = doc.as_dict() + except AttributeError: + # Fallback: access attributes directly + metadata = { + "title": getattr(doc, "title", None), + "author": getattr(doc, "author", None), + "date": getattr(doc, "date", None), + "description": getattr(doc, "description", None), + "categories": getattr(doc, "categories", None), + "tags": getattr(doc, "tags", None), + } + else: + metadata = doc # Already a dict or None + + result = { + "url": url, "method": "tier1.5-trafilatura", + "title": metadata.get("title") if metadata else None, + "author": metadata.get("author") if metadata else None, + "date": metadata.get("date") if metadata else None, + "text": text[:5000], + "text_length": len(text), + } + + # Include full text if requested + if metadata: + result["description"] = metadata.get("description") + result["categories"] = metadata.get("categories") + result["tags"] = metadata.get("tags") + + return result + + except Exception as e: + return {"url": url, "method": "tier1.5", "error": str(e)} + + +# ────────────────────────────────────────────────────────── +# Tier 2: BeautifulSoup Structured Extraction +# ────────────────────────────────────────────────────────── + +def tier2(url): + """Tier 2: curl + BeautifulSoup 结构化提取""" + print(f"[Tier 2] BeautifulSoup: {url}") + try: + r = _safe_get(url, timeout=15) + r.raise_for_status() + soup = BeautifulSoup(r.text, "lxml") + title = _extract_title(soup) + + result = {"url": url, "method": "tier2-bs", "title": title} + + # Extract metadata from meta tags + _extract_meta(soup, result) + + # Extract JSON-LD structured data + jsonld = _extract_jsonld(soup) + if jsonld: + result["jsonld"] = jsonld + + # Extract OpenGraph data + og = _extract_opengraph(soup) + if og: + result["opengraph"] = og + + # Site-specific extraction + u = url.lower() + + if "scholar.google" in u: + _extract_scholar(soup, result) + elif "arxiv.org" in u: + _extract_arxiv(soup, result) + elif "nature.com" in u: + _extract_nature(soup, result) + elif "springer.com" in u or "link.springer" in u: + _extract_springer(soup, result) + elif "reddit.com" in u or "old.reddit.com" in u: + _extract_reddit(soup, result) + elif "github.com" in u: + _extract_github(soup, result) + elif "medium.com" in u: + _extract_medium(soup, result) + else: + # Generic: extract main content + _extract_generic(soup, result) + + return result + + except Exception as e: + return {"url": url, "method": "tier2", "error": str(e)} + + +def _extract_meta(soup, result): + """Extract common meta tags.""" + for name in ["description", "author", "keywords", "citation_doi", + "citation_journal_title", "citation_volume", "citation_issue", + "citation_firstpage", "citation_lastpage", "citation_date"]: + tag = soup.find("meta", attrs={"name": name}) + if tag and tag.get("content"): + key = name.replace("citation_", "").replace("_", "-") + result[f"meta_{key}"] = tag["content"] + + +def _extract_jsonld(soup): + """Extract JSON-LD structured data.""" + scripts = soup.find_all("script", type="application/ld+json") + if not scripts: + return None + data = [] + for s in scripts: + if s.string: + try: + data.append(json.loads(s.string)) + except json.JSONDecodeError: + pass + return data if data else None + + +def _extract_opengraph(soup): + """Extract OpenGraph + Twitter Card metadata.""" + og = {} + for tag in soup.find_all("meta", attrs={"property": True}): + if tag["property"].startswith("og:"): + og[tag["property"]] = tag.get("content", "") + for tag in soup.find_all("meta", attrs={"name": True}): + if tag["name"].startswith("twitter:"): + og[tag["name"]] = tag.get("content", "") + return og if og else None + + +def _extract_scholar(soup, result): + """Google Scholar result extraction.""" + papers = [] + for card in soup.select("div.gs_ri"): + title_el = card.select_one("h3.gs_rt") + abstract_el = card.select_one("div.gs_rs") + link_el = card.select_one("h3.gs_rt a") + pdf_tag = card.select_one("a.gs_or_ggsm") + papers.append({ + "title": title_el.get_text(strip=True) if title_el else None, + "link": link_el["href"] if link_el else None, + "abstract": abstract_el.get_text(strip=True) if abstract_el else None, + "pdf": pdf_tag["href"] if pdf_tag else None, + }) + result["papers"] = papers[:10] + result["count"] = len(papers) + + +def _extract_arxiv(soup, result): + """arXiv page extraction.""" + abstract_el = soup.find("blockquote", class_="abstract") + if abstract_el: + result["abstract"] = abstract_el.get_text(strip=True) + # Try subject areas + subjects = soup.find("span", class_="primary-subject") + if subjects: + result["subject"] = subjects.get_text(strip=True) + + +def _extract_nature(soup, result): + """Nature.com page extraction.""" + og_desc = soup.find("meta", property="og:description") + if og_desc: + result["og_description"] = og_desc["content"] + # Article body + article = soup.find("div", class_="c-article-body") or soup.find("article") + if article: + result["body_preview"] = article.get_text(strip=True)[:2000] + + +def _extract_springer(soup, result): + """Springer page extraction.""" + article = soup.find("article") or soup.find("div", class_="c-article-body") + if article: + result["body_preview"] = article.get_text(strip=True)[:2000] + + +def _extract_reddit(soup, result): + """Reddit page extraction.""" + post = soup.find("div", attrs={"data-testid": "post-container"}) + if not post: + post = soup.find("div", id="siteTable") + if post: + result["post_preview"] = post.get_text(strip=True)[:2000] + + +def _extract_github(soup, result): + """GitHub page extraction.""" + readme = soup.find("article", class_="markdown-body") or soup.find("div", id="readme") + if readme: + result["readme_preview"] = readme.get_text(strip=True)[:3000] + # Star count + star_el = soup.find("span", id="repo-stars-counter-star") + if star_el: + result["stars"] = star_el.get_text(strip=True) + + +def _extract_medium(soup, result): + """Medium page extraction.""" + article = soup.find("article") + if article: + result["body_preview"] = article.get_text(strip=True)[:3000] + + +def _extract_generic(soup, result): + """Generic content extraction: main content areas.""" + # Try common content containers + for selector in ["article", "main", "[role='main']", ".post-content", + ".entry-content", ".article-body", "#content", ".content"]: + container = soup.select_one(selector) + if container: + text = container.get_text(strip=True) + if len(text) > 200: + result["body_preview"] = text[:3000] + result["text_length"] = len(text) + return + + # Last resort: body text + body = soup.find("body") + if body: + result["body_preview"] = body.get_text(strip=True)[:2000] + + +# ────────────────────────────────────────────────────────── +# Tier 3: Playwright Stealth +# ────────────────────────────────────────────────────────── + +def tier3(url, wait_time=3): + """Tier 3: Playwright stealth — JS 渲染 / 保护页面""" + print(f"[Tier 3] Playwright stealth: {url}") + try: + from playwright.sync_api import sync_playwright + try: + from playwright_stealth import Stealth + _stealth_v2 = True + except ImportError: + try: + from playwright_stealth import stealth_sync + _stealth_v2 = False + except ImportError: + raise ImportError("playwright-stealth not installed") + except ImportError: + return {"url": url, "method": "tier3", "error": + "Playwright 未安装: pip install playwright && playwright install chromium && " + "pip install playwright-stealth"} + + try: + with sync_playwright() as p: + browser = p.chromium.launch(headless=True) + page = browser.new_page() + # playwright-stealth v2.0.3+ uses Stealth().use_sync(), old API was stealth_sync(page) + if _stealth_v2: + Stealth().use_sync(page) + else: + stealth_sync(page) + + # Resource blocking for speed + def block_resources(route): + if route.request.resource_type in ["image", "stylesheet", "font", "media"]: + route.abort() + else: + route.continue_() + page.route("**/*", block_resources) + + # ⚠️ Nature/Springer 不用 networkidle,会超时! + page.goto(url, wait_until="domcontentloaded", timeout=30000) + page.wait_for_timeout(wait_time * 1000) + + title = page.title() + content = page.inner_text("body") + html = page.content() + final_url = page.url + browser.close() + + return { + "url": final_url, "method": "tier3-playwright-stealth", + "title": title, + "body_preview": content[:3000], + "text_length": len(content), + "html_len": len(html), + } + except Exception as e: + return {"url": url, "method": "tier3", "error": str(e)} + + +# ────────────────────────────────────────────────────────── +# Tier 4: Jina Reader API Fallback +# ────────────────────────────────────────────────────────── + +def tier4(url): + """Tier 4: Jina Reader API — 云端 URL 转 markdown""" + print(f"[Tier 4] Jina Reader: {url}") + try: + jina_url = f"https://r.jina.ai/{url}" + r = _safe_get(jina_url, + headers={"Accept": "text/markdown", + "X-Return-Format": "markdown"}, + timeout=45) + if r.status_code != 200: + return {"url": url, "method": "tier4-jina", + "error": f"Jina returned HTTP {r.status_code}"} + + text = r.text + if not text or len(text.strip()) < 50: + return {"url": url, "method": "tier4-jina", + "error": f"Jina 返回内容过短 ({len(text)} chars)"} + + # Try to extract title from markdown + title = None + first_line = text.strip().split("\n")[0] if text.strip() else "" + if first_line.startswith("#"): + title = first_line.lstrip("#").strip() + + return { + "url": url, "method": "tier4-jina-reader", + "title": title, + "text": text[:5000], + "text_length": len(text), + } + except Exception as e: + return {"url": url, "method": "tier4", "error": str(e)} + + +# ────────────────────────────────────────────────────────── +# Tier 5: AI Search (Tavily / Exa / DuckDuckGo) +# ────────────────────────────────────────────────────────── + +def tier5(query, provider="duckduckgo", api_key=None, max_results=5): + """Tier 5: AI 搜索引擎 — 发现式搜索""" + print(f"[Tier 5] AI Search ({provider}): {query}") + + if provider == "tavily": + return _search_tavily(query, api_key, max_results) + elif provider == "exa": + return _search_exa(query, api_key, max_results) + elif provider == "duckduckgo": + return _search_duckduckgo(query, max_results) + else: + return {"query": query, "method": f"tier5-{provider}", + "error": f"未知搜索引擎: {provider}"} + + +def _search_duckduckgo(query, max_results=10): + """DuckDuckGo search (free, no key). Supports both ddgs and duckduckgo_search.""" + try: + try: + from ddgs import DDGS # New package name (v9+) + except ImportError: + from duckduckgo_search import DDGS # Legacy name (v8) + with DDGS() as ddgs: + results = list(ddgs.text(query, max_results=max_results)) + return { + "query": query, "method": "tier5-duckduckgo", + "results": [{"title": r["title"], "url": r["href"], + "snippet": r["body"]} for r in results], + "count": len(results), + } + except ImportError: + return {"query": query, "method": "tier5-duckduckgo", "error": + "搜索库未安装: pip install ddgs"} + except Exception as e: + return {"query": query, "method": "tier5-duckduckgo", "error": str(e)} + + +def _search_tavily(query, api_key, max_results=5): + """Tavily AI search.""" + if not api_key: + api_key = os.environ.get("TAVILY_API_KEY") + if not api_key: + return {"query": query, "method": "tier5-tavily", + "error": "需要 Tavily API key (--api-key 或 TAVILY_API_KEY 环境变量)"} + try: + r = requests.post("https://api.tavily.com/search", + json={"api_key": api_key, "query": query, + "search_depth": "advanced", + "include_answer": True, + "max_results": max_results, + "include_raw_content": False}, + timeout=30) + r.raise_for_status() + data = r.json() + return { + "query": query, "method": "tier5-tavily", + "answer": data.get("answer"), + "results": data.get("results", []), + "count": len(data.get("results", [])), + } + except Exception as e: + return {"query": query, "method": "tier5-tavily", "error": str(e)} + + +def _search_exa(query, api_key, max_results=5): + """Exa neural search.""" + if not api_key: + api_key = os.environ.get("EXA_API_KEY") + if not api_key: + return {"query": query, "method": "tier5-exa", + "error": "需要 Exa API key (--api-key 或 EXA_API_KEY 环境变量)"} + try: + r = requests.post("https://api.exa.ai/search", + headers={"x-api-key": api_key}, + json={"query": query, "type": "auto", + "numResults": max_results, + "contents": {"text": {"maxCharacters": 3000}}}, + timeout=30) + r.raise_for_status() + data = r.json() + return { + "query": query, "method": "tier5-exa", + "results": data.get("results", []), + "count": len(data.get("results", [])), + } + except Exception as e: + return {"query": query, "method": "tier5-exa", "error": str(e)} + + +# ────────────────────────────────────────────────────────── +# Auto Tier Decision Engine +# ────────────────────────────────────────────────────────── + +def auto_tier(url): + """ + 自动选择最优提取层。 + 返回 int (0, 1, 2, 3) 或 float (1.5)。 + """ + u = url.lower() + + # PDF — includes both .pdf suffix and arXiv-style /pdf/ID routes (no .pdf suffix) + if u.endswith(".pdf") or "/pdf/" in u: + return 0 + + # Known API endpoints + api_domains = ["arxiv.org/abs", "arxiv.org/html", + "openalex.org", "crossref.org", "export.arxiv", + "pubmed.ncbi", "eutils.ncbi", + "api.core.ac.uk", "api.unpaywall", + "dblp.org/search", "paperswithcode.com/api", + "doaj.org/api", "zenodo.org/api", + "api.semanticscholar"] + for domain in api_domains: + if domain in u: + return 1 + + # DOI redirect + if "doi.org" in u: + return 1 + + # Static content sites → trafilatura (1.5) is best first try + static_sites = ["wikipedia.org", "substack.com", + "blog.", "news.ycombinator.com", "bbc.com", + "nytimes.com", "theguardian.com", + "stackoverflow.com", "stackexchange.com", + "docs.python.org", "docs.google.com"] + for domain in static_sites: + if domain in u: + return 1.5 + + # Sites needing JS rendering → Playwright + js_sites = ["scholar.google", "springer.com", "link.springer", + "sciencedirect.com", "wiley.com", "ieee.org", + "dl.acm.org", "tandfonline.com", + "medium.com", "reuters.com"] + for domain in js_sites: + if domain in u: + return 3 + + # Nature.com → BeautifulSoup works for most pages + if "nature.com" in u: + return 2 + + # Reddit → JSON API if .json, else BS + if "reddit.com" in u: + if u.endswith(".json"): + return 1 + return 2 + + # GitHub → BS works well + if "github.com" in u: + return 2 + + # Default: try trafilatura first (fastest for most content) + return 1.5 + + +# ────────────────────────────────────────────────────────── +# Fallback Chain +# ────────────────────────────────────────────────────────── + +# The escalation order for fallback +TIER_ORDER = [0, 1, 1.5, 2, 3, 4] + + +def extract_with_fallback(url, start_tier=None, max_tier=4, save_pdf=None): + """ + 自动降级提取:从 start_tier 开始,失败时自动尝试下一层。 + """ + if start_tier is None: + start_tier = auto_tier(url) + + # Build tier chain + tier_chain = [t for t in TIER_ORDER if t >= start_tier and t <= max_tier] + + for i, tier in enumerate(tier_chain): + print(f"\n{'='*50}") + print(f"[Fallback {i+1}/{len(tier_chain)}] Trying Tier {tier}") + + if tier == 0: + result = tier0(url, save_pdf=save_pdf) + elif tier == 1: + result = tier1(url) + elif tier == 1.5: + result = tier1_5(url) + elif tier == 2: + result = tier2(url) + elif tier == 3: + result = tier3(url) + elif tier == 4: + result = tier4(url) + else: + continue + + if "error" not in result: + result["fallback_used"] = i + return result + + print(f"[Tier {tier}] 失败: {result['error']}") + + if i < len(tier_chain) - 1: + delay = min(2 ** i + random.random(), 10) + print(f"[等待 {delay:.1f}s 后尝试下一层...]") + time.sleep(delay) + + return {"url": url, "method": "fallback-exhausted", + "error": f"所有层 ({tier_chain}) 均失败"} + + +# ────────────────────────────────────────────────────────── +# CLI +# ────────────────────────────────────────────────────────── + +def main(): + parser = argparse.ArgumentParser( + description="万能网页内容提取 v3.0 — 七层渐进式提取", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +示例: + %(prog)s "https://arxiv.org/abs/1706.03762" # auto + %(prog)s "https://example.com" --tier 1.5 # trafilatura + %(prog)s "https://example.com" --fallback # 自动降级 + %(prog)s "quantum computing" --search # DuckDuckGo 搜索 + %(prog)s "quantum computing" --search --search-provider tavily --api-key KEY + """) + + parser.add_argument("url", help="目标 URL 或搜索关键词") + parser.add_argument("--tier", type=str, default="auto", + help="提取层: 0/1/1.5/2/3/4/5/auto (默认 auto)") + parser.add_argument("--fallback", action="store_true", + help="失败时自动降级到下一层") + parser.add_argument("--save", help="保存 PDF 到文件") + parser.add_argument("--wait", type=int, default=3, + help="Playwright 等待秒数 (默认 3)") + parser.add_argument("--json", dest="json_file", help="保存结果为 JSON") + parser.add_argument("--delay", type=float, default=1.0, + help="请求间隔秒数 (默认 1.0)") + parser.add_argument("--max-tier", type=int, default=4, + help="fallback 最高层 (默认 4)") + parser.add_argument("--search", action="store_true", + help="搜索模式 (URL 参数作为搜索关键词)") + parser.add_argument("--search-provider", default="duckduckgo", + choices=["duckduckgo", "tavily", "exa"], + help="搜索引擎 (默认 duckduckgo)") + parser.add_argument("--api-key", help="搜索 API key (或环境变量)") + parser.add_argument("--max-results", type=int, default=5, + help="搜索结果数 (默认 5)") + + args = parser.parse_args() + + # ── Search mode ── + if args.search: + result = tier5(args.url, provider=args.search_provider, + api_key=args.api_key, max_results=args.max_results) + _print_result(result) + if args.json_file: + _save_json(result, args.json_file) + sys.exit(0 if "error" not in result else 1) + + # ── URL mode ── + url = args.url + + # Normalize bare identifiers into URLs + doi_match = re.match(r'^(10\.\d{4,}/[^\s]+)$', url) + if doi_match: + url = f"https://doi.org/{doi_match.group(1)}" + print(f"[INFO] 裸 DOI 已转换为: {url}") + elif re.match(r'^\d{4}\.\d{4,5}(?:v\d+)?$', url): + url = f"https://arxiv.org/abs/{url}" + print(f"[INFO] 裸 arXiv ID 已转换为: {url}") + elif not url.startswith(("http://", "https://", "ftp://")): + # Treat as search query if no scheme and not a known ID + print(f"[INFO] 非 URL 输入,切换到搜索模式: {url}") + result = tier5(url, provider=args.search_provider, + api_key=args.api_key, max_results=args.max_results) + _print_result(result) + if args.json_file: + _save_json(result, args.json_file) + sys.exit(0 if "error" not in result else 1) + + if args.fallback: + start = auto_tier(url) if args.tier == "auto" else float(args.tier) + result = extract_with_fallback(url, start_tier=start, + max_tier=args.max_tier, + save_pdf=args.save) + else: + # Determine tier + if args.tier == "auto": + chosen = auto_tier(url) + print(f"[INFO] 自动选择 Tier {chosen}") + else: + chosen = float(args.tier) + + # Execute + if chosen == 0: + result = tier0(url, save_pdf=args.save) + elif chosen == 1: + result = tier1(url) + elif chosen == 1.5: + result = tier1_5(url) + elif chosen == 2: + result = tier2(url) + elif chosen == 3: + result = tier3(url, wait_time=args.wait) + elif chosen == 4: + result = tier4(url) + elif chosen == 5: + result = tier5(url, provider=args.search_provider, + api_key=args.api_key, + max_results=args.max_results) + else: + print(f"[错误] 无效 tier: {chosen}") + sys.exit(1) + + _print_result(result) + + if args.json_file: + _save_json(result, args.json_file) + + sys.exit(0 if "error" not in result else 1) + + +def _print_result(result): + """Pretty-print extraction result.""" + print(f"\n{'='*50}") + print(f"[结果] 方法: {result.get('method', 'unknown')}") + + for key in ["title", "doi", "arxiv_id", "pmid"]: + if result.get(key): + print(f"[{key.upper()}] {result[key]}") + + if "papers" in result: + print(f"[论文数] {result.get('count', len(result['papers']))}") + for i, p in enumerate(result.get("papers", [])[:3], 1): + print(f" {i}. {p.get('title', 'N/A')[:80]}") + + if "abstract" in result: + print(f"[摘要] {str(result['abstract'])[:150]}...") + + if "text" in result: + print(f"[正文预览] {result['text'][:200]}...") + print(f"[正文长度] {result.get('text_length', len(result['text']))} 字符") + + if "text_preview" in result: + print(f"[文本预览] {result['text_preview'][:200]}...") + + if "body_preview" in result: + print(f"[正文预览] {result['body_preview'][:200]}...") + + if "pdf_url" in result: + print(f"[PDF] {result['pdf_url']}") + + if "answer" in result and result["answer"]: + print(f"[AI回答] {result['answer'][:300]}...") + + if "results" in result and isinstance(result["results"], list): + print(f"[搜索结果] {result.get('count', len(result['results']))} 条") + for i, r in enumerate(result["results"][:3], 1): + title = r.get("title", "N/A") + print(f" {i}. {title[:80]}") + if r.get("url") or r.get("href"): + print(f" {r.get('url') or r.get('href')}") + + if "fallback_used" in result: + print(f"[降级次数] {result['fallback_used']}") + + if "error" in result: + print(f"[错误] {result['error']}") + + +def _save_json(result, path): + """Save result to JSON file.""" + with open(path, "w", encoding="utf-8") as f: + json.dump(result, f, ensure_ascii=False, indent=2) + print(f"[INFO] 结果已保存到 {path}") + + +# ────────────────────────────────────────────────────────── +# Smoke Tests +# ────────────────────────────────────────────────────────── + +def _smoke_test(): + """Smoke test: auto_tier 决策 + import 检查 + 浊输入/畸响应验证""" + import traceback + + # ── Part 1: auto_tier 决策 (净例 + 浊例) ── + cases = [ + # 净例(标准 URL) + ("https://arxiv.org/abs/1706.03762", 1), + ("https://arxiv.org/pdf/1706.03762.pdf", 0), + ("https://arxiv.org/html/2401.12345", 1), + ("https://scholar.google.com/scholar?q=solar", 3), + ("https://www.nature.com/articles/s41586-023-05995-9", 2), + ("https://link.springer.com/article/10.12942/lrr-2014-3", 3), + ("https://doi.org/10.1038/s41586-023-05995-9", 1), + ("https://en.wikipedia.org/wiki/Quantum_computing", 1.5), + ("https://medium.com/some-article", 3), + ("https://github.com/user/repo", 2), + ("https://www.reddit.com/r/programming/hot", 2), + ("https://www.reddit.com/r/programming/hot.json", 1), + ("https://stackoverflow.com/questions/12345", 1.5), + ("https://example.com/some-page", 1.5), + ("https://docs.python.org/3/library/os.html", 1.5), + # 浊例(实测踩坑的边界 URL) + ("https://arxiv.org/pdf/1706.03762", 0), # 无 .pdf 后缀 + ("https://arxiv.org/pdf/2401.12345v2", 0), # 有版本号,无后缀 + ("https://reuters.com/technology/", 3), # 需 JS + ("https://www.reuters.com/article/some-id", 3), + ] + all_ok = True + for url, expected in cases: + got = auto_tier(url) + if got != expected: + print(f"FAIL auto_tier({url!r}): expected {expected}, got {got}") + all_ok = False + + print("[TEST] auto_tier: " + ("all passed" if all_ok else "SOME FAILED")) + + # ── Part 2: trafilatura bare_extraction 返回类型验证 ── + try: + import trafilatura + print("[TEST] trafilatura: OK") + # 浊测试:bare_extraction 是否返回 dict(而非 Document) + try: + sample_html = "Test

Hello world content here.

" + doc = trafilatura.bare_extraction(sample_html, include_comments=False) + if doc is not None: + if isinstance(doc, dict): + print("[TEST] trafilatura bare_extraction: returns dict ✅") + else: + # Document object — verify our conversion code handles it + try: + _ = doc.as_dict() + print(f"[TEST] trafilatura bare_extraction: returns {type(doc).__name__}, .as_dict() works ✅") + except AttributeError: + print(f"[TEST] trafilatura bare_extraction: returns {type(doc).__name__}, attribute fallback needed ✅") + except Exception as e: + print(f"[TEST] trafilatura bare_extraction: ERROR {e} ❌") + except ImportError: + print("[TEST] trafilatura: NOT INSTALLED (tier 1.5 unavailable)") + + # ── Part 3: playwright-stealth API 兼容性 ── + try: + from playwright.sync_api import sync_playwright + try: + from playwright_stealth import Stealth # v2.0.3+ + print("[TEST] playwright+stealth (v2 API): OK") + except ImportError: + from playwright_stealth import stealth_sync # legacy + print("[TEST] playwright+stealth (legacy API): OK") + except ImportError: + print("[TEST] playwright+stealth: NOT INSTALLED (tier 3 unavailable)") + + # ── Part 4: DuckDuckGo search 包名兼容 ── + try: + try: + from ddgs import DDGS # New package name + except ImportError: + from duckduckgo_search import DDGS # Legacy + print("[TEST] ddgs/duckduckgo-search: OK") + except ImportError: + print("[TEST] ddgs/duckduckgo-search: NOT INSTALLED (tier 5 DDG unavailable)") + + # ── Part 5: fitz (pymupdf) ── + try: + import fitz + print("[TEST] pymupdf (fitz): OK") + except ImportError: + print("[TEST] pymupdf (fitz): NOT INSTALLED (PDF text extraction unavailable)") + + # ── Part 6: 浊输入 — _is_pdf_response 边界 ── + class FakeResponse: + def __init__(self, ct): + self.headers = {"Content-Type": ct} + pdf_edge_ok = True + # 正常 PDF + if not _is_pdf_response(FakeResponse("application/pdf"), "https://example.com/p.pdf"): + print("FAIL _is_pdf_response: normal PDF missed"); pdf_edge_ok = False + # URL 含 /pdf/ 但无 .pdf 后缀(arXiv 实际 URL) + if not _is_pdf_response(FakeResponse("text/html"), "https://arxiv.org/pdf/1706.03762"): + print("FAIL _is_pdf_response: arXiv /pdf/ URL missed"); pdf_edge_ok = False + # 非 PDF + if _is_pdf_response(FakeResponse("text/html"), "https://example.com/page"): + print("FAIL _is_pdf_response: false positive on non-PDF"); pdf_edge_ok = False + # Content-Type 含 pdf 但 URL 不含 + if not _is_pdf_response(FakeResponse("application/pdf;charset=utf-8"), "https://cdn.example.com/download/abc123"): + print("FAIL _is_pdf_response: Content-Type PDF missed"); pdf_edge_ok = False + print("[TEST] _is_pdf_response edges: " + ("all passed" if pdf_edge_ok else "SOME FAILED")) + + # ── Part 7: 浊输入 — _extract_doi 边界 ── + doi_ok = True + for url, expected_doi in [ + ("https://doi.org/10.1038/s41586-023-05995-9", "10.1038/s41586-023-05995-9"), + ("https://doi.org/10.1109/iccv48922.2021.00041", "10.1109/iccv48922.2021.00041"), + ]: + got = _extract_doi(url) + if got != expected_doi: + print(f"FAIL _extract_doi({url!r}): expected {expected_doi!r}, got {got!r}") + doi_ok = False + print("[TEST] _extract_doi: " + ("all passed" if doi_ok else "SOME FAILED")) + + print("[TEST] All smoke tests complete.") + + +if __name__ == "__main__": + if "--test" in sys.argv: + _smoke_test() + sys.exit(0) + main() diff --git a/src/lingtai/tools/write/__init__.py b/src/lingtai/tools/write/__init__.py index 791804170..1f78e1311 100644 --- a/src/lingtai/tools/write/__init__.py +++ b/src/lingtai/tools/write/__init__.py @@ -7,23 +7,25 @@ from typing import TYPE_CHECKING from .._file_paths import resolve_workdir_path +from .._manual import load_installed_manual if TYPE_CHECKING: from lingtai.kernel.base_agent import BaseAgent def get_description(lang: str = "en") -> str: - return 'Create or overwrite a file with the given content. Parent directories are created automatically. Use this for creating new files or complete rewrites. For small changes to existing files, prefer edit.' + return "Create or overwrite a file with the given content. Parent directories are created automatically. Use this for creating new files or complete rewrites. For small changes to existing files, prefer edit. Call write(action='manual') to return the installed file-manual skill." def get_schema(lang: str = "en") -> dict: return { "type": "object", "properties": { - "file_path": {"type": "string", "description": 'Absolute path to the file to write'}, - "content": {"type": "string", "description": 'Content to write'}, + "action": {"type": "string", "enum": ["manual"], "description": "Use action='manual' to return the installed file-manual skill without writing a file."}, + "file_path": {"type": "string", "description": "Absolute path to the file to write. Required for ordinary writes; omit for action='manual'."}, + "content": {"type": "string", "description": "Content to write. Required for ordinary writes; omit for action='manual'."}, }, - "required": ["file_path", "content"], + "required": [], } @@ -32,10 +34,14 @@ def setup(agent: "BaseAgent") -> None: """Set up the write capability on an agent.""" def handle_write(args: dict) -> dict: + if args.get("action") == "manual": + return load_installed_manual(agent, "file-manual") path = args.get("file_path", "") - content = args.get("content", "") if not path: return {"status": "error", "message": "file_path is required"} + if "content" not in args: + return {"status": "error", "message": "content is required"} + content = args["content"] path = resolve_workdir_path(agent, path) try: agent._file_io.write(path, content) diff --git a/tests/test_bash_async.py b/tests/test_bash_async.py index 9ee85d290..beba86cf0 100644 --- a/tests/test_bash_async.py +++ b/tests/test_bash_async.py @@ -264,7 +264,7 @@ def test_async_stderr_captured(self, tmp_path): def test_schema_requires_reminder_with_runtime_default(self, tmp_path): schema = get_schema() - assert "reminder" in schema["required"] + assert "reminder" not in schema["required"] # manual has no action-specific inputs assert schema["properties"]["reminder"]["default"] == 1800.0 mgr = self._make_manager(tmp_path) diff --git a/tests/test_intrinsic_manual_actions.py b/tests/test_intrinsic_manual_actions.py new file mode 100644 index 000000000..3ea4363f3 --- /dev/null +++ b/tests/test_intrinsic_manual_actions.py @@ -0,0 +1,158 @@ +"""Focused coverage for built-in tools that expose installed manual skills.""" +from __future__ import annotations + +from pathlib import Path + +from lingtai.tools import daemon as daemon_tool +from lingtai.tools import edit as edit_tool +from lingtai.tools import email as email_tool +from lingtai.tools import glob as glob_tool +from lingtai.tools import grep as grep_tool +from lingtai.tools import psyche as psyche_tool +from lingtai.tools import read as read_tool +from lingtai.tools import soul as soul_tool +from lingtai.tools import system as system_tool +from lingtai.tools import write as write_tool +from lingtai.tools import web_search as web_search_tool +from lingtai.tools import bash as shell_tool + + +class _StubAgent: + def __init__(self, working_dir: Path): + self._working_dir = working_dir + self.handlers: dict[str, object] = {} + + def add_tool(self, name: str, *, handler=None, **_kwargs) -> None: + self.handlers[name] = handler + + +def _install_manual(workdir: Path, skill_name: str) -> tuple[str, Path]: + path = ( + workdir + / ".library" + / "intrinsic" + / "capabilities" + / skill_name + / "SKILL.md" + ) + path.parent.mkdir(parents=True, exist_ok=True) + body = f"---\nname: {skill_name}\n---\n\n# {skill_name} sentinel\n" + path.write_text(body, encoding="utf-8") + return body, path + + +def test_manual_actions_return_their_installed_skills(tmp_path: Path) -> None: + agent = _StubAgent(tmp_path) + expected = { + skill: _install_manual(tmp_path, skill) + for skill in ( + "shell", + "daemon", + "email", + "psyche-manual", + "read-manual", + "soul-manual", + "system-manual", + "web_search", + "file-manual", + ) + } + + read_tool.setup(agent) + write_tool.setup(agent) + edit_tool.setup(agent) + glob_tool.setup(agent) + grep_tool.setup(agent) + + shell_manager = shell_tool.ShellManager.__new__(shell_tool.ShellManager) + shell_manager._agent = agent + daemon_manager = daemon_tool.DaemonManager.__new__(daemon_tool.DaemonManager) + daemon_manager._agent = agent + web_search_manager = web_search_tool.WebSearchManager.__new__(web_search_tool.WebSearchManager) + web_search_manager._agent = agent + + calls = { + "shell": ("shell", lambda: shell_manager.handle({"action": "manual"})), + "daemon": ("daemon", lambda: daemon_manager.handle({"action": "manual"})), + "email": ("email", lambda: email_tool.handle(agent, {"action": "manual"})), + "psyche": ("psyche-manual", lambda: psyche_tool.handle(agent, {"action": "manual"})), + "read": ("read-manual", lambda: agent.handlers["read"]({"action": "manual"})), + "soul": ("soul-manual", lambda: soul_tool.handle(agent, {"action": "manual"})), + "system": ("system-manual", lambda: system_tool.handle(agent, {"action": "manual"})), + "web_search": ("web_search", lambda: web_search_manager.handle({"action": "manual"})), + "write": ("file-manual", lambda: agent.handlers["write"]({"action": "manual"})), + "edit": ("file-manual", lambda: agent.handlers["edit"]({"action": "manual"})), + "glob": ("file-manual", lambda: agent.handlers["glob"]({"action": "manual"})), + "grep": ("file-manual", lambda: agent.handlers["grep"]({"action": "manual"})), + } + + for tool_name, (skill_name, call) in calls.items(): + body, path = expected[skill_name] + assert call() == { + "status": "ok", + "manual": body, + "manual_path": str(path), + }, tool_name + + +def test_manual_schemas_preserve_runtime_checks_for_ordinary_file_calls( + tmp_path: Path, +) -> None: + modules = ( + shell_tool, + daemon_tool, + email_tool, + psyche_tool, + read_tool, + soul_tool, + system_tool, + web_search_tool, + write_tool, + edit_tool, + glob_tool, + grep_tool, + ) + for module in modules: + schema = module.get_schema() + action = schema["properties"]["action"] + assert "manual" in action.get("enum", ()) or "manual" in action["description"] + + assert shell_tool.get_schema()["required"] == [] + assert psyche_tool.get_schema()["required"] == ["action"] + assert web_search_tool.get_schema()["required"] == [] + for module in (read_tool, write_tool, edit_tool, glob_tool, grep_tool): + assert module.get_schema()["required"] == [] + + agent = _StubAgent(tmp_path) + for module in (read_tool, write_tool, edit_tool, glob_tool, grep_tool): + module.setup(agent) + + assert agent.handlers["read"]({})["message"] == "file_path is required" + assert agent.handlers["write"]({"file_path": str(tmp_path / "x")})["message"] == "content is required" + assert agent.handlers["edit"]({"file_path": str(tmp_path / "x"), "old_string": "a"})["message"] == "new_string is required" + assert agent.handlers["glob"]({})["message"] == "pattern is required" + assert agent.handlers["grep"]({})["message"] == "pattern is required" + assert not (tmp_path / "x").exists() + + +def test_missing_installed_manual_degrades_without_side_effects(tmp_path: Path) -> None: + agent = _StubAgent(tmp_path) + expected_path = ( + tmp_path + / ".library" + / "intrinsic" + / "capabilities" + / "system-manual" + / "SKILL.md" + ) + + assert system_tool.handle(agent, {"action": "manual"}) == { + "status": "degraded", + "manual": "", + "manual_path": str(expected_path), + "error": ( + "system-manual manual missing — initializer may have failed or " + "capability not installed correctly" + ), + } + assert not (tmp_path / ".library").exists() diff --git a/tests/test_skills.py b/tests/test_skills.py index f1d4838e0..c65c32e90 100644 --- a/tests/test_skills.py +++ b/tests/test_skills.py @@ -184,6 +184,13 @@ def test_skills_setup_hard_copies_intrinsics(tmp_path): assert "reference/scheduled-work/SKILL.md" in bash_body assert "reference/notification-reminders/SKILL.md" in bash_body assert "reference/debugging-cleanup/SKILL.md" in bash_body + + web_search_root = ( + workdir / ".library" / "intrinsic" / "capabilities" / "web_search" + ) + assert "name: web-search-manual" in (web_search_root / "SKILL.md").read_text(encoding="utf-8") + assert (web_search_root / "scripts" / "extract_page.py").is_file() + assert (web_search_root / "reference" / "tier-quick-refs" / "SKILL.md").is_file() for moved_reference in ( "reference/bash-claude-code/SKILL.md", "reference/bash-openai-codex/SKILL.md", diff --git a/tests/test_soul.py b/tests/test_soul.py index c73565f75..0bebe1c0d 100644 --- a/tests/test_soul.py +++ b/tests/test_soul.py @@ -188,7 +188,7 @@ def test_schema_exposes_five_actions(self): # at runtime), voice (agent picks/customizes own soul-flow # prompt — read or set), and dismiss (clear soul notification). assert schema["properties"]["action"]["enum"] == [ - "inquiry", "flow", "config", "voice", "dismiss", + "inquiry", "flow", "config", "voice", "dismiss", "manual", ] def test_schema_inquiry_property_present(self): diff --git a/tests/test_tools_package_data.py b/tests/test_tools_package_data.py index 014f57922..933813aa5 100644 --- a/tests/test_tools_package_data.py +++ b/tests/test_tools_package_data.py @@ -75,6 +75,36 @@ "lingtai/intrinsic_skills/notification-manual/reference/dismissal-safety/SKILL.md", ) +# The three per-tool glossary languages that each package must ship. +_WEB_SEARCH_MANUAL_FILES = ( + "lingtai/tools/web_search/manual/SKILL.md", + "lingtai/tools/web_search/manual/assets/api-endpoints.json", + "lingtai/tools/web_search/manual/assets/css-selectors.json", + "lingtai/tools/web_search/manual/assets/extraction-pipeline.json", + "lingtai/tools/web_search/manual/assets/regex-patterns.json", + "lingtai/tools/web_search/manual/assets/search-providers.json", + "lingtai/tools/web_search/manual/assets/site-templates.json", + "lingtai/tools/web_search/manual/reference/academic-pipeline.md", + "lingtai/tools/web_search/manual/reference/maintenance-bundles/SKILL.md", + "lingtai/tools/web_search/manual/reference/migration-from-v2.md", + "lingtai/tools/web_search/manual/reference/news-and-rss.md", + "lingtai/tools/web_search/manual/reference/realtime-data.md", + "lingtai/tools/web_search/manual/reference/routing-and-sites/SKILL.md", + "lingtai/tools/web_search/manual/reference/search-strategies.md", + "lingtai/tools/web_search/manual/reference/social-media.md", + "lingtai/tools/web_search/manual/reference/stealth.md", + "lingtai/tools/web_search/manual/reference/tier-0-pdf.md", + "lingtai/tools/web_search/manual/reference/tier-1-5-trafilatura.md", + "lingtai/tools/web_search/manual/reference/tier-1-apis.md", + "lingtai/tools/web_search/manual/reference/tier-2-beautifulsoup.md", + "lingtai/tools/web_search/manual/reference/tier-3-playwright.md", + "lingtai/tools/web_search/manual/reference/tier-4-jina-firecrawl.md", + "lingtai/tools/web_search/manual/reference/tier-5-ai-search.md", + "lingtai/tools/web_search/manual/reference/tier-quick-refs/SKILL.md", + "lingtai/tools/web_search/manual/scripts/cached_get.py", + "lingtai/tools/web_search/manual/scripts/extract_page.py", +) + # The three per-tool glossary languages that each package must ship. _GLOSSARY_LANGS = ("en", "zh", "wen") @@ -167,6 +197,11 @@ def test_wheel_ships_vision_manual(wheel_entries: set[str]): assert "lingtai/tools/vision/manual/SKILL.md" in wheel_entries +def test_wheel_ships_complete_web_search_manual_bundle(wheel_entries: set[str]): + missing = [path for path in _WEB_SEARCH_MANUAL_FILES if path not in wheel_entries] + assert not missing, "web_search manual files missing from wheel: %r" % missing + + def test_wheel_ships_exact_expected_tool_contracts(wheel_entries: set[str]): # Keep the manifest closed: the 18 top-level tool contracts plus the one # intentional daemon component contract. No other nested/manual contract @@ -312,6 +347,11 @@ def test_sdist_ships_every_glossary_resource(sdist_entries: set[str]): assert not missing, "glossary resources missing from sdist: %r" % missing +def test_sdist_ships_complete_web_search_manual_bundle(sdist_entries: set[str]): + missing = [path for path in _WEB_SEARCH_MANUAL_FILES if path not in sdist_entries] + assert not missing, "web_search manual files missing from sdist: %r" % missing + + def test_sdist_ships_exactly_54_glossary_resources(sdist_entries: set[str]): glossary_files = { e