From d236850e0ed4d07db77c8915b6ef182613b9adb4 Mon Sep 17 00:00:00 2001 From: Aspetta <892024285@qq.com> Date: Tue, 30 Jun 2026 18:42:00 +0800 Subject: [PATCH 1/8] Add AgentReach external discovery backend --- .github/workflows/trend-radar-daily.yml | 2 + .github/workflows/trend-radar-weekly.yml | 1 + .gitignore | 4 + README.en.md | 6 +- README.md | 6 +- data/README.md | 6 + ...-external-discovery-and-evidence-v0.1.json | 8 + ...-external-discovery-and-evidence-design.md | 26 +- ...very-and-evidence-named-actors.addendum.md | 110 ++++++ ...l-discovery-and-evidence-v0.1.exec-plan.md | 150 ++++++-- .../externalDiscoveryAdapter.test.ts | 287 +++++++++++++++ .../externalDiscoveryAggregate.test.ts | 114 ++++++ .../externalDiscoveryEntityRegistry.test.ts | 61 ++++ .../externalDiscoveryMatching.test.ts | 66 ++++ .../externalDiscoveryRedaction.test.ts | 75 ++++ .../externalDiscoveryStructure.test.ts | 40 ++ .../externalDiscoveryTypeContract.test.ts | 25 ++ src/externalDiscovery/agentReachProvider.ts | 342 ++++++++++++++++++ src/externalDiscovery/aggregate.ts | 186 ++++++++++ src/externalDiscovery/entityRegistry.ts | 153 ++++++++ src/externalDiscovery/matching.ts | 54 +++ src/externalDiscovery/paths.ts | 21 ++ src/externalDiscovery/redaction.ts | 86 +++++ src/externalDiscovery/types.ts | 136 +++++++ src/storage/files.ts | 1 + 25 files changed, 1927 insertions(+), 39 deletions(-) create mode 100644 docs/specs/agent-work/code-implementation-preflight.agent-reach-external-discovery-and-evidence-v0.1.json create mode 100644 docs/specs/exec-plans/agent-reach-external-discovery-and-evidence-named-actors.addendum.md create mode 100644 src/__tests__/externalDiscoveryAdapter.test.ts create mode 100644 src/__tests__/externalDiscoveryAggregate.test.ts create mode 100644 src/__tests__/externalDiscoveryEntityRegistry.test.ts create mode 100644 src/__tests__/externalDiscoveryMatching.test.ts create mode 100644 src/__tests__/externalDiscoveryRedaction.test.ts create mode 100644 src/__tests__/externalDiscoveryStructure.test.ts create mode 100644 src/__tests__/externalDiscoveryTypeContract.test.ts create mode 100644 src/externalDiscovery/agentReachProvider.ts create mode 100644 src/externalDiscovery/aggregate.ts create mode 100644 src/externalDiscovery/entityRegistry.ts create mode 100644 src/externalDiscovery/matching.ts create mode 100644 src/externalDiscovery/paths.ts create mode 100644 src/externalDiscovery/redaction.ts create mode 100644 src/externalDiscovery/types.ts diff --git a/.github/workflows/trend-radar-daily.yml b/.github/workflows/trend-radar-daily.yml index 8b80124..1706fa1 100644 --- a/.github/workflows/trend-radar-daily.yml +++ b/.github/workflows/trend-radar-daily.yml @@ -133,6 +133,8 @@ jobs: data/raw/github/${{ steps.options.outputs.target_date }}.enrichment.json data/raw/github-stars/${{ steps.options.outputs.target_date }}.json data/raw/github-stars/tracked-repos.json + data/external-discovery/${{ steps.options.outputs.target_date }}.aggregate.json + data/external-discovery/latest.aggregate.json data/kb/latest.json if-no-files-found: warn diff --git a/.github/workflows/trend-radar-weekly.yml b/.github/workflows/trend-radar-weekly.yml index 4c159ca..8efb17a 100644 --- a/.github/workflows/trend-radar-weekly.yml +++ b/.github/workflows/trend-radar-weekly.yml @@ -109,5 +109,6 @@ jobs: data/reports/${{ steps.options.outputs.target_date }}.weekly.json data/reports/${{ steps.options.outputs.target_date }}.weekly.md data/reports/${{ steps.options.outputs.target_date }}.weekly.audit.json + data/external-discovery/*.aggregate.json data/kb/latest.json if-no-files-found: warn diff --git a/.gitignore b/.gitignore index ee87224..92e8b69 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,7 @@ data/agent-memory/** !data/agent-memory/.gitkeep data/upstream/** !data/upstream/.gitkeep +data/raw/external-discovery/** +!data/raw/external-discovery/ +!data/raw/external-discovery/fixtures/ +!data/raw/external-discovery/fixtures/** diff --git a/README.en.md b/README.en.md index d09d1ad..c998ce8 100644 --- a/README.en.md +++ b/README.en.md @@ -104,7 +104,11 @@ AgentRadar connects those steps into one workflow and stores the results as insp The repository includes a lightweight local read-only web console for browsing generated artifacts. -### 6. Hosted online app +### 6. AgentReach external discovery boundary + +The open-source edition only consumes local AgentReach JSON artifacts. It does not include platform login, account state, cookies, sessions, OAuth, or private provider diagnostics. `data/raw/external-discovery/` is local-only input by default; only public-safe `data/external-discovery/*.aggregate.json` files may be committed. + +### 7. Hosted online app - Hosted URL: [`https://app.agentradar.top/`](https://app.agentradar.top/) - Browse the homepage, project library, weekly trends, run health, and emerging-project views directly diff --git a/README.md b/README.md index f037c9c..356028a 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,11 @@ AgentRadar 把这些步骤连成一套完整工作流，并把结果沉淀成每 仓库内开源版自带一个轻量本地 Web Console，用来浏览已生成产物。 -### 6. 在线托管版本 +### 6. AgentReach 外部发现边界 + +开源版只消费本地 AgentReach JSON artifact，不内置平台登录、账号态、cookie、session、OAuth 或私有 provider diagnostics。`data/raw/external-discovery/` 默认只作为本地输入目录；可公开提交的是通过脱敏校验的 `data/external-discovery/*.aggregate.json`。 + +### 7. 在线托管版本 - 在线地址：[`https://app.agentradar.top/`](https://app.agentradar.top/) - 可直接查看首页、项目库、本周趋势、数据状态和新兴潜力项目 diff --git a/data/README.md b/data/README.md index 22c89e3..f9de376 100644 --- a/data/README.md +++ b/data/README.md @@ -10,15 +10,21 @@ This repository intentionally commits public historical artifacts under `data/`. - `data/scores/` - `data/reports/` - `data/kb/` +- `data/external-discovery/*.aggregate.json` These files are part of the project deliverable. They let readers inspect historical runs, compare scoring behavior over time, and review the generated reports without reproducing every upstream fetch. ## What is not version-controlled - `data/upstream/` +- `data/raw/external-discovery/` `data/upstream/` is reserved for optional local scratch checkouts or caches, such as an explicit self-hosted `agents-radar` mirror. It is ignored by Git and is not part of the public artifact history. +`data/raw/external-discovery/` is local-only AgentReach provider input. It may contain imported platform references or private runner context and must not be committed or uploaded by default. Only sanitized fixtures under `data/raw/external-discovery/fixtures/` may be version-controlled. + +Public external discovery history belongs in `data/external-discovery/*.aggregate.json`. These aggregates must be public-safe: no raw social text, no profile URLs, no raw handles, no cookie/session/token/OAuth material, and no public `*.events.jsonl` default artifact. + ## Automation behavior The daily and weekly GitHub Actions workflows update the tracked public artifacts and commit them back into this repository. This is intentional: the repo is both code and a public historical data log. diff --git a/docs/specs/agent-work/code-implementation-preflight.agent-reach-external-discovery-and-evidence-v0.1.json b/docs/specs/agent-work/code-implementation-preflight.agent-reach-external-discovery-and-evidence-v0.1.json new file mode 100644 index 0000000..d627ec6 --- /dev/null +++ b/docs/specs/agent-work/code-implementation-preflight.agent-reach-external-discovery-and-evidence-v0.1.json @@ -0,0 +1,8 @@ +{ + "skill_path": "docs/specs/agent-work/CodeImplementation_Skill.md", + "skill_sha256": "a15bb25d65ff2b12867a3255812d109d0821d6838a89b232392cbfaa70b7a95d", + "exec_plan_path": "docs/specs/exec-plans/agent-reach-external-discovery-and-evidence-v0.1.exec-plan.md", + "exec_plan_sha256": "e539dadd9aa4f55c5cf29cc0fc6e15a310e126b7a4ae7a13747d1aed1636a4ca", + "generated_at": "2026-06-30T04:31:29.714Z", + "acknowledged": true +} diff --git a/docs/specs/design-docs/agent-reach-external-discovery-and-evidence-design.md b/docs/specs/design-docs/agent-reach-external-discovery-and-evidence-design.md index 396f7e1..3cd1b1a 100644 --- a/docs/specs/design-docs/agent-reach-external-discovery-and-evidence-design.md +++ b/docs/specs/design-docs/agent-reach-external-discovery-and-evidence-design.md @@ -2,7 +2,8 @@ ## 文档状态 -- 状态：`Draft for Review` +- 状态：`Approved for ExecPlan` +- 批准备注：`2026-06-30 design review 通过；需求文档已冻结 V1 平台、按日/7 日窗口、头部人群机构/团队/个人三类主体、次级信号边界和降级语义。本文已冻结 AgentReach 本地 artifact 消费、public-safe aggregate、entity registry tier、named_registry_actors 具名讨论者契约、daily / weekly / verify 消费边界，可进入 ExecPlan。` - 对应需求：`docs/specs/product-specs/外部发现与补证信号层需求分析.md` - 需求状态：`Frozen for Design` - 设计范围：外部发现与补证信号层，即 `external discovery & evidence layer` @@ -357,6 +358,17 @@ interface ExternalSignalEvent { `ExternalEvidence` 是给项目级或方向级消费的外部证据摘要。该模型只表达外部讨论与补证信号，不等同于 `ScoreComponent.evidence`、`WeeklyEvidenceProject` 或项目成熟度事实。 ```ts +interface ExternalNamedRegistryActor { + entity_id: string; + display_name: string; + actor_type: "institution" | "team" | "person"; + registry_tier: ExternalRegistryTier; + event_count: number; + platforms: ExternalPlatform[]; + first_seen_at: string; + last_seen_at: string; +} + interface ExternalEvidence { evidence_id: string; event_ids: string[]; @@ -364,6 +376,7 @@ interface ExternalEvidence { target_key: string; derived_signal_kinds: ExternalSignalKind[]; platforms: ExternalPlatform[]; + named_registry_actors: ExternalNamedRegistryActor[]; actor_tiers: Partial>; actor_types: Partial>; mention_count: number; @@ -381,6 +394,15 @@ interface ExternalEvidence { `actor_tiers` 必须按事件 actor 的 `effective_tier` 聚合，不能按 `provider_tier_hint` 聚合。若 registry 未命中，作者只能计入 `ordinary` 或 `unknown`。 +`named_registry_actors` 是 UI 展示“谁在讨论”的唯一具名数据入口。该字段只能由满足以下条件的事件聚合生成： + +- 事件 actor 存在 `registry_entity_id`、`registry_tier`，且 `tier_basis=registry`。 +- `display_name` 来自 entity registry 或等价 public-safe canonical entity，不得直接复制 provider 原始 `actor.display_name`。 +- 字段内不得包含 handle、profile URL、raw social text 或私有 diagnostics。 +- 未命中 registry 的 actor 不得进入该列表，只能进入 `actor_tiers` / `actor_types` / `distinct_actor_count` 等匿名统计。 + +消费层展示时应优先使用 `named_registry_actors` 渲染 `具名讨论者`；当该列表为空时，只展示 `actor_tiers`、`actor_types` 和 `distinct_actor_count` 形成的匿名摘要。 + ### 4.3 `ObservationCandidate` `ObservationCandidate` 是外部层可以生成但不能直接升级为主结论的观察对象。 @@ -1011,7 +1033,7 @@ data/external-discovery/entities/ # entity registry 或 tier registry，若拆 - 不改变现有主 score 公式。 - 不新增未同步 spec / config / tests 的 score component。 - 不让 LLM 直接决定外部层是否构成主趋势。 -- 不创建或更新 exec-plan，直到设计获得明确批准。 +- 未获新的设计批准前，不创建或更新超出本文冻结范围的 exec-plan。 ## 18. 设计自检清单 diff --git a/docs/specs/exec-plans/agent-reach-external-discovery-and-evidence-named-actors.addendum.md b/docs/specs/exec-plans/agent-reach-external-discovery-and-evidence-named-actors.addendum.md new file mode 100644 index 0000000..8b50424 --- /dev/null +++ b/docs/specs/exec-plans/agent-reach-external-discovery-and-evidence-named-actors.addendum.md @@ -0,0 +1,110 @@ +# 临时 Addendum：AgentReach “谁在讨论”具名讨论者契约 + +## 文档状态 + +- 状态：`Merged into Main ExecPlan / Design Review Passed` +- 适用范围：`AgentReach External Discovery & Evidence` 中“谁在讨论”的具名讨论者展示与 public aggregate 契约。 +- 关联需求：`docs/specs/product-specs/外部发现与补证信号层需求分析.md` +- 关联设计： + - `docs/specs/design-docs/agent-reach-external-discovery-and-evidence-design.md` + - `docs/specs/design-docs/trend-radar-ui-v3-stage-redesign-design.md` +- 关联主计划：`docs/specs/exec-plans/agent-reach-external-discovery-and-evidence-v0.1.exec-plan.md` + +本文件原为临时补充。2026-06-30 设计复审通过后，成熟内容已合入主 exec-plan；本文件仅作为审计记录保留，不再作为独立实现入口。实现阶段仍应以主 exec-plan 为准，不得仅凭本 addendum 直接写代码。 + +## 背景 + +需求要求系统能表达“谁在讨论”，并区分机构、团队、个人和普通社区讨论层级。原主 exec-plan 已覆盖 entity registry、tier 统计和 public aggregate，但当前公开聚合契约主要表达 actor tier/type/count，尚不足以让 UI 安全展示具名讨论者。 + +为避免 UI 从 provider raw actor 字段、未脱敏事件或自由文本中临时拼接身份，本 addendum 暂存一个候选执行补充：由聚合层产出 public-safe 的 `named_registry_actors`，UI 只消费该字段。 + +## 设计复审确认结果 + +1. 允许 public aggregate 展示 registry 命中的 `display_name + actor_type + registry_tier`，前提是 `display_name` 来自 entity registry 或等价 public-safe canonical entity。 +2. `named_registry_actors` 作为 `ExternalEvidence` 的正式字段进入 V1 contract。 +3. 空 registry、registry miss、仅 provider tier hint 时，UI 只展示匿名摘要。 +4. 具名讨论者需要在 daily、weekly、项目详情和外部发现页复用同一字段。 +5. 相关设计文档已更新为 `Approved for ExecPlan`；本补充已合入主 exec-plan，后续复审主 exec-plan。 + +## 候选契约 + +```ts +interface ExternalNamedRegistryActor { + entity_id: string; + display_name: string; + actor_type: "institution" | "team" | "person"; + registry_tier: ExternalRegistryTier; + event_count: number; + platforms: ExternalPlatform[]; + first_seen_at: string; + last_seen_at: string; +} + +interface ExternalEvidence { + named_registry_actors: ExternalNamedRegistryActor[]; +} +``` + +`named_registry_actors` 是“谁在讨论”的唯一具名数据入口。字段为空时，不代表没有讨论，只代表没有可 public-safe 具名展示的 registry 命中讨论者。 + +## 生成规则 + +1. 仅 `tier_basis="registry"` 且存在 `registry_entity_id` / `registry_tier` 的 canonical event 可以进入 `named_registry_actors`。 +2. `display_name` 必须来自 entity registry 或等价 public-safe canonical entity，不得直接复制 provider raw `actor.display_name`。 +3. 同一 `ExternalEvidence(scope + target_key)` 内按 `entity_id` 合并。 +4. `event_count` 统计该 entity 对当前 evidence 的参与事件数。 +5. `platforms` 去重排序。 +6. `first_seen_at` / `last_seen_at` 来自该 entity 的事件时间窗口。 +7. 输出按 `registry_tier` 优先级 `core -> proven -> watch`、再按 `event_count` 降序、再按 `display_name` 稳定排序。 + +## 禁止项 + +- provider raw `actor.display_name` 不得直接成为可信身份。 +- raw handle、profile URL、raw social text、provider diagnostics、private metadata 不得进入 `named_registry_actors`。 +- 未命中 registry、仅有 `provider_tier_hint`、仅有 provider display name 的 actor 不得具名展示。 +- provider hint 不得伪装成 registry tier。 +- UI 不得从 provider raw actor 字段、未脱敏事件或自由文本里临时拼接“具名讨论者”。 + +## 匿名降级 + +1. 空 registry：`named_registry_actors=[]`，所有 actor 只能进入 `ordinary / unknown` 或匿名统计；`registry_tier_participation` 不成立。 +2. 单个 actor registry miss：该 actor 不贡献 `core/proven/watch`，但不影响同一 aggregate 中其他 registry 命中 actor 的统计。 +3. 混合 hit/miss：`top_tier_actor_count` 只统计 registry 命中的 `core / proven / watch` actor；miss actor 仅进入 `actor_tiers` / `actor_types` / `distinct_actor_count`。 +4. UI 展示：`named_registry_actors` 非空时展示 `具名讨论者`；为空时只展示 `actor_type / effective_tier / count` 匿名摘要。 + +## 已合入主 exec-plan 的位置 + +设计复审通过后，本补充已按以下位置合入主 exec-plan： + +1. `Phase 1：类型、路径与 redaction 基座` + - 增加 `ExternalNamedRegistryActor`。 + - 要求 `ExternalEvidence.named_registry_actors` 存在。 + - 增加 public-safe 字段测试。 +2. `Phase 3：entity registry、matching 与 topic canonicalization` + - 增加 registry 命中到 public-safe actor match 的规则。 + - 增加 raw display name / handle / profile URL 不得进入具名身份的测试。 + - 澄清 mixed hit/miss 下 `top_tier_actor_count` 统计口径。 +3. `Phase 4：aggregate 与 public artifact 写入` + - 增加 `named_registry_actors` 聚合规则。 + - 增加空 registry / miss / provider hint 负例。 +4. `Phase 6：daily / run-summary / verify 输出` + - 要求 daily / run-summary 的 external evidence summary 保留或派生“具名讨论者 + 匿名汇总”。 + - verify 检查 public aggregate 不含未脱敏身份字段。 +5. `Phase 7：weekly 7 日窗口消费` + - 要求 weekly direction / project evidence summary 复用同一 `named_registry_actors` 契约。 +6. `验收标准 / 验证矩阵` + - 增加“具名讨论者只来自 registry 命中，未命中只匿名统计”的通过标准。 + +## 合入验收口径 + +本 addendum 已在以下条件满足后合入主 exec-plan： + +1. 关联设计文档完成复审，并明确允许 public-safe 具名展示 registry 命中实体。 +2. 主 exec-plan 的设计来源与实际设计状态一致。 +3. `provider_run_id` 可选/必填契约与设计文档对齐，不在本补充中顺手收紧。 +4. Phase 6/7 的消费输出承接补齐，避免只生成 aggregate 但用户看不到“谁在讨论”。 +5. 合入后重新运行 ExecPlan Review。 + +## 当前结论 + +`named_registry_actors` 已从候选方向升级为主 exec-plan 承接的 V1 契约补充。本文件只保留补充来源和边界说明；代码实现必须等待主 exec-plan 复审通过后再开始。 diff --git a/docs/specs/exec-plans/agent-reach-external-discovery-and-evidence-v0.1.exec-plan.md b/docs/specs/exec-plans/agent-reach-external-discovery-and-evidence-v0.1.exec-plan.md index 5101a8c..481ff52 100644 --- a/docs/specs/exec-plans/agent-reach-external-discovery-and-evidence-v0.1.exec-plan.md +++ b/docs/specs/exec-plans/agent-reach-external-discovery-and-evidence-v0.1.exec-plan.md @@ -3,12 +3,27 @@ ## 文档状态 - 版本：`v0.1` -- 当前状态：`Draft` +- 当前状态：`Approved for Implementation` - 设计来源： - `docs/specs/product-specs/外部发现与补证信号层需求分析.md` - `docs/specs/design-docs/agent-reach-external-discovery-and-evidence-design.md` + - `docs/specs/design-docs/trend-radar-ui-v3-stage-redesign-design.md` - 说明：本计划只负责把已获批设计落到可执行实现阶段，不新增产品行为，不直接调用外部平台 API，不创建登录、OAuth、session 或账号配置能力。 +## 设计复审记录 + +- 复审日期：`2026-06-30` +- 复审结论：AgentReach external discovery 设计与 UI V3 外部发现页补充设计均已更新为 `Approved for ExecPlan`。 +- 本次合入范围：`ExternalEvidence.named_registry_actors`、`ExternalNamedRegistryActor`、registry/public-safe 具名讨论者生成规则、匿名降级规则，以及 daily / weekly / 项目详情 / 外部发现页复用同一“谁在讨论”契约。 +- 边界：本合入不引入新的 provider、平台采集、账号态、登录态、主评分公式或 UI 侧重算逻辑；UI 只能消费 public aggregate 中的 `named_registry_actors` 与匿名统计，不得从 provider raw actor 字段、未脱敏事件或自由文本临时拼接身份。本文中的“项目详情 / 外部发现页复用”指本计划必须产出同一 public contract 和消费语义，具体 visual console 页面代码若不在本计划触达面内，应由后续 UI exec-plan 承接，不得在本计划内顺手扩权。 + +## ExecPlan 复审记录 + +- 复审日期：`2026-06-30` +- 复审结论：`APPROVE` +- 风险等级：`High`，原因是 external discovery 涉及 public artifact、redaction、registry tier、daily / weekly 消费和 verify 门禁；当前计划已用 Phase 0 结构测试、contract-first 类型、public-safe 验证、负例测试和回滚策略约束风险。 +- 无阻塞项：计划已基于批准设计，承接 `named_registry_actors` 具名讨论者契约，明确禁止 provider raw actor 身份拼接，并保留 preflight-sync 未完成前不得开始生产代码实现的入口门禁。 + ## 任务信息 | 字段 | 内容 | @@ -33,6 +48,7 @@ - 冻结 `ExternalPlatform = "x_twitter" | "reddit" | "hacker_news" | "official_web" | "official_blog"`。 - 冻结 `ExternalTargetType = "project" | "paper" | "product" | "topic"`；`direction` 只能作为 `scope` / consumption 语义，不得作为 target type。 - 冻结 `derived_signal_kinds: ExternalSignalKind[]`，允许同一事件同时具备 `discovery` 与 `evidence`。 + - 定义 `ExternalNamedRegistryActor`，并要求 `ExternalEvidence.named_registry_actors` 作为“谁在讨论”的唯一 public-safe 具名数据入口。 - 定义 `ObservationCandidate`，并固定 `cannot_be_primary_conclusion: true`。 - 只承接 external layer，不扩展 `RawSignal.source`。 @@ -43,6 +59,7 @@ - `src/externalDiscovery/redaction.ts` - 实现 public-safe 字段扫描、raw text/profile URL/token/cookie/session/password/OAuth 字样拦截、public aggregate 验证。 + - 验证 `named_registry_actors` 只包含 `entity_id`、`display_name`、`actor_type`、`registry_tier`、`event_count`、`platforms`、`first_seen_at`、`last_seen_at`，不得包含 raw handle、profile URL、raw social text、provider diagnostics 或 private metadata。 - 输出稳定 reason code，供 adapter、aggregate、verify 和结构测试复用。 - `src/externalDiscovery/agentReachProvider.ts` @@ -56,6 +73,7 @@ - `src/externalDiscovery/entityRegistry.ts` - 读取 `data/external-discovery/entity-registry.json`，registry 可空启动。 - 只有 registry 命中的实体可产生 `registry_tier=core/proven/watch` 与 `effective_tier=core/proven/watch`。 + - 只有 `tier_basis="registry"` 且存在 `registry_entity_id` / `registry_tier` 的 canonical event，可参与后续 `named_registry_actors` 生成。 - 空 registry 或 registry miss 不阻塞 external layer，但必须记录 `registry_empty` / `registry_miss` warning，且 `top_tier_actor_count=0`。 - 覆盖机构、团队、个人三类主体，不得收缩为仅机构账号。 @@ -72,6 +90,7 @@ - 固定 `public_safe=true`、`contains_raw_text=false`、`contains_profile_urls=false`、`redaction_policy_version` 和 `source_input_hash`。 - 只写 public aggregate 与 latest 指针；V1 不写 public `events.jsonl`。 - 拒绝把 `content_text`、provider raw `text`、`profile_url`、未脱敏 handle 或 private diagnostics 写入 public aggregate。 + - 为每个 `ExternalEvidence(scope + target_key)` 聚合 `named_registry_actors`，只使用 registry/public-safe canonical entity 的 `display_name`，并按 `registry_tier` 优先级、`event_count`、`display_name` 稳定排序。 - `src/externalDiscovery/dailyIntegration.ts` - 封装 `run-daily` / `recover-daily` 的 external discovery orchestration。 @@ -99,6 +118,7 @@ - `src/action/dailyReport.ts` - 增加 `external_discovery` section 到 daily JSON / Markdown。 + - daily external section 展示“具名讨论者 + 匿名/未命中汇总”，具名讨论者只能来自 `ExternalEvidence.named_registry_actors`。 - 外部候选和补证只作为次级观察展示，不进入 `today_star_projects` 的主源确认语义。 - `src/action/runSummary.ts` @@ -108,6 +128,7 @@ - `src/action/weeklyEnhancement.ts`、`src/action/weeklyReport.ts` - weekly 只消费 7 日 aggregate window。 - 输出 direction observation、project evidence summary 和 external window status。 + - weekly direction / project evidence summary 复用同一 `named_registry_actors` 契约，不新增第二套具名讨论者字段。 - `src/action/dailyVerification.ts` - skipped / failed 作为 warn。 @@ -174,22 +195,22 @@ - 需求分析：`Approved` - 设计文档：`Approved` -- 实施：`Not Started` -- 验证：`Not Started` +- 实施：`Phase 0-2 Done; Phase 3/4/8 In Progress` +- 验证：`Phase 0-4 named_registry_actors 基座通过 targeted tests / typecheck / preflight check` ## 阶段进度 | 阶段 | 状态 | 目标 | 完成标志 | | --- | --- | --- | --- | -| Phase 0：执行前对齐 | `Pending` | 固定文件边界、fixture 规则、preflight-sync 和测试入口 | 结构测试先失败，且实现前 preflight-sync 已完成并记录结果 | -| Phase 1：类型、路径与 redaction 基座 | `Pending` | 建立 frozen enum、类型、路径和 public-safe 验证 | redaction / path / type 单测通过 | -| Phase 2：AgentReach 本地 artifact adapter | `Pending` | 只消费本地 JSON artifact，输出 canonical event/audit/status | adapter 单测覆盖 provider schema 与 ok / skipped / partial / failed | -| Phase 3：entity registry、matching 与 topic canonicalization | `Pending` | 落地 tier 维护、项目匹配与方向 topic 归一 | registry / matching / topic 测试通过 | -| Phase 4：aggregate 与 public artifact 写入 | `Pending` | 生成 public daily aggregate 和 latest 指针，不落盘 public events JSONL | aggregate 不含 raw social text 且 public-safe 测试通过 | +| Phase 0：执行前对齐 | `DONE` | 固定文件边界、fixture 规则、preflight-sync 和测试入口 | preflight receipt 已刷新并校验；structure test 通过 | +| Phase 1：类型、路径与 redaction 基座 | `DONE` | 建立 frozen enum、类型、路径和 public-safe 验证 | redaction / path / type 单测通过 | +| Phase 2：AgentReach 本地 artifact adapter | `DONE` | 只消费本地 JSON artifact，输出 canonical event/audit/status | adapter 单测覆盖 provider schema、顶层审计字段、非空 `derived_signal_kinds` 与 ok / skipped / partial / failed | +| Phase 3：entity registry、matching 与 topic canonicalization | `IN_PROGRESS` | 落地 tier 维护、项目匹配与方向 topic 归一 | 本轮已完成 registry named actor eligibility、repo 精确匹配和 topic key 基座；低置信 audit 细化留后续 | +| Phase 4：aggregate 与 public artifact 写入 | `IN_PROGRESS` | 生成 public daily aggregate 和 latest 指针，不落盘 public events JSONL | 本轮已完成 in-memory public aggregate 与 `named_registry_actors` 聚合；文件写入/latest 指针接入留后续 | | Phase 5：CLI command matrix | `Pending` | 落地 `run-daily`、`recover-daily`、`run-weekly`、`verify-daily` flags 语义 | CLI 单测覆盖 flags、fail-fast、dry-run | | Phase 6：daily / run-summary / verify 输出 | `Pending` | daily 展示、run-summary 审计、verify warn/fail 与污染检测 | action output 与 verification 测试通过 | | Phase 7：weekly 7 日窗口消费 | `Pending` | weekly 只读 `DailyExternalAggregate[]`，direction gate 4 选 2 | weekly 与 direction gate 测试通过 | -| Phase 8：OSS 文档、gitignore、workflow、spec 同步 | `Pending` | 公共 artifact 策略在文档、spec 和自动化中一致 | structure test 与人工 diff review 通过 | +| Phase 8：OSS 文档、gitignore、workflow、spec 同步 | `IN_PROGRESS` | 公共 artifact 策略在文档、spec 和自动化中一致 | 本轮已完成 `.gitignore`、README、data README、workflow 的 external raw/public aggregate 边界；spec 全量同步留后续 | | Phase 9：总体验收 | `Pending` | typecheck、test、preflight 和关键 CLI dry-run 全部通过 | 验证记录更新到本计划 | ## 实施阶段 @@ -233,6 +254,7 @@ - `ExternalRegistryTier = "core" | "proven" | "watch"` - `ExternalProviderTierHint = "core" | "proven" | "watch" | "ordinary" | "unknown"` - `ExternalSignalEvent` + - `ExternalNamedRegistryActor` - `ExternalEvidence` - `ObservationCandidate` - `DailyExternalAggregate` @@ -241,6 +263,8 @@ - `derived_signal_kinds` 必须是非空数组，允许同时包含 `discovery` 与 `evidence`。 - `direction` 不得出现在 `ExternalTargetType`，方向级表达只能来自 `scope="direction"` 与 `target_type="topic"`。 - `ObservationCandidate.cannot_be_primary_conclusion` 固定为 `true`。 + - `ExternalNamedRegistryActor.actor_type` 只能是 `"institution" | "team" | "person"`，不得包含 `community` 或 `unknown`。 + - `ExternalEvidence.named_registry_actors` 必须存在；为空数组表示没有可 public-safe 具名展示的 registry 命中讨论者，不代表没有外部讨论。 3. 在 `src/types.ts` 只追加公开消费类型或 re-export，不改变 `RawSignal`、`ScoreBreakdown`、`ScoreComponentName`。 4. 在 `src/externalDiscovery/paths.ts` 定义： - `externalRawInputPath(date: string): string` @@ -262,6 +286,8 @@ - raw `content_text` 被 public aggregate redaction 拒绝。 - provider raw `text` 被 public aggregate redaction 拒绝。 - `profile_url` 被 public aggregate redaction 拒绝。 + - `named_registry_actors` 中出现 raw handle、profile URL、raw social text、provider diagnostics 或 private metadata 时被拒绝。 + - `named_registry_actors.display_name` 只能来自 registry/public-safe canonical entity，测试 fixture 不得使用 provider raw `actor.display_name` 作为可信身份来源。 - cookie/session/token/password/OAuth 字样被 public aggregate redaction 拒绝。 - sanitized aggregate 包含 `public_safe=true`、`contains_raw_text=false`、`contains_profile_urls=false`、`redaction_policy_version`、`source_input_hash` 时通过。 9. 运行： @@ -331,6 +357,8 @@ - 未命中 actor 产生 `registry_miss` warning。 - 空 registry 或 miss 时，actor 只能计入 `ordinary` 或 `unknown`。 - `top_tier_actor_count` 必须为 0，除非 registry 命中 `core/proven/watch`。 + - 只有 `tier_basis="registry"` 且具备 `registry_entity_id` / `registry_tier` 的 canonical event，可进入后续 `named_registry_actors` 聚合。 + - provider `actor.display_name`、`provider_tier_hint`、handle 或 profile URL 不得替代 registry 命中。 2. Registry entry 必须覆盖： - `entity_id` - `display_name` @@ -343,35 +371,39 @@ - `handles` 与 `profile_urls` 只能保存 curated public entity reference。 - registry 不得包含私有账号态字段、cookie、token、session、OAuth、私有 diagnostics 或 provider raw profile dump。 - registry 中的 handle / profile URL 不得复制进 `DailyExternalAggregate` 或 public aggregate。 - - public aggregate 只能保留计数、脱敏 actor summary、tier 统计和 audit 状态。 + - public aggregate 只能保留计数、脱敏 actor summary、tier 统计、`named_registry_actors` 中的 public-safe registry identity 和 audit 状态。 - 若 `data/external-discovery/entity-registry.json` 作为公开 artifact 维护，必须被 structure / public-safe 测试覆盖。 4. 在 `src/externalDiscovery/matching.ts` 实现项目与方向匹配： - repo URL 精确命中现有 `NormalizedProject.repo_url` 时，生成 `ExternalEvidence(scope=project)`。 - paper / product 明确 URL 可以生成 project-scope 或 evidence candidate，但不得伪造 repo。 - 低置信名称匹配默认不进入 daily 主展示，只能 rejected 或保留为 low-confidence audit evidence。 - 未命中但有明确 repo / paper / product 时，生成 `ObservationCandidate(scope=project, qualification=needs_primary_confirmation)`。 -4. 固定 topic canonicalization： +5. 固定 topic canonicalization： - 优先命中 `userInterestProfile.topics[].name`。 - 再命中现有 weekly trend key / paradigm label。 - 再使用 provider `topic_hint` 的小写 kebab-case。 - 无稳定 `topic_key` 的 topic event 不得进入 weekly direction observation。 -5. 固定 `ObservationCandidate` 字段： +6. 固定 `ObservationCandidate` 字段： - `candidate_kind` - `qualification` - `can_enter_daily` - `can_enter_weekly` - `cannot_be_primary_conclusion=true` - direction candidate 必须说明“尚未绑定明确 repo / paper”。 -6. 写测试覆盖： +7. 写测试覆盖： - 空 registry 不阻塞 external layer。 - 空 registry 时 `top_tier_actor_count=0`。 - provider tier hint 不得产生 `effective_tier=core/proven/watch`。 - registry miss 记录 warning。 + - registry 命中 actor 可进入 `named_registry_actors`，且 `display_name` 来自 registry。 + - registry miss 或仅 provider hint actor 不得进入 `named_registry_actors`。 + - mixed hit/miss 时，命中 actor 仍可进入 `named_registry_actors` 和 top-tier 统计，miss actor 只进入匿名统计。 + - raw display name / handle / profile URL 不得进入具名身份输出。 - repo URL 精确匹配生成 project evidence。 - low-confidence name match 不进入 daily 主展示。 - topic event 必须有稳定 `topic_key`。 - 无稳定 `topic_key` 的 topic event 不进入 weekly direction observation。 -7. 运行： +8. 运行： - `pnpm test -- externalDiscoveryEntityRegistry.test.ts` - `pnpm test -- externalDiscoveryMatching.test.ts` - `pnpm typecheck` @@ -402,26 +434,39 @@ - `observation_candidates` - `audit.rejected_events` - `audit.warnings` -3. Aggregate 不得包含： +3. 每个 `ExternalEvidence` 必须包含 `named_registry_actors`： + - 同一 `ExternalEvidence(scope + target_key)` 内按 `entity_id` 合并。 + - `event_count` 统计该 entity 对当前 evidence 的参与事件数。 + - `platforms` 去重排序。 + - `first_seen_at` / `last_seen_at` 来自该 entity 在当前 evidence 内的事件时间窗口。 + - 输出按 `registry_tier` 优先级 `core -> proven -> watch`、再按 `event_count` 降序、再按 `display_name` 稳定排序。 + - 字段为空数组时只表示没有可具名展示的 registry 命中 actor，不表示没有外部讨论。 +4. Aggregate 不得包含： - raw social text - full provider text - profile URL + - raw handle + - provider raw `actor.display_name` 作为可信身份来源 - cookie/token/session/password/OAuth - private diagnostics -4. V1 artifact 策略： +5. V1 artifact 策略： - 只写 `data/external-discovery/YYYY-MM-DD.aggregate.json`。 - 只写 `data/external-discovery/latest.aggregate.json`。 - 不写 `data/external-discovery/YYYY-MM-DD.events.jsonl`。 - 如果未来需要 canonical events JSONL，必须另开设计或 exec-plan 修订。 -5. 在 `src/storage/files.ts` 增加 `data/external-discovery` 到 `DATA_DIRS`。 -6. 在 daily integration 中只写 public aggregate 和 latest 指针，不写 raw input。 -7. 写测试覆盖： +6. 在 `src/storage/files.ts` 增加 `data/external-discovery` 到 `DATA_DIRS`。 +7. 在 daily integration 中只写 public aggregate 和 latest 指针，不写 raw input。 +8. 写测试覆盖： - public aggregate 通过 redaction check。 - aggregate 包含 `source_input_hash`，但不包含 raw input 原文。 - aggregate 不生成 events JSONL。 - dry-run 只报告 planned writes，不创建 aggregate/latest 文件。 - actor tier counts 使用 `effective_tier`，不使用 `provider_tier_hint`。 -8. 运行： + - `named_registry_actors` 只由 registry 命中 actor 生成。 + - 空 registry、registry miss、仅 provider hint 时 `named_registry_actors=[]`。 + - mixed hit/miss 时命中 actor 保留，miss actor 只进入匿名统计。 + - raw display name / handle / profile URL / raw social text 不得进入 `named_registry_actors` 或 public aggregate。 +9. 运行： - `pnpm test -- externalDiscoveryAggregate.test.ts` - `pnpm test -- externalDiscoveryRedaction.test.ts` - `pnpm typecheck` @@ -470,11 +515,14 @@ - `external_project_evidence_summaries` - `external_direction_signal_summary` - `external_audit_summary` + - `who_is_discussing` 或等价消费字段，来源只能是 `ExternalEvidence.named_registry_actors` 与匿名 actor summary。 3. Daily 输出语义： - `external_observation_candidates` 只放 `scope=project` 且 `can_enter_daily=true` 的外部观察候选。 - external project candidate 不得进入 `today_star_projects`。 - direction candidate 不得伪装成“今日新项目”。 - skipped / failed 时列表字段为空数组，`status_reason` 必须说明原因。 + - `named_registry_actors` 非空时展示具名讨论者；为空时只展示 `actor_type / effective_tier / count` 匿名摘要。 + - Daily Markdown 不得从 provider raw actor 字段、未脱敏事件或自由文本里拼接具名讨论者。 4. 在 `DailyRunSummary` 中追加 external secondary layer 状态： - provider status - aggregate path @@ -487,6 +535,8 @@ - daily 声称使用 external layer 但 audit 缺失：fail。 - public aggregate 含未脱敏 raw 字段：fail。 - public aggregate 缺少 required public-safe 字段：fail。 + - `named_registry_actors` 含 raw handle、profile URL、raw social text、provider diagnostics、private metadata 或 provider raw display name：fail。 + - daily / run-summary 输出出现非 `named_registry_actors` 来源的具名讨论者：fail。 - skipped / failed external layer：warn。 6. Verify score contamination 检测必须覆盖： - `RawSignal.source` 没有新增 external provider。 @@ -500,6 +550,8 @@ - verify 对 redaction violation fail。 - verify 对 score contamination fail。 - provider tier hint 不参与头部讨论统计。 + - daily external section 能同时表达具名讨论者和匿名/未命中汇总。 + - 未命中 registry 的普通账号只进入匿名汇总，不得显示成具体个人或机构。 8. 运行： - `pnpm test -- externalDiscoveryActionOutput.test.ts` - `pnpm test -- externalDiscoveryVerification.test.ts` @@ -517,6 +569,7 @@ - `weekly_direction_observations` - `external_project_evidence_summaries` - `external_cross_platform_confirmations` + - 基于 `named_registry_actors` 派生的 `who_is_discussing` 摘要，字段来源和 daily 保持一致。 4. Weekly direction gate 固定为 4 个收敛条件至少满足 2 个： - `cross_platform_confirmation`：同一 `topic_key` 在至少 2 个 V1 平台出现。 - `multi_actor_confirmation`：同一 `topic_key` 至少有 2 个独立 actor 讨论。 @@ -530,12 +583,16 @@ - 方向级 observation 不成为项目级结论。 - external cross-platform confirmation 不等同主源多源确认。 - external evidence 不创建高置信主趋势。 + - weekly direction / project evidence summary 中的具名讨论者只能来自 `ExternalEvidence.named_registry_actors`。 + - `named_registry_actors=[]` 时保留匿名 actor summary，不得把空列表解释成“没有讨论”。 7. 写测试覆盖： - weekly 只读 aggregate paths。 - weekly 不访问 `data/raw/external-discovery/`。 - 7 日窗口部分缺失仍生成 weekly，并标记 usable day count。 - direction observation 必须满足 4 个收敛条件中的至少 2 个。 - 仅 provider tier hint 不满足 registry tier participation。 + - weekly 复用 daily aggregate 中的 `named_registry_actors`，不读取 provider raw input 重新拼接身份。 + - registry miss actor 在 weekly 中只进入匿名汇总。 8. 运行： - `pnpm test -- externalDiscoveryWeekly.test.ts` - `pnpm test -- externalDiscoveryDirectionGate.test.ts` @@ -621,25 +678,30 @@ 12. `ExternalPlatform`、`ExternalTargetType`、`derived_signal_kinds` 与设计冻结契约一致。 13. Entity registry 可空启动；空 registry 不产生 top-tier 统计。 14. Provider tier hint 不得产生 core / proven / watch 头部判断。 -15. Weekly direction observation 必须满足 4 个收敛条件中的至少 2 个。 -16. V1 不默认落盘 public canonical events JSONL。 -17. Phase 8 列出的 specs、README、`data/README.md`、`.gitignore` 与 workflows 全部同步。 -18. 实现前 preflight-sync 已完成并记录结果，不再悬空。 +15. `ExternalEvidence.named_registry_actors` 必须存在，且只由 registry/public-safe canonical entity 命中的 actor 生成。 +16. `named_registry_actors` 不得包含 raw handle、profile URL、raw social text、provider diagnostics、private metadata 或 provider raw display name。 +17. 空 registry、registry miss、仅 provider tier hint 时，UI / daily / weekly 只能展示匿名 actor summary，不得展示具名讨论者。 +18. mixed hit/miss 时，registry 命中 actor 仍可进入具名列表和 top-tier 统计，miss actor 只进入匿名统计。 +19. Daily / weekly / 项目详情 / 外部发现页复用同一 `named_registry_actors` public contract；本计划只保证 artifact / report / summary 层提供统一消费语义，不新增第二套具名讨论者字段或 UI 侧拼接逻辑。 +20. Weekly direction observation 必须满足 4 个收敛条件中的至少 2 个。 +21. V1 不默认落盘 public canonical events JSONL。 +22. Phase 8 列出的 specs、README、`data/README.md`、`.gitignore` 与 workflows 全部同步。 +23. 实现前 preflight-sync 已完成并记录结果，不再悬空。 ## 验证矩阵 | 文件位置或类型 | 验证内容 | 验证方式或命令 | 对应 Spec | 通过标准 | | --- | --- | --- | --- | --- | -| `src/externalDiscovery/types.ts` | frozen enum / target / derived signal contract | `pnpm test -- externalDiscoveryTypeContract.test.ts` | 设计 4.1、18 | 平台、target、derived signal 与设计一致 | -| `src/externalDiscovery/redaction.ts` | public-safe / redaction 规则 | `pnpm test -- externalDiscoveryRedaction.test.ts` | 设计 3.6、11、12、14 | 禁止字段全部 fail，sanitized aggregate pass | +| `src/externalDiscovery/types.ts` | frozen enum / target / derived signal / `named_registry_actors` contract | `pnpm test -- externalDiscoveryTypeContract.test.ts` | 设计 4.1、4.2、18 | 平台、target、derived signal 与具名讨论者字段契约一致 | +| `src/externalDiscovery/redaction.ts` | public-safe / redaction / named actor safety 规则 | `pnpm test -- externalDiscoveryRedaction.test.ts` | 设计 3.6、4.2、11、12、14 | 禁止字段全部 fail，sanitized aggregate 和 registry 命中具名 actor pass | | `src/externalDiscovery/agentReachProvider.ts` | 本地 JSON adapter 状态机与 provider schema | `pnpm test -- externalDiscoveryAdapter.test.ts` | 设计 3.2、3.3、3.4 | ok/skipped/partial/failed 全覆盖 | -| `src/externalDiscovery/entityRegistry.ts` | registry 空启动与 tier 维护 | `pnpm test -- externalDiscoveryEntityRegistry.test.ts` | 需求 3、设计 7 | 空 registry 不阻塞，provider hint 不产生 top-tier | +| `src/externalDiscovery/entityRegistry.ts` | registry 空启动、tier 维护与 named actor eligibility | `pnpm test -- externalDiscoveryEntityRegistry.test.ts` | 需求 3、设计 4.2、7 | 空 registry 不阻塞，provider hint 不产生 top-tier，只有 registry 命中进入具名列表 | | `src/externalDiscovery/matching.ts` | repo matching 与 topic canonicalization | `pnpm test -- externalDiscoveryMatching.test.ts` | 需求 7、设计 6 | repo 精确匹配、topic_key、低置信降级全覆盖 | -| `src/externalDiscovery/aggregate.ts` | public daily aggregate contract | `pnpm test -- externalDiscoveryAggregate.test.ts` | 设计 4、8、12 | aggregate 字段齐全、无 raw text、不写 events JSONL | +| `src/externalDiscovery/aggregate.ts` | public daily aggregate 与 `named_registry_actors` 聚合 contract | `pnpm test -- externalDiscoveryAggregate.test.ts` | 设计 4、8、12 | aggregate 字段齐全、无 raw text、不写 events JSONL，具名列表仅来自 registry 命中 | | `src/cli.ts` | CLI command matrix | `pnpm test -- externalDiscoveryCli.test.ts` | 设计 3.5 | 四个命令 flag 语义一致 | -| `src/action/dailyReport.ts`、`src/action/runSummary.ts` | daily 与 run-summary external section | `pnpm test -- externalDiscoveryActionOutput.test.ts` | 设计 9、11 | 输出 secondary layer 审计，不污染主榜单 | -| `src/action/dailyVerification.ts` | verify warn/fail 与 contamination 检测 | `pnpm test -- externalDiscoveryVerification.test.ts` | 设计 11.4、13 | skipped/failed warn，redaction/score contamination fail | -| `src/action/weeklyEnhancement.ts`、`src/externalDiscovery/weeklyWindow.ts` | weekly 7 日 aggregate window | `pnpm test -- externalDiscoveryWeekly.test.ts` | 设计 10 | weekly 不读 raw input | +| `src/action/dailyReport.ts`、`src/action/runSummary.ts` | daily 与 run-summary external section / 谁在讨论消费 | `pnpm test -- externalDiscoveryActionOutput.test.ts` | 设计 4.2、9、11 | 输出 secondary layer 审计，不污染主榜单，具名讨论者和匿名汇总分层 | +| `src/action/dailyVerification.ts` | verify warn/fail、contamination 与 named actor safety 检测 | `pnpm test -- externalDiscoveryVerification.test.ts` | 设计 4.2、11.4、13 | skipped/failed warn，redaction/score/named actor safety violation fail | +| `src/action/weeklyEnhancement.ts`、`src/externalDiscovery/weeklyWindow.ts` | weekly 7 日 aggregate window 与具名讨论者复用 | `pnpm test -- externalDiscoveryWeekly.test.ts` | 设计 4.2、10 | weekly 不读 raw input，复用 daily aggregate 的 `named_registry_actors` | | `src/externalDiscovery/weeklyWindow.ts` | direction gate 4 选 2 | `pnpm test -- externalDiscoveryDirectionGate.test.ts` | 需求 7、设计 6.2 | 未达 2 个条件不得进入 weekly direction observation | | README / data README / gitignore / workflows / specs | OSS public artifact 与 spec 同步 | `pnpm test -- externalDiscoveryStructure.test.ts` | 设计 3.6、12、14 | raw input 不默认 commit/upload，spec 同步项齐全 | | 全仓 | 类型与回归 | `pnpm typecheck`、`pnpm test` | 全设计 | 全部通过 | @@ -654,16 +716,34 @@ ## 当前残余风险 -- 当前仓库测试目录较薄，Phase 0 必须先补结构测试，否则后续容易漏掉 workflow / data README / gitignore 同步。 -- 现有 `scripts/execPlanPreflight.ts` 仍绑定旧的 `github-star-delta-trust-v0.1.exec-plan.md`；本计划已要求实现前必须完成 preflight-sync 并记录结果，未完成前不得开始生产代码实现。 -- README 当前仍描述托管版登录能力；实现阶段需要精确区分 hosted app 与 OSS local console，避免把 hosted 登录说明误删。 +- Phase 3 仍需在后续实现中补齐 low-confidence name match 的 rejected / audit evidence 细化，不能让低置信名称匹配进入 daily 主展示。 +- Phase 4 仍需接入 public aggregate / latest 指针文件写入；当前只完成 in-memory aggregate 和 public-safe 校验。 +- Phase 5-7 尚未开始，CLI、daily / run-summary / verify、weekly 7 日窗口仍未接入 external discovery。 +- `scripts/execPlanPreflight.ts` 默认路径仍是旧计划；当前已通过显式 `--exec-plan` 方式完成本计划 receipt。若后续要依赖默认命令，仍需通用化。 ## 下一阶段入口 -进入实现前，先完成 ExecPlan Review。审核通过后，从 Phase 0 开始执行，并在每个 Phase 完成后更新本计划的阶段状态与验证记录。 +ExecPlan Review 已通过。进入实现时从 Phase 0 开始执行，并在每个 Phase 完成后更新本计划的阶段状态与验证记录；未完成 preflight-sync 并记录结果前，不得开始生产代码实现。 ## 验证记录 | 日期 | 命令 | 结果 | 备注 | | --- | --- | --- | --- | | 2026-06-13 | 未运行 | `Not Started` | 本轮修订 exec-plan 初稿；未改实现代码 | +| 2026-06-30 | `corepack pnpm run code-implementation:preflight -- --exec-plan docs/specs/exec-plans/agent-reach-external-discovery-and-evidence-v0.1.exec-plan.md --write` | `Passed` | 生成当前 exec-plan 对应 receipt：`docs/specs/agent-work/code-implementation-preflight.agent-reach-external-discovery-and-evidence-v0.1.json` | +| 2026-06-30 | `.\\node_modules\\.bin\\vitest.cmd run src/__tests__/externalDiscoveryTypeContract.test.ts src/__tests__/externalDiscoveryRedaction.test.ts src/__tests__/externalDiscoveryAdapter.test.ts src/__tests__/externalDiscoveryEntityRegistry.test.ts src/__tests__/externalDiscoveryMatching.test.ts src/__tests__/externalDiscoveryAggregate.test.ts src/__tests__/externalDiscoveryStructure.test.ts` | `Passed` | 7 files / 21 tests passed；覆盖 Phase 0-4 的 named_registry_actors 基座 | +| 2026-06-30 | `.\\node_modules\\.bin\\tsc.cmd --noEmit` | `Passed` | 类型检查通过 | +| 2026-06-30 | `.\\node_modules\\.bin\\tsx.cmd scripts\\execPlanPreflight.ts --exec-plan docs/specs/exec-plans/agent-reach-external-discovery-and-evidence-v0.1.exec-plan.md --check` | `Passed` | preflight receipt 校验通过 | +| 2026-06-30 | `.\\node_modules\\.bin\\vitest.cmd run src/__tests__/externalDiscoveryAdapter.test.ts` | `Passed` | 1 file / 5 tests passed；修复 Phase 2 审核问题后，adapter 负例覆盖缺失顶层审计字段与空 `derived_signal_kinds` | +| 2026-06-30 | `.\\node_modules\\.bin\\vitest.cmd run src/__tests__/externalDiscoveryTypeContract.test.ts src/__tests__/externalDiscoveryRedaction.test.ts src/__tests__/externalDiscoveryAdapter.test.ts src/__tests__/externalDiscoveryEntityRegistry.test.ts src/__tests__/externalDiscoveryMatching.test.ts src/__tests__/externalDiscoveryAggregate.test.ts src/__tests__/externalDiscoveryStructure.test.ts` | `Passed` | 7 files / 23 tests passed；Phase 2 adapter 合同校验重新通过 | +| 2026-06-30 | `.\\node_modules\\.bin\\tsc.cmd --noEmit` | `Passed` | Phase 2 adapter 类型收窄修复后类型检查通过 | + +## 本轮实现记录 + +| 阶段 | 状态 | 文件 | 说明 | +| --- | --- | --- | --- | +| Phase 0 | `DONE` | `docs/specs/agent-work/code-implementation-preflight.agent-reach-external-discovery-and-evidence-v0.1.json`、`src/__tests__/externalDiscoveryStructure.test.ts`、`.gitignore`、`data/README.md`、`.github/workflows/trend-radar-daily.yml`、`.github/workflows/trend-radar-weekly.yml`、`README.md`、`README.en.md` | 完成 preflight receipt、external raw local-only、public aggregate 上传边界和 OSS no-account 说明 | +| Phase 1 | `DONE` | `src/externalDiscovery/types.ts`、`src/externalDiscovery/paths.ts`、`src/externalDiscovery/redaction.ts`、`src/__tests__/externalDiscoveryTypeContract.test.ts`、`src/__tests__/externalDiscoveryRedaction.test.ts` | 建立 V1 enum、路径、`ExternalNamedRegistryActor`、`ExternalEvidence.named_registry_actors` 与 public-safe redaction | +| Phase 2 | `DONE` | `src/externalDiscovery/agentReachProvider.ts`、`src/__tests__/externalDiscoveryAdapter.test.ts` | 实现本地 JSON adapter 的 missing/default skipped、explicit failed、ok、partial 语义；已补齐顶层 provider contract 校验与空 `derived_signal_kinds` 负例 | +| Phase 3 | `IN_PROGRESS` | `src/externalDiscovery/entityRegistry.ts`、`src/externalDiscovery/matching.ts`、`src/__tests__/externalDiscoveryEntityRegistry.test.ts`、`src/__tests__/externalDiscoveryMatching.test.ts` | 完成 registry hit/miss、provider hint 不升级、repo 精确匹配和 topic key 基座 | +| Phase 4 | `IN_PROGRESS` | `src/externalDiscovery/aggregate.ts`、`src/__tests__/externalDiscoveryAggregate.test.ts` | 完成 in-memory public aggregate、`named_registry_actors` 聚合、空 registry/miss/provider hint 匿名降级和 mixed hit/miss 统计 | diff --git a/src/__tests__/externalDiscoveryAdapter.test.ts b/src/__tests__/externalDiscoveryAdapter.test.ts new file mode 100644 index 0000000..7789e8f --- /dev/null +++ b/src/__tests__/externalDiscoveryAdapter.test.ts @@ -0,0 +1,287 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { readAgentReachProviderArtifact } from "../externalDiscovery/agentReachProvider.ts"; +import { buildDailyExternalAggregate } from "../externalDiscovery/aggregate.ts"; + +function tempFile(name: string): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "external-discovery-")); + return path.join(dir, name); +} + +describe("agent reach provider artifact adapter", () => { + it("returns skipped for missing default input and failed for missing explicit input", () => { + const missingPath = path.join(os.tmpdir(), "missing-agent-reach-input.json"); + + expect(readAgentReachProviderArtifact(missingPath).status).toBe("skipped"); + expect(readAgentReachProviderArtifact(missingPath, { explicitInput: true }).status).toBe("failed"); + }); + + it("reads valid local JSON artifacts into canonical events", () => { + const filepath = tempFile("agent-reach.json"); + fs.writeFileSync( + filepath, + JSON.stringify({ + provider: "agent-reach", + schema_version: "agent-reach.external-discovery.v1", + provider_run_id: "run-1", + generated_at: "2026-06-30T00:00:00.000Z", + query: { keyword: "agents sdk" }, + platforms: ["x_twitter"], + status: "ok", + items: [ + { + event_id: "evt-1", + platform: "x_twitter", + raw_event_kind: "discussion", + derived_signal_kinds: ["evidence"], + scope: "project", + target_type: "project", + target_key: "openai/agents-sdk", + actor: { + actor_type: "institution", + effective_tier: "core", + tier_basis: "registry", + registry_entity_id: "entity-openai", + registry_display_name: "OpenAI", + registry_tier: "core", + }, + observed_at: "2026-06-30T00:00:00.000Z", + raw_ref: "provider:event:1", + }, + ], + }), + "utf-8", + ); + + const result = readAgentReachProviderArtifact(filepath, { explicitInput: true }); + expect(result.status).toBe("ok"); + expect(result.provider_run_id).toBe("run-1"); + expect(result.events).toHaveLength(1); + expect(result.events[0]?.actor.registry_display_name).toBe("OpenAI"); + expect(result.source_input_hash).toHaveLength(64); + }); + + it("enriches provider actors with the local registry before aggregate output", () => { + const filepath = tempFile("agent-reach-registry-enriched.json"); + fs.writeFileSync( + filepath, + JSON.stringify({ + provider: "agent-reach", + schema_version: "agent-reach.external-discovery.v1", + provider_run_id: "run-registry-enriched", + generated_at: "2026-06-30T00:00:00.000Z", + query: { keyword: "agents sdk" }, + platforms: ["x_twitter"], + status: "ok", + items: [ + { + event_id: "evt-registry-enriched", + platform: "x_twitter", + raw_event_kind: "discussion", + derived_signal_kinds: ["evidence"], + scope: "project", + target_type: "project", + target_key: "openai/agents-sdk", + actor: { + actor_type: "institution", + effective_tier: "unknown", + tier_basis: "none", + provider_actor_id: "@openai", + provider_tier_hint: "core", + }, + observed_at: "2026-06-30T00:00:00.000Z", + raw_ref: "provider:event:registry-enriched", + }, + ], + }), + "utf-8", + ); + + const result = readAgentReachProviderArtifact(filepath, { + explicitInput: true, + entityRegistry: [ + { + entity_id: "entity-openai", + display_name: "OpenAI", + actor_type: "institution", + tier: "core", + handles: ["openai"], + profile_urls: ["https://x.com/openai"], + updated_at: "2026-06-30T00:00:00.000Z", + }, + ], + }); + const aggregate = buildDailyExternalAggregate({ + date: "2026-06-30", + generated_at: "2026-06-30T01:00:00.000Z", + provider_result: result, + }); + + expect(result.warnings).toEqual([]); + expect(result.events[0]?.actor).toMatchObject({ + effective_tier: "core", + tier_basis: "registry", + registry_display_name: "OpenAI", + }); + expect(aggregate.project_evidence[0]?.named_registry_actors).toEqual([ + { + entity_id: "entity-openai", + display_name: "OpenAI", + actor_type: "institution", + registry_tier: "core", + event_count: 1, + platforms: ["x_twitter"], + first_seen_at: "2026-06-30T00:00:00.000Z", + last_seen_at: "2026-06-30T00:00:00.000Z", + }, + ]); + }); + + it("accepts nested target and identity hash fields from real AgentReach artifacts", () => { + const filepath = tempFile("agent-reach-real-shape.json"); + fs.writeFileSync( + filepath, + JSON.stringify({ + provider: "agent-reach", + schema_version: "agent-reach.external-discovery.v1", + provider_run_id: "run-real-shape", + generated_at: "2026-06-30T00:00:00.000Z", + query: { keyword: "agents sdk" }, + platforms: ["reddit"], + status: "ok", + items: [ + { + platform: "reddit", + raw_ref: "opencli:reddit:abc", + url: "https://reddit.com/r/LocalLLaMA/comments/abc", + observed_at: "2026-06-30T00:00:00.000Z", + raw_event_kind: "discussion", + derived_signal_kinds: ["discovery", "evidence"], + actor: { + actor_type: "community", + identity_hash: "sha256:community-actor", + }, + target: { + name: "new agent memory framework", + topic_hint: "agent memory framework", + }, + }, + ], + }), + "utf-8", + ); + + const result = readAgentReachProviderArtifact(filepath, { explicitInput: true }); + + expect(result.status).toBe("ok"); + expect(result.events).toHaveLength(1); + expect(result.events[0]).toMatchObject({ + platform: "reddit", + scope: "direction", + target_type: "topic", + target_key: "new agent memory framework", + actor: { + actor_type: "community", + provider_actor_id: "sha256:community-actor", + }, + }); + expect(result.events[0]?.event_id).toMatch(/^agent-reach:/); + }); + + it("marks mixed valid and invalid events as partial", () => { + const filepath = tempFile("agent-reach-partial.json"); + fs.writeFileSync( + filepath, + JSON.stringify({ + provider: "agent-reach", + schema_version: "agent-reach.external-discovery.v1", + provider_run_id: "run-partial", + generated_at: "2026-06-30T00:00:00.000Z", + query: { keyword: "agents sdk" }, + platforms: ["reddit"], + status: "ok", + items: [ + { + event_id: "evt-valid", + platform: "reddit", + raw_event_kind: "discussion", + derived_signal_kinds: ["evidence"], + scope: "project", + target_type: "project", + target_key: "openai/agents-sdk", + actor: { actor_type: "community", effective_tier: "ordinary", tier_basis: "none" }, + observed_at: "2026-06-30T00:00:00.000Z", + raw_ref: "provider:event:valid", + }, + { + event_id: "evt-invalid", + platform: "reddit", + }, + ], + }), + "utf-8", + ); + + const result = readAgentReachProviderArtifact(filepath, { explicitInput: true }); + expect(result.status).toBe("partial"); + expect(result.events).toHaveLength(1); + expect(result.rejected_events[0]?.reason_code).toBe("event_schema_invalid"); + }); + + it("fails ok artifacts that miss required top-level audit fields", () => { + const filepath = tempFile("agent-reach-missing-audit.json"); + fs.writeFileSync( + filepath, + JSON.stringify({ + provider: "agent-reach", + schema_version: "agent-reach.external-discovery.v1", + status: "ok", + platforms: ["x_twitter"], + items: [], + }), + "utf-8", + ); + + const result = readAgentReachProviderArtifact(filepath, { explicitInput: true }); + expect(result.status).toBe("failed"); + expect(result.status_reason).toBe("provider_run_id_missing"); + }); + + it("rejects events with empty derived_signal_kinds", () => { + const filepath = tempFile("agent-reach-empty-derived-kinds.json"); + fs.writeFileSync( + filepath, + JSON.stringify({ + provider: "agent-reach", + schema_version: "agent-reach.external-discovery.v1", + provider_run_id: "run-empty-kinds", + generated_at: "2026-06-30T00:00:00.000Z", + query: { keyword: "agents sdk" }, + platforms: ["x_twitter"], + status: "ok", + items: [ + { + event_id: "evt-empty-kinds", + platform: "x_twitter", + raw_event_kind: "discussion", + derived_signal_kinds: [], + scope: "project", + target_type: "project", + target_key: "openai/agents-sdk", + actor: { actor_type: "community", effective_tier: "ordinary", tier_basis: "none" }, + observed_at: "2026-06-30T00:00:00.000Z", + raw_ref: "provider:event:empty-kinds", + }, + ], + }), + "utf-8", + ); + + const result = readAgentReachProviderArtifact(filepath, { explicitInput: true }); + expect(result.status).toBe("partial"); + expect(result.events).toHaveLength(0); + expect(result.rejected_events[0]?.reason_code).toBe("event_schema_invalid"); + }); +}); diff --git a/src/__tests__/externalDiscoveryAggregate.test.ts b/src/__tests__/externalDiscoveryAggregate.test.ts new file mode 100644 index 0000000..30478de --- /dev/null +++ b/src/__tests__/externalDiscoveryAggregate.test.ts @@ -0,0 +1,114 @@ +import { describe, expect, it } from "vitest"; +import { buildDailyExternalAggregate } from "../externalDiscovery/aggregate.ts"; +import type { ExternalSignalEvent } from "../externalDiscovery/types.ts"; + +function event(overrides: Partial): ExternalSignalEvent { + return { + event_id: "evt-1", + platform: "x_twitter", + raw_event_kind: "discussion", + derived_signal_kinds: ["evidence"], + scope: "project", + target_type: "project", + target_key: "openai/agents-sdk", + actor: { + actor_type: "institution", + effective_tier: "core", + tier_basis: "registry", + registry_entity_id: "entity-openai", + registry_display_name: "OpenAI", + registry_tier: "core", + }, + observed_at: "2026-06-30T00:00:00.000Z", + raw_ref: "provider:event:1", + ...overrides, + }; +} + +describe("external discovery aggregate", () => { + it("builds named_registry_actors only from registry-based actors", () => { + const aggregate = buildDailyExternalAggregate({ + date: "2026-06-30", + generated_at: "2026-06-30T01:00:00.000Z", + events: [ + event({ event_id: "evt-1" }), + event({ + event_id: "evt-2", + platform: "reddit", + actor: { + actor_type: "person", + effective_tier: "ordinary", + tier_basis: "provider_hint", + provider_actor_id: "raw-person-1", + provider_tier_hint: "core", + }, + observed_at: "2026-06-30T02:00:00.000Z", + }), + ], + }); + + const evidence = aggregate.project_evidence[0]!; + expect(evidence.named_registry_actors).toEqual([ + { + entity_id: "entity-openai", + display_name: "OpenAI", + actor_type: "institution", + registry_tier: "core", + event_count: 1, + platforms: ["x_twitter"], + first_seen_at: "2026-06-30T00:00:00.000Z", + last_seen_at: "2026-06-30T00:00:00.000Z", + }, + ]); + expect(evidence.actor_tiers).toMatchObject({ core: 1, ordinary: 1 }); + expect(evidence.top_tier_actor_count).toBe(1); + }); + + it("keeps registry miss and provider-only actors anonymous", () => { + const aggregate = buildDailyExternalAggregate({ + date: "2026-06-30", + generated_at: "2026-06-30T01:00:00.000Z", + events: [ + event({ + actor: { + actor_type: "person", + effective_tier: "ordinary", + tier_basis: "provider_hint", + provider_actor_id: "raw-person-2", + provider_tier_hint: "proven", + }, + }), + ], + }); + + const evidence = aggregate.project_evidence[0]!; + expect(evidence.named_registry_actors).toEqual([]); + expect(evidence.top_tier_actor_count).toBe(0); + expect(evidence.actor_types).toMatchObject({ person: 1 }); + }); + + it("sorts named registry actors by tier, count, then name", () => { + const aggregate = buildDailyExternalAggregate({ + date: "2026-06-30", + generated_at: "2026-06-30T01:00:00.000Z", + events: [ + event({ + event_id: "evt-watch", + actor: { + actor_type: "team", + effective_tier: "watch", + tier_basis: "registry", + registry_entity_id: "entity-zeta", + registry_display_name: "Zeta Lab", + registry_tier: "watch", + }, + }), + event({ event_id: "evt-core-1" }), + event({ event_id: "evt-core-2", platform: "official_blog", observed_at: "2026-06-30T03:00:00.000Z" }), + ], + }); + + expect(aggregate.project_evidence[0]?.named_registry_actors.map((actor) => actor.display_name)).toEqual(["OpenAI", "Zeta Lab"]); + expect(aggregate.project_evidence[0]?.named_registry_actors[0]?.event_count).toBe(2); + }); +}); diff --git a/src/__tests__/externalDiscoveryEntityRegistry.test.ts b/src/__tests__/externalDiscoveryEntityRegistry.test.ts new file mode 100644 index 0000000..a6d8a92 --- /dev/null +++ b/src/__tests__/externalDiscoveryEntityRegistry.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it } from "vitest"; +import { applyEntityRegistry, type ExternalEntityRegistryEntry } from "../externalDiscovery/entityRegistry.ts"; + +const registry: ExternalEntityRegistryEntry[] = [ + { + entity_id: "entity-openai", + display_name: "OpenAI", + actor_type: "institution", + tier: "core", + handles: ["openai"], + profile_urls: ["https://x.com/openai"], + updated_at: "2026-06-30T00:00:00.000Z", + }, +]; + +describe("external discovery entity registry", () => { + it("promotes only registry hits to registry-based effective tiers", () => { + const result = applyEntityRegistry({ registry_entity_id: "entity-openai", provider_actor_id: "raw-1", provider_tier_hint: "core" }, registry); + + expect(result.warnings).toEqual([]); + expect(result.actor).toMatchObject({ + actor_type: "institution", + effective_tier: "core", + tier_basis: "registry", + registry_entity_id: "entity-openai", + registry_display_name: "OpenAI", + registry_tier: "core", + }); + }); + + it("matches provider actor ids against registry handles", () => { + const result = applyEntityRegistry({ actor_type: "institution", provider_actor_id: "@openai", provider_tier_hint: "core" }, registry); + + expect(result.warnings).toEqual([]); + expect(result.actor).toMatchObject({ + actor_type: "institution", + effective_tier: "core", + tier_basis: "registry", + registry_entity_id: "entity-openai", + registry_display_name: "OpenAI", + registry_tier: "core", + }); + }); + + it("keeps provider tier hints anonymous when registry misses", () => { + const result = applyEntityRegistry({ actor_type: "person", provider_actor_id: "raw-2", provider_tier_hint: "core" }, registry); + + expect(result.warnings[0]?.reason_code).toBe("registry_miss"); + expect(result.actor.effective_tier).toBe("ordinary"); + expect(result.actor.tier_basis).toBe("provider_hint"); + expect(result.actor.registry_entity_id).toBeUndefined(); + }); + + it("allows empty registry startup without creating top-tier actors", () => { + const result = applyEntityRegistry({ actor_type: "community", provider_tier_hint: "watch" }, []); + + expect(result.warnings[0]?.reason_code).toBe("registry_empty"); + expect(result.actor.effective_tier).toBe("ordinary"); + expect(result.actor.registry_tier).toBeUndefined(); + }); +}); diff --git a/src/__tests__/externalDiscoveryMatching.test.ts b/src/__tests__/externalDiscoveryMatching.test.ts new file mode 100644 index 0000000..6543eee --- /dev/null +++ b/src/__tests__/externalDiscoveryMatching.test.ts @@ -0,0 +1,66 @@ +import { describe, expect, it } from "vitest"; +import { observationCandidateForUnmatchedEvent, projectEvidenceTargetForEvent, topicKeyFromHints } from "../externalDiscovery/matching.ts"; +import type { ExternalSignalEvent } from "../externalDiscovery/types.ts"; + +const baseEvent: ExternalSignalEvent = { + event_id: "evt-1", + platform: "official_blog", + raw_event_kind: "official_release", + derived_signal_kinds: ["discovery", "evidence"], + scope: "project", + target_type: "project", + target_key: "https://github.com/openai/agents-sdk", + actor: { + actor_type: "institution", + effective_tier: "core", + tier_basis: "registry", + registry_entity_id: "entity-openai", + registry_display_name: "OpenAI", + registry_tier: "core", + }, + observed_at: "2026-06-30T00:00:00.000Z", + raw_ref: "provider:event:1", +}; + +describe("external discovery matching", () => { + it("matches project events by exact repo URL", () => { + expect( + projectEvidenceTargetForEvent(baseEvent, [ + { + repo_url: "https://github.com/openai/agents-sdk", + repo_full_name: "openai/agents-sdk", + }, + ]), + ).toBe("openai/agents-sdk"); + }); + + it("canonicalizes topic hints in the frozen priority order", () => { + expect(topicKeyFromHints({ userInterestTopics: ["Browser Computer Use"], providerTopicHint: "ignored" })).toBe("browser-computer-use"); + expect(topicKeyFromHints({ weeklyTrendKeys: ["Agent Runtime"], providerTopicHint: "ignored" })).toBe("agent-runtime"); + expect(topicKeyFromHints({ providerTopicHint: "Voice Agent Ops" })).toBe("voice-agent-ops"); + expect(topicKeyFromHints({})).toBeNull(); + }); + + it("creates bounded observation candidates without primary-conclusion authority", () => { + expect(observationCandidateForUnmatchedEvent(baseEvent)).toMatchObject({ + candidate_kind: "project", + qualification: "needs_primary_confirmation", + can_enter_daily: true, + cannot_be_primary_conclusion: true, + }); + expect( + observationCandidateForUnmatchedEvent({ + ...baseEvent, + scope: "direction", + target_type: "topic", + target_key: "browser-computer-use", + }), + ).toMatchObject({ + candidate_kind: "direction", + qualification: "direction_observation", + can_enter_daily: false, + can_enter_weekly: true, + cannot_be_primary_conclusion: true, + }); + }); +}); diff --git a/src/__tests__/externalDiscoveryRedaction.test.ts b/src/__tests__/externalDiscoveryRedaction.test.ts new file mode 100644 index 0000000..e098ac0 --- /dev/null +++ b/src/__tests__/externalDiscoveryRedaction.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, it } from "vitest"; +import { assertPublicSafeAggregate, containsForbiddenPublicArtifactText, stableSourceInputHash } from "../externalDiscovery/redaction.ts"; + +function safeAggregate(overrides: Record = {}): Record { + return { + public_safe: true, + contains_raw_text: false, + contains_profile_urls: false, + redaction_policy_version: "external-discovery-redaction.v1", + source_input_hash: stableSourceInputHash("fixture"), + named_registry_actors: [ + { + entity_id: "entity-openai", + display_name: "OpenAI", + actor_type: "institution", + registry_tier: "core", + event_count: 1, + platforms: ["x_twitter"], + first_seen_at: "2026-06-30T00:00:00.000Z", + last_seen_at: "2026-06-30T00:00:00.000Z", + }, + ], + ...overrides, + }; +} + +describe("external discovery redaction", () => { + it("allows sanitized aggregates with registry-derived named actors", () => { + expect(assertPublicSafeAggregate(safeAggregate()).ok).toBe(true); + }); + + it("rejects raw text, profile URLs, handles, provider diagnostics, and secrets", () => { + expect(containsForbiddenPublicArtifactText({ content_text: "raw social text" })).toBe(true); + expect(containsForbiddenPublicArtifactText({ profile_url: "https://x.com/raw-profile" })).toBe(true); + expect(containsForbiddenPublicArtifactText({ handle: "@raw" })).toBe(true); + expect(containsForbiddenPublicArtifactText({ provider_diagnostics: { trace: "private" } })).toBe(true); + expect(containsForbiddenPublicArtifactText({ status_reason: "oauth token leaked" })).toBe(true); + }); + + it("rejects named_registry_actors that carry raw identity fields", () => { + const result = assertPublicSafeAggregate( + safeAggregate({ + named_registry_actors: [ + { + entity_id: "entity-openai", + display_name: "OpenAI", + actor_type: "institution", + registry_tier: "core", + event_count: 1, + platforms: ["x_twitter"], + first_seen_at: "2026-06-30T00:00:00.000Z", + last_seen_at: "2026-06-30T00:00:00.000Z", + handle: "@openai", + }, + ], + }), + ); + + expect(result.ok).toBe(false); + expect(result.reason_codes).toContain("forbidden_key:handle"); + }); + + it("requires public aggregate safety flags", () => { + const result = assertPublicSafeAggregate( + safeAggregate({ + public_safe: false, + contains_raw_text: true, + }), + ); + + expect(result.ok).toBe(false); + expect(result.reason_codes).toContain("public_safe_not_true"); + expect(result.reason_codes).toContain("contains_raw_text_not_false"); + }); +}); diff --git a/src/__tests__/externalDiscoveryStructure.test.ts b/src/__tests__/externalDiscoveryStructure.test.ts new file mode 100644 index 0000000..07c1dc1 --- /dev/null +++ b/src/__tests__/externalDiscoveryStructure.test.ts @@ -0,0 +1,40 @@ +import fs from "node:fs"; +import { describe, expect, it } from "vitest"; + +function read(filepath: string): string { + return fs.readFileSync(filepath, "utf-8"); +} + +describe("external discovery structure boundaries", () => { + it("keeps AgentReach raw input local-only and public aggregates explicit", () => { + const gitignore = read(".gitignore"); + const dataReadme = read("data/README.md"); + + expect(gitignore).toContain("data/raw/external-discovery/**"); + expect(gitignore).toContain("!data/raw/external-discovery/fixtures/**"); + expect(dataReadme).toContain("data/raw/external-discovery/"); + expect(dataReadme).toContain("local-only"); + expect(dataReadme).toContain("data/external-discovery/*.aggregate.json"); + expect(dataReadme).toContain("no public `*.events.jsonl` default artifact"); + }); + + it("does not upload external raw input in GitHub Actions", () => { + const dailyWorkflow = read(".github/workflows/trend-radar-daily.yml"); + const weeklyWorkflow = read(".github/workflows/trend-radar-weekly.yml"); + + expect(dailyWorkflow).not.toContain("data/raw/external-discovery"); + expect(weeklyWorkflow).not.toContain("data/raw/external-discovery"); + expect(dailyWorkflow).toContain("data/external-discovery/${{ steps.options.outputs.target_date }}.aggregate.json"); + expect(dailyWorkflow).toContain("data/external-discovery/latest.aggregate.json"); + }); + + it("documents OSS no-account external discovery boundaries", () => { + const readme = read("README.md"); + const readmeEn = read("README.en.md"); + + expect(readme).toContain("开源版只消费本地 AgentReach JSON artifact"); + expect(readme).toContain("cookie、session、OAuth"); + expect(readmeEn).toContain("only consumes local AgentReach JSON artifacts"); + expect(readmeEn).toContain("cookies, sessions, OAuth"); + }); +}); diff --git a/src/__tests__/externalDiscoveryTypeContract.test.ts b/src/__tests__/externalDiscoveryTypeContract.test.ts new file mode 100644 index 0000000..1706a66 --- /dev/null +++ b/src/__tests__/externalDiscoveryTypeContract.test.ts @@ -0,0 +1,25 @@ +import { describe, expect, it } from "vitest"; +import { externalAggregateLatestPath, externalAggregatePath, externalEntityRegistryPath, externalRawInputPath, externalSanitizedFixtureDirPath } from "../externalDiscovery/paths.ts"; +import { EXTERNAL_PLATFORMS, EXTERNAL_TARGET_TYPES } from "../externalDiscovery/types.ts"; + +describe("external discovery type and path contract", () => { + it("freezes the V1 platform and target enums", () => { + expect(EXTERNAL_PLATFORMS).toEqual(["x_twitter", "reddit", "hacker_news", "official_web", "official_blog"]); + expect(EXTERNAL_PLATFORMS).not.toContain("x"); + expect(EXTERNAL_TARGET_TYPES).toEqual(["project", "paper", "product", "topic"]); + expect(EXTERNAL_TARGET_TYPES).not.toContain("direction"); + expect(EXTERNAL_TARGET_TYPES).not.toContain("unknown"); + }); + + it("keeps external discovery raw input and public aggregate paths separate", () => { + expect(slash(externalRawInputPath("2026-06-30"))).toBe("data/raw/external-discovery/2026-06-30.agent-reach.json"); + expect(slash(externalAggregatePath("2026-06-30"))).toBe("data/external-discovery/2026-06-30.aggregate.json"); + expect(slash(externalAggregateLatestPath())).toBe("data/external-discovery/latest.aggregate.json"); + expect(slash(externalEntityRegistryPath())).toBe("data/external-discovery/entity-registry.json"); + expect(slash(externalSanitizedFixtureDirPath())).toBe("data/raw/external-discovery/fixtures"); + }); +}); + +function slash(value: string): string { + return value.replace(/\\/g, "/"); +} diff --git a/src/externalDiscovery/agentReachProvider.ts b/src/externalDiscovery/agentReachProvider.ts new file mode 100644 index 0000000..5c7bdc2 --- /dev/null +++ b/src/externalDiscovery/agentReachProvider.ts @@ -0,0 +1,342 @@ +import fs from "node:fs"; + +import { applyEntityRegistry, readEntityRegistry, type ExternalEntityRegistryEntry } from "./entityRegistry.ts"; +import { externalEntityRegistryPath } from "./paths.ts"; +import { stableSourceInputHash } from "./redaction.ts"; +import type { + AgentReachProviderReadResult, + ExternalPlatform, + ExternalProviderStatus, + ExternalRawEventKind, + ExternalSignalEvent, + ExternalSignalKind, + ExternalTargetType, +} from "./types.ts"; + +interface ReadOptions { + explicitInput?: boolean; + entityRegistry?: ExternalEntityRegistryEntry[]; + entityRegistryPath?: string; +} + +interface ValidProviderArtifact extends Record { + provider_run_id: string; + generated_at: string; + query: string | Record; + platforms: ExternalPlatform[]; + status: ExternalProviderStatus; + items: unknown[]; + status_reason?: unknown; +} + +export function readAgentReachProviderArtifact(filepath: string, options: ReadOptions = {}): AgentReachProviderReadResult { + if (!fs.existsSync(filepath)) { + return emptyResult(options.explicitInput ? "failed" : "skipped", options.explicitInput ? "input_missing_explicit" : "input_missing"); + } + + let raw: string; + try { + raw = fs.readFileSync(filepath, "utf-8"); + } catch { + return emptyResult("failed", "input_unreadable"); + } + + let parsed: unknown; + try { + parsed = JSON.parse(raw) as unknown; + } catch { + return emptyResult("failed", "input_invalid_json", stableSourceInputHash(raw)); + } + + if (!isRecord(parsed)) return emptyResult("failed", "provider_schema_invalid", stableSourceInputHash(raw)); + if (parsed.provider !== "agent-reach" || parsed.schema_version !== "agent-reach.external-discovery.v1") { + return emptyResult("failed", "provider_schema_invalid", stableSourceInputHash(raw)); + } + + const topLevelContract = validateTopLevelContract(parsed); + if (!topLevelContract.ok) { + return emptyResult("failed", topLevelContract.reason_code, stableSourceInputHash(raw)); + } + + const artifact = parsed as ValidProviderArtifact; + const status = artifact.status; + const platforms = artifact.platforms; + const events: ExternalSignalEvent[] = []; + const rejectedEvents: AgentReachProviderReadResult["rejected_events"] = []; + const registry = resolveEntityRegistry(options); + const warnings: AgentReachProviderReadResult["warnings"] = []; + + for (const item of artifact.items) { + const event = parseEvent(item); + if (event.ok) { + const enriched = enrichEventActor(event.value, registry); + events.push(enriched.event); + warnings.push(...enriched.warnings); + } else { + rejectedEvents.push(event.rejected); + } + } + + const effectiveStatus: ExternalProviderStatus = + status === "ok" && rejectedEvents.length > 0 ? "partial" : status === "ok" && events.length === 0 ? "ok" : status; + + return { + provider: "agent-reach", + schema_version: "agent-reach.external-discovery.v1", + provider_run_id: artifact.provider_run_id, + generated_at: artifact.generated_at, + status: effectiveStatus, + status_reason: typeof artifact.status_reason === "string" ? artifact.status_reason : undefined, + platforms, + events, + rejected_events: rejectedEvents, + warnings: uniqueWarnings(warnings), + source_input_hash: stableSourceInputHash(raw), + }; +} + +function resolveEntityRegistry(options: ReadOptions): + | { + shouldApply: true; + entries: ExternalEntityRegistryEntry[]; + } + | { + shouldApply: false; + entries: []; + } { + if (options.entityRegistry) return { shouldApply: true, entries: options.entityRegistry }; + + const registryPath = options.entityRegistryPath ?? externalEntityRegistryPath(); + if (!fs.existsSync(registryPath)) return { shouldApply: false, entries: [] }; + return { shouldApply: true, entries: readEntityRegistry(registryPath) }; +} + +function enrichEventActor( + event: ExternalSignalEvent, + registry: + | { + shouldApply: true; + entries: ExternalEntityRegistryEntry[]; + } + | { + shouldApply: false; + entries: []; + }, +): { event: ExternalSignalEvent; warnings: AgentReachProviderReadResult["warnings"] } { + if (!registry.shouldApply) return { event, warnings: [] }; + + const lookup = applyEntityRegistry(event.actor, registry.entries); + return { + event: { + ...event, + actor: lookup.actor, + }, + warnings: lookup.warnings, + }; +} + +function uniqueWarnings(warnings: AgentReachProviderReadResult["warnings"]): AgentReachProviderReadResult["warnings"] { + const byKey = new Map(); + for (const warning of warnings) { + byKey.set(`${warning.reason_code}:${warning.reason_detail}`, warning); + } + return [...byKey.values()]; +} + +function parseEvent(value: unknown): + | { ok: true; value: ExternalSignalEvent } + | { ok: false; rejected: AgentReachProviderReadResult["rejected_events"][number] } { + if (!isRecord(value)) { + return { ok: false, rejected: { reason_code: "event_not_object", reason_detail: "provider item is not an object" } }; + } + + const rawRef = typeof value.raw_ref === "string" ? value.raw_ref : undefined; + const url = typeof value.url === "string" ? value.url : undefined; + const observedAt = typeof value.observed_at === "string" ? value.observed_at : undefined; + const eventId = typeof value.event_id === "string" ? value.event_id : generatedEventId({ rawRef, url, observedAt }); + const actor = isRecord(value.actor) ? value.actor : undefined; + const target = isRecord(value.target) ? value.target : undefined; + const targetType = isTargetType(value.target_type) ? value.target_type : inferTargetType(target); + const scope = isScope(value.scope) ? value.scope : inferScope(targetType); + const targetKey = typeof value.target_key === "string" ? value.target_key : inferTargetKey(value, target); + if ( + !eventId || + !isExternalPlatform(value.platform) || + !isRawEventKind(value.raw_event_kind) || + !Array.isArray(value.derived_signal_kinds) || + value.derived_signal_kinds.length === 0 || + value.derived_signal_kinds.some((kind) => !isSignalKind(kind)) || + !targetType || + !scope || + !targetKey || + !actor || + !observedAt + ) { + return { + ok: false, + rejected: { event_id: eventId, reason_code: "event_schema_invalid", reason_detail: "provider item misses required canonical fields" }, + }; + } + + if (!url && !rawRef) { + return { + ok: false, + rejected: { event_id: eventId, reason_code: "event_source_ref_missing", reason_detail: "url or raw_ref is required" }, + }; + } + + return { + ok: true, + value: { + event_id: eventId, + platform: value.platform, + raw_event_kind: value.raw_event_kind, + derived_signal_kinds: value.derived_signal_kinds, + scope, + target_type: targetType, + target_key: targetKey, + actor: { + actor_type: isActorType(actor.actor_type) ? actor.actor_type : "unknown", + effective_tier: isActorTier(actor.effective_tier) ? actor.effective_tier : "unknown", + tier_basis: actor.tier_basis === "registry" || actor.tier_basis === "provider_hint" ? actor.tier_basis : isProviderTierHint(actor.provider_tier_hint) || isProviderTierHint(actor.tier_hint) ? "provider_hint" : "none", + provider_actor_id: providerActorId(actor), + provider_tier_hint: isProviderTierHint(actor.provider_tier_hint) ? actor.provider_tier_hint : isProviderTierHint(actor.tier_hint) ? actor.tier_hint : undefined, + registry_entity_id: typeof actor.registry_entity_id === "string" ? actor.registry_entity_id : undefined, + registry_display_name: typeof actor.registry_display_name === "string" ? actor.registry_display_name : undefined, + registry_tier: isRegistryTier(actor.registry_tier) ? actor.registry_tier : undefined, + }, + observed_at: observedAt, + source_published_at: typeof value.source_published_at === "string" ? value.source_published_at : undefined, + ingested_at: typeof value.ingested_at === "string" ? value.ingested_at : undefined, + url, + raw_ref: rawRef, + }, + }; +} + +function generatedEventId(input: { rawRef?: string; url?: string; observedAt?: string }): string | undefined { + const seed = input.rawRef ?? input.url; + if (!seed || !input.observedAt) return undefined; + return `agent-reach:${stableSourceInputHash(`${seed}:${input.observedAt}`).slice(0, 16)}`; +} + +function inferTargetType(target: Record | undefined): ExternalTargetType | undefined { + if (!target) return undefined; + if (isTargetType(target.target_type)) return target.target_type; + if (typeof target.repo_url === "string") return "project"; + if (typeof target.paper_url === "string") return "paper"; + if (typeof target.product_url === "string") return "product"; + if (typeof target.url === "string" && /github\.com/i.test(target.url)) return "project"; + return "topic"; +} + +function inferScope(targetType: ExternalTargetType | undefined): ExternalSignalEvent["scope"] | undefined { + if (!targetType) return undefined; + return targetType === "topic" ? "direction" : "project"; +} + +function inferTargetKey(value: Record, target: Record | undefined): string | undefined { + const candidates = [ + target?.repo_url, + target?.paper_url, + target?.url, + target?.name, + target?.topic_hint, + value.title, + value.raw_ref, + value.url, + ]; + return candidates.find((candidate): candidate is string => typeof candidate === "string" && candidate.trim().length > 0)?.trim(); +} + +function providerActorId(actor: Record): string | undefined { + const candidates = [actor.provider_actor_id, actor.identity_hash, actor.identity_id]; + return candidates.find((candidate): candidate is string => typeof candidate === "string" && candidate.trim().length > 0)?.trim(); +} + +function emptyResult(status: ExternalProviderStatus, statusReason: string, sourceInputHash = ""): AgentReachProviderReadResult { + return { + provider: "agent-reach", + schema_version: "agent-reach.external-discovery.v1", + status, + status_reason: statusReason, + platforms: [], + events: [], + rejected_events: [], + warnings: [], + source_input_hash: sourceInputHash, + }; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function validateTopLevelContract(value: Record): + | { + ok: true; + } + | { + ok: false; + reason_code: string; + } { + if (typeof value.provider_run_id !== "string" || value.provider_run_id.length === 0) { + return { ok: false, reason_code: "provider_run_id_missing" }; + } + if (typeof value.generated_at !== "string" || value.generated_at.length === 0) { + return { ok: false, reason_code: "generated_at_missing" }; + } + if (!(typeof value.query === "string" || isRecord(value.query))) { + return { ok: false, reason_code: "query_missing" }; + } + if (!Array.isArray(value.platforms) || value.platforms.length === 0 || value.platforms.some((platform) => !isExternalPlatform(platform))) { + return { ok: false, reason_code: "platforms_invalid" }; + } + if (!isProviderStatus(value.status)) { + return { ok: false, reason_code: "status_invalid" }; + } + if (!Array.isArray(value.items)) { + return { ok: false, reason_code: "items_invalid" }; + } + return { ok: true }; +} + +function isExternalPlatform(value: unknown): value is ExternalPlatform { + return value === "x_twitter" || value === "reddit" || value === "hacker_news" || value === "official_web" || value === "official_blog"; +} + +function isProviderStatus(value: unknown): value is ExternalProviderStatus { + return value === "ok" || value === "skipped" || value === "partial" || value === "failed"; +} + +function isSignalKind(value: unknown): value is ExternalSignalKind { + return value === "discovery" || value === "evidence"; +} + +function isRawEventKind(value: unknown): value is ExternalRawEventKind { + return value === "mention" || value === "discussion" || value === "official_release" || value === "blog_post" || value === "question" || value === "showcase" || value === "unknown"; +} + +function isTargetType(value: unknown): value is ExternalTargetType { + return value === "project" || value === "paper" || value === "product" || value === "topic"; +} + +function isScope(value: unknown): value is ExternalSignalEvent["scope"] { + return value === "project" || value === "direction"; +} + +function isActorType(value: unknown): value is ExternalSignalEvent["actor"]["actor_type"] { + return value === "institution" || value === "team" || value === "person" || value === "community" || value === "unknown"; +} + +function isActorTier(value: unknown): value is ExternalSignalEvent["actor"]["effective_tier"] { + return value === "core" || value === "proven" || value === "watch" || value === "ordinary" || value === "unknown"; +} + +function isProviderTierHint(value: unknown): value is ExternalSignalEvent["actor"]["provider_tier_hint"] { + return value === "core" || value === "proven" || value === "watch" || value === "ordinary" || value === "unknown"; +} + +function isRegistryTier(value: unknown): value is ExternalSignalEvent["actor"]["registry_tier"] { + return value === "core" || value === "proven" || value === "watch"; +} diff --git a/src/externalDiscovery/aggregate.ts b/src/externalDiscovery/aggregate.ts new file mode 100644 index 0000000..7bf24f5 --- /dev/null +++ b/src/externalDiscovery/aggregate.ts @@ -0,0 +1,186 @@ +import { REDACTION_POLICY_VERSION, assertPublicSafeAggregate, stableSourceInputHash } from "./redaction.ts"; +import type { + AgentReachProviderReadResult, + DailyExternalAggregate, + ExternalActorTier, + ExternalActorType, + ExternalEvidence, + ExternalNamedRegistryActor, + ExternalPlatform, + ExternalSignalEvent, + ExternalSignalKind, + ObservationCandidate, +} from "./types.ts"; + +export interface BuildDailyExternalAggregateInput { + date: string; + generated_at: string; + provider_result?: AgentReachProviderReadResult; + events?: ExternalSignalEvent[]; + observation_candidates?: ObservationCandidate[]; + source_input_hash?: string; +} + +const registryTierRank: Record = { + core: 0, + proven: 1, + watch: 2, +}; + +export function buildDailyExternalAggregate(input: BuildDailyExternalAggregateInput): DailyExternalAggregate { + const events = input.events ?? input.provider_result?.events ?? []; + const rejectedEvents = input.provider_result?.rejected_events ?? []; + const aggregate: DailyExternalAggregate = { + schema_version: "external-discovery.aggregate.v1", + date: input.date, + generated_at: input.generated_at, + provider: "agent-reach", + provider_run_id: input.provider_result?.provider_run_id, + status: input.provider_result?.status ?? "ok", + status_reason: input.provider_result?.status_reason, + source_input_hash: input.source_input_hash ?? input.provider_result?.source_input_hash ?? stableSourceInputHash(JSON.stringify(events)), + public_safe: true, + redaction_policy_version: REDACTION_POLICY_VERSION, + contains_raw_text: false, + contains_profile_urls: false, + event_count: events.length + rejectedEvents.length, + accepted_event_count: events.length, + rejected_event_count: rejectedEvents.length, + platform_counts: countPlatforms(events), + derived_signal_kind_counts: countSignalKinds(events), + project_evidence: buildEvidence(events.filter((event) => event.scope === "project")), + direction_evidence: buildEvidence(events.filter((event) => event.scope === "direction")), + observation_candidates: input.observation_candidates ?? [], + audit: { + rejected_events: rejectedEvents, + warnings: input.provider_result?.warnings ?? [], + }, + }; + + const redaction = assertPublicSafeAggregate(aggregate); + if (!redaction.ok) { + throw new Error(`external aggregate is not public-safe: ${redaction.reason_codes.join(",")}`); + } + + return aggregate; +} + +function buildEvidence(events: ExternalSignalEvent[]): ExternalEvidence[] { + const grouped = new Map(); + for (const event of events) { + const groupKey = `${event.scope}:${event.target_key}`; + grouped.set(groupKey, [...(grouped.get(groupKey) ?? []), event]); + } + + return Array.from(grouped.values()) + .map((groupEvents) => evidenceFromEvents(groupEvents)) + .sort((a, b) => a.target_key.localeCompare(b.target_key)); +} + +function evidenceFromEvents(events: ExternalSignalEvent[]): ExternalEvidence { + const firstEvent = events[0]; + if (!firstEvent) throw new Error("cannot build external evidence from empty events"); + const observedTimes = events.map((event) => event.observed_at).sort(); + + return { + evidence_id: `${firstEvent.scope}:${firstEvent.target_key}`, + event_ids: events.map((event) => event.event_id).sort(), + scope: firstEvent.scope, + target_key: firstEvent.target_key, + derived_signal_kinds: unique(events.flatMap((event) => event.derived_signal_kinds)).sort() as ExternalSignalKind[], + platforms: unique(events.map((event) => event.platform)).sort() as ExternalPlatform[], + named_registry_actors: buildNamedRegistryActors(events), + actor_tiers: countBy(events.map((event) => event.actor.effective_tier)), + actor_types: countBy(events.map((event) => event.actor.actor_type)), + mention_count: events.length, + distinct_actor_count: distinctActorIds(events).size, + top_tier_actor_count: topTierActorIds(events).size, + first_seen_at: observedTimes[0]!, + last_seen_at: observedTimes[observedTimes.length - 1]!, + }; +} + +function buildNamedRegistryActors(events: ExternalSignalEvent[]): ExternalNamedRegistryActor[] { + const byEntity = new Map(); + + for (const event of events) { + const actor = event.actor; + if ( + actor.tier_basis !== "registry" || + !actor.registry_entity_id || + !actor.registry_display_name || + !actor.registry_tier || + (actor.actor_type !== "institution" && actor.actor_type !== "team" && actor.actor_type !== "person") + ) { + continue; + } + + const existing = byEntity.get(actor.registry_entity_id); + if (!existing) { + byEntity.set(actor.registry_entity_id, { + actor: { + entity_id: actor.registry_entity_id, + display_name: actor.registry_display_name, + actor_type: actor.actor_type, + registry_tier: actor.registry_tier, + event_count: 1, + platforms: [event.platform], + first_seen_at: event.observed_at, + last_seen_at: event.observed_at, + }, + dates: [event.observed_at], + }); + continue; + } + + existing.actor.event_count += 1; + existing.actor.platforms = unique([...existing.actor.platforms, event.platform]).sort() as ExternalPlatform[]; + existing.dates.push(event.observed_at); + existing.dates.sort(); + existing.actor.first_seen_at = existing.dates[0]!; + existing.actor.last_seen_at = existing.dates[existing.dates.length - 1]!; + } + + return Array.from(byEntity.values()) + .map((entry) => entry.actor) + .sort( + (a, b) => + registryTierRank[a.registry_tier] - registryTierRank[b.registry_tier] || + b.event_count - a.event_count || + a.display_name.localeCompare(b.display_name), + ); +} + +function countPlatforms(events: ExternalSignalEvent[]): Partial> { + return countBy(events.map((event) => event.platform)); +} + +function countSignalKinds(events: ExternalSignalEvent[]): Partial> { + return countBy(events.flatMap((event) => event.derived_signal_kinds)); +} + +function countBy(values: T[]): Partial> { + const counts: Partial> = {}; + for (const value of values) { + counts[value] = (counts[value] ?? 0) + 1; + } + return counts; +} + +function distinctActorIds(events: ExternalSignalEvent[]): Set { + return new Set( + events.map((event, index) => event.actor.registry_entity_id ?? event.actor.provider_actor_id ?? `${event.actor.actor_type}:${index}`), + ); +} + +function topTierActorIds(events: ExternalSignalEvent[]): Set { + return new Set( + events + .filter((event) => event.actor.tier_basis === "registry" && event.actor.registry_entity_id && event.actor.registry_tier) + .map((event) => event.actor.registry_entity_id!), + ); +} + +function unique(values: T[]): T[] { + return Array.from(new Set(values)); +} diff --git a/src/externalDiscovery/entityRegistry.ts b/src/externalDiscovery/entityRegistry.ts new file mode 100644 index 0000000..10c69ea --- /dev/null +++ b/src/externalDiscovery/entityRegistry.ts @@ -0,0 +1,153 @@ +import fs from "node:fs"; + +import type { + ExternalActorType, + ExternalRegistryTier, + ExternalSignalActor, + ExternalTierBasis, +} from "./types.ts"; + +export interface ExternalEntityRegistryEntry { + entity_id: string; + display_name: string; + actor_type: "institution" | "team" | "person"; + tier: ExternalRegistryTier; + handles: string[]; + profile_urls: string[]; + updated_at: string; +} + +export interface EntityRegistryLookupResult { + actor: ExternalSignalActor; + warnings: Array<{ + reason_code: "registry_empty" | "registry_miss"; + reason_detail: string; + }>; +} + +export function readEntityRegistry(filepath: string): ExternalEntityRegistryEntry[] { + if (!fs.existsSync(filepath)) return []; + const value = JSON.parse(fs.readFileSync(filepath, "utf-8")) as unknown; + if (!Array.isArray(value)) return []; + return value.filter(isRegistryEntry); +} + +export function applyEntityRegistry( + actor: { + actor_type?: ExternalActorType; + provider_actor_id?: string; + provider_tier_hint?: ExternalSignalActor["provider_tier_hint"]; + registry_entity_id?: string; + }, + registry: ExternalEntityRegistryEntry[], +): EntityRegistryLookupResult { + if (registry.length === 0) { + return { + actor: anonymousActor(actor, "none"), + warnings: [{ reason_code: "registry_empty", reason_detail: "entity registry is empty or missing" }], + }; + } + + const matched = findRegistryMatch(actor, registry); + if (!matched) { + return { + actor: anonymousActor(actor, actor.provider_tier_hint ? "provider_hint" : "none"), + warnings: [{ reason_code: "registry_miss", reason_detail: "actor did not match entity registry" }], + }; + } + + return { + actor: { + actor_type: matched.actor_type, + effective_tier: matched.tier, + tier_basis: "registry", + provider_actor_id: actor.provider_actor_id, + provider_tier_hint: actor.provider_tier_hint, + registry_entity_id: matched.entity_id, + registry_display_name: matched.display_name, + registry_tier: matched.tier, + }, + warnings: [], + }; +} + +function findRegistryMatch( + actor: { + provider_actor_id?: string; + registry_entity_id?: string; + }, + registry: ExternalEntityRegistryEntry[], +): ExternalEntityRegistryEntry | undefined { + if (actor.registry_entity_id) { + const byEntityId = registry.find((entry) => entry.entity_id === actor.registry_entity_id); + if (byEntityId) return byEntityId; + } + + if (!actor.provider_actor_id) return undefined; + const providerActorId = actor.provider_actor_id.trim(); + if (providerActorId.length === 0) return undefined; + + const providerHandle = normalizeHandle(providerActorId); + const providerUrl = normalizeUrl(providerActorId); + return registry.find((entry) => { + if (providerActorId === entry.entity_id) return true; + if (entry.handles.map(normalizeHandle).includes(providerHandle)) return true; + if (entry.profile_urls.map(normalizeUrl).includes(providerUrl)) return true; + return false; + }); +} + +function normalizeHandle(value: string): string { + const trimmed = value.trim().toLowerCase(); + if (trimmed.startsWith("http://") || trimmed.startsWith("https://")) { + try { + const url = new URL(trimmed); + return url.pathname.split("/").filter(Boolean)[0]?.replace(/^@/, "") ?? ""; + } catch { + return trimmed.replace(/^@/, ""); + } + } + return trimmed.replace(/^@/, ""); +} + +function normalizeUrl(value: string): string { + try { + const url = new URL(value.trim().toLowerCase()); + url.hash = ""; + url.search = ""; + return url.toString().replace(/\/$/, ""); + } catch { + return value.trim().toLowerCase().replace(/\/$/, ""); + } +} + +function anonymousActor( + actor: { + actor_type?: ExternalActorType; + provider_actor_id?: string; + provider_tier_hint?: ExternalSignalActor["provider_tier_hint"]; + }, + tierBasis: ExternalTierBasis, +): ExternalSignalActor { + return { + actor_type: actor.actor_type ?? "unknown", + effective_tier: actor.actor_type === "unknown" || !actor.actor_type ? "unknown" : "ordinary", + tier_basis: tierBasis, + provider_actor_id: actor.provider_actor_id, + provider_tier_hint: actor.provider_tier_hint, + }; +} + +function isRegistryEntry(value: unknown): value is ExternalEntityRegistryEntry { + if (typeof value !== "object" || value === null || Array.isArray(value)) return false; + const entry = value as Partial; + return ( + typeof entry.entity_id === "string" && + typeof entry.display_name === "string" && + (entry.actor_type === "institution" || entry.actor_type === "team" || entry.actor_type === "person") && + (entry.tier === "core" || entry.tier === "proven" || entry.tier === "watch") && + Array.isArray(entry.handles) && + Array.isArray(entry.profile_urls) && + typeof entry.updated_at === "string" + ); +} diff --git a/src/externalDiscovery/matching.ts b/src/externalDiscovery/matching.ts new file mode 100644 index 0000000..4d5b351 --- /dev/null +++ b/src/externalDiscovery/matching.ts @@ -0,0 +1,54 @@ +import type { NormalizedProject } from "../types.ts"; +import type { ExternalSignalEvent, ObservationCandidate } from "./types.ts"; + +export function projectEvidenceTargetForEvent(event: ExternalSignalEvent, projects: Pick[]): string | null { + if (event.scope !== "project") return null; + if (event.target_type !== "project") return null; + const matched = projects.find((project) => project.repo_url === event.target_key || project.repo_full_name === event.target_key); + return matched?.repo_full_name ?? null; +} + +export function topicKeyFromHints(args: { + userInterestTopics?: string[]; + weeklyTrendKeys?: string[]; + providerTopicHint?: string; +}): string | null { + const userTopic = args.userInterestTopics?.find((topic) => topic.trim().length > 0); + if (userTopic) return kebabCase(userTopic); + const weeklyKey = args.weeklyTrendKeys?.find((key) => key.trim().length > 0); + if (weeklyKey) return kebabCase(weeklyKey); + if (args.providerTopicHint && args.providerTopicHint.trim().length > 0) return kebabCase(args.providerTopicHint); + return null; +} + +export function observationCandidateForUnmatchedEvent(event: ExternalSignalEvent): ObservationCandidate | null { + if (event.scope === "direction" && event.target_type === "topic") { + return { + candidate_kind: "direction", + target_key: event.target_key, + qualification: "direction_observation", + can_enter_daily: false, + can_enter_weekly: true, + cannot_be_primary_conclusion: true, + }; + } + if (event.scope === "project" && (event.target_type === "project" || event.target_type === "paper" || event.target_type === "product")) { + return { + candidate_kind: event.target_type, + target_key: event.target_key, + qualification: "needs_primary_confirmation", + can_enter_daily: true, + can_enter_weekly: true, + cannot_be_primary_conclusion: true, + }; + } + return null; +} + +function kebabCase(value: string): string { + return value + .trim() + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, ""); +} diff --git a/src/externalDiscovery/paths.ts b/src/externalDiscovery/paths.ts new file mode 100644 index 0000000..bf992b7 --- /dev/null +++ b/src/externalDiscovery/paths.ts @@ -0,0 +1,21 @@ +import path from "node:path"; + +export function externalRawInputPath(date: string): string { + return path.join("data", "raw", "external-discovery", `${date}.agent-reach.json`); +} + +export function externalAggregatePath(date: string): string { + return path.join("data", "external-discovery", `${date}.aggregate.json`); +} + +export function externalAggregateLatestPath(): string { + return path.join("data", "external-discovery", "latest.aggregate.json"); +} + +export function externalEntityRegistryPath(): string { + return path.join("data", "external-discovery", "entity-registry.json"); +} + +export function externalSanitizedFixtureDirPath(): string { + return path.join("data", "raw", "external-discovery", "fixtures"); +} diff --git a/src/externalDiscovery/redaction.ts b/src/externalDiscovery/redaction.ts new file mode 100644 index 0000000..af5ecec --- /dev/null +++ b/src/externalDiscovery/redaction.ts @@ -0,0 +1,86 @@ +import crypto from "node:crypto"; + +export const REDACTION_POLICY_VERSION = "external-discovery-redaction.v1"; + +export interface RedactionCheckResult { + ok: boolean; + reason_codes: string[]; +} + +const forbiddenKeys = new Set([ + "content_text", + "text", + "raw_text", + "full_text", + "provider_display_name", + "actor_display_name", + "handle", + "raw_handle", + "profile_url", + "platform_profile_url", + "private_diagnostics", + "provider_diagnostics", +]); + +const secretPattern = /\b(cookie|session|token|password|oauth)\b/i; +const profileUrlPattern = /^https?:\/\/(?:www\.)?(?:twitter\.com|x\.com|reddit\.com|news\.ycombinator\.com)\/[^\s/]+/i; + +export function stableSourceInputHash(input: string | Buffer): string { + return crypto.createHash("sha256").update(input).digest("hex").toLowerCase(); +} + +export function containsForbiddenPublicArtifactText(value: unknown): boolean { + return collectRedactionReasonCodes(value).length > 0; +} + +export function assertPublicSafeAggregate(value: unknown): RedactionCheckResult { + const reasonCodes = collectRedactionReasonCodes(value); + if (!isRecord(value)) { + reasonCodes.push("not_object"); + } else { + if (value.public_safe !== true) reasonCodes.push("public_safe_not_true"); + if (value.contains_raw_text !== false) reasonCodes.push("contains_raw_text_not_false"); + if (value.contains_profile_urls !== false) reasonCodes.push("contains_profile_urls_not_false"); + if (typeof value.redaction_policy_version !== "string" || value.redaction_policy_version.length === 0) { + reasonCodes.push("missing_redaction_policy_version"); + } + if (typeof value.source_input_hash !== "string" || value.source_input_hash.length === 0) { + reasonCodes.push("missing_source_input_hash"); + } + } + + return { + ok: reasonCodes.length === 0, + reason_codes: Array.from(new Set(reasonCodes)).sort(), + }; +} + +function collectRedactionReasonCodes(value: unknown): string[] { + const reasonCodes: string[] = []; + visit(value, (currentValue, key) => { + if (key && forbiddenKeys.has(key)) { + reasonCodes.push(`forbidden_key:${key}`); + } + if (typeof currentValue === "string") { + if (secretPattern.test(currentValue)) reasonCodes.push("forbidden_secret_text"); + if (profileUrlPattern.test(currentValue)) reasonCodes.push("forbidden_profile_url_text"); + } + }); + return reasonCodes; +} + +function visit(value: unknown, visitor: (value: unknown, key?: string) => void, key?: string): void { + visitor(value, key); + if (Array.isArray(value)) { + for (const item of value) visit(item, visitor); + return; + } + if (!isRecord(value)) return; + for (const [childKey, childValue] of Object.entries(value)) { + visit(childValue, visitor, childKey); + } +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/src/externalDiscovery/types.ts b/src/externalDiscovery/types.ts new file mode 100644 index 0000000..4564606 --- /dev/null +++ b/src/externalDiscovery/types.ts @@ -0,0 +1,136 @@ +export type ExternalPlatform = "x_twitter" | "reddit" | "hacker_news" | "official_web" | "official_blog"; +export type ExternalProviderStatus = "ok" | "skipped" | "partial" | "failed"; +export type ExternalSignalKind = "discovery" | "evidence"; +export type ExternalRawEventKind = "mention" | "discussion" | "official_release" | "blog_post" | "question" | "showcase" | "unknown"; +export type ExternalTargetType = "project" | "paper" | "product" | "topic"; +export type ExternalActorType = "institution" | "team" | "person" | "community" | "unknown"; +export type ExternalActorTier = "core" | "proven" | "watch" | "ordinary" | "unknown"; +export type ExternalRegistryTier = "core" | "proven" | "watch"; +export type ExternalProviderTierHint = "core" | "proven" | "watch" | "ordinary" | "unknown"; +export type ExternalEvidenceScope = "project" | "direction"; +export type ExternalTierBasis = "registry" | "provider_hint" | "none"; + +export const EXTERNAL_PLATFORMS = ["x_twitter", "reddit", "hacker_news", "official_web", "official_blog"] as const; +export const EXTERNAL_TARGET_TYPES = ["project", "paper", "product", "topic"] as const; +export const EXTERNAL_ACTOR_TYPES = ["institution", "team", "person", "community", "unknown"] as const; +export const EXTERNAL_REGISTRY_TIERS = ["core", "proven", "watch"] as const; + +export interface ExternalSignalActor { + actor_type: ExternalActorType; + effective_tier: ExternalActorTier; + tier_basis: ExternalTierBasis; + provider_actor_id?: string; + provider_tier_hint?: ExternalProviderTierHint; + registry_entity_id?: string; + registry_display_name?: string; + registry_tier?: ExternalRegistryTier; +} + +export interface ExternalSignalEvent { + event_id: string; + platform: ExternalPlatform; + raw_event_kind: ExternalRawEventKind; + derived_signal_kinds: ExternalSignalKind[]; + scope: ExternalEvidenceScope; + target_type: ExternalTargetType; + target_key: string; + actor: ExternalSignalActor; + observed_at: string; + source_published_at?: string; + ingested_at?: string; + url?: string; + raw_ref?: string; +} + +export interface ExternalNamedRegistryActor { + entity_id: string; + display_name: string; + actor_type: "institution" | "team" | "person"; + registry_tier: ExternalRegistryTier; + event_count: number; + platforms: ExternalPlatform[]; + first_seen_at: string; + last_seen_at: string; +} + +export interface ExternalEvidence { + evidence_id: string; + event_ids: string[]; + scope: ExternalEvidenceScope; + target_key: string; + derived_signal_kinds: ExternalSignalKind[]; + platforms: ExternalPlatform[]; + named_registry_actors: ExternalNamedRegistryActor[]; + actor_tiers: Partial>; + actor_types: Partial>; + mention_count: number; + distinct_actor_count: number; + top_tier_actor_count: number; + first_seen_at: string; + last_seen_at: string; +} + +export interface ObservationCandidate { + candidate_kind: "project" | "paper" | "product" | "direction"; + target_key: string; + qualification: "needs_primary_confirmation" | "direction_observation"; + can_enter_daily: boolean; + can_enter_weekly: boolean; + cannot_be_primary_conclusion: true; +} + +export interface ExternalDiscoveryAudit { + rejected_events: Array<{ + event_id?: string; + reason_code: string; + reason_detail: string; + }>; + warnings: Array<{ + reason_code: string; + reason_detail: string; + }>; +} + +export interface DailyExternalAggregate { + schema_version: "external-discovery.aggregate.v1"; + date: string; + generated_at: string; + provider: "agent-reach"; + provider_run_id?: string; + status: ExternalProviderStatus; + status_reason?: string; + source_input_hash: string; + public_safe: true; + redaction_policy_version: string; + contains_raw_text: false; + contains_profile_urls: false; + event_count: number; + accepted_event_count: number; + rejected_event_count: number; + platform_counts: Partial>; + derived_signal_kind_counts: Partial>; + project_evidence: ExternalEvidence[]; + direction_evidence: ExternalEvidence[]; + observation_candidates: ObservationCandidate[]; + audit: ExternalDiscoveryAudit; +} + +export interface ProviderRejectedEvent { + event_id?: string; + reason_code: string; + reason_detail: string; +} + +export interface AgentReachProviderReadResult { + provider: "agent-reach"; + schema_version: "agent-reach.external-discovery.v1"; + provider_run_id?: string; + generated_at?: string; + status: ExternalProviderStatus; + status_reason?: string; + platforms: ExternalPlatform[]; + events: ExternalSignalEvent[]; + rejected_events: ProviderRejectedEvent[]; + warnings: ExternalDiscoveryAudit["warnings"]; + source_input_hash: string; +} diff --git a/src/storage/files.ts b/src/storage/files.ts index 3694029..b9dd780 100644 --- a/src/storage/files.ts +++ b/src/storage/files.ts @@ -7,6 +7,7 @@ export const DATA_DIRS = [ "data/raw/github", "data/raw/github-stars", "data/raw/trendshift", + "data/external-discovery", "data/classifications", "data/normalized", "data/scores", From 814f48727eacf65aefc7799223fb10905888f3f7 Mon Sep 17 00:00:00 2001 From: Aspetta <892024285@qq.com> Date: Tue, 30 Jun 2026 21:53:55 +0800 Subject: [PATCH 2/8] feat(agentreach): add named actor registry roles --- .../externalDiscoveryAdapter.test.ts | 205 ++++++++++++++- .../externalDiscoveryAggregate.test.ts | 37 ++- .../externalDiscoveryEntityRegistry.test.ts | 109 +++++++- .../externalDiscoveryRedaction.test.ts | 2 + .../externalDiscoveryTypeContract.test.ts | 3 +- .../externalDiscoveryVerification.test.ts | 215 +++++++++++++++ src/action/dailyVerification.ts | 197 +++++++++++--- src/externalDiscovery/agentReachProvider.ts | 52 ++-- src/externalDiscovery/aggregate.ts | 43 ++- src/externalDiscovery/entityRegistry.ts | 246 ++++++++++++++++-- src/externalDiscovery/types.ts | 10 + 11 files changed, 1031 insertions(+), 88 deletions(-) create mode 100644 src/__tests__/externalDiscoveryVerification.test.ts diff --git a/src/__tests__/externalDiscoveryAdapter.test.ts b/src/__tests__/externalDiscoveryAdapter.test.ts index 7789e8f..96be54f 100644 --- a/src/__tests__/externalDiscoveryAdapter.test.ts +++ b/src/__tests__/externalDiscoveryAdapter.test.ts @@ -131,6 +131,7 @@ describe("agent reach provider artifact adapter", () => { display_name: "OpenAI", actor_type: "institution", registry_tier: "core", + source_roles: ["social_discussant"], event_count: 1, platforms: ["x_twitter"], first_seen_at: "2026-06-30T00:00:00.000Z", @@ -139,6 +140,208 @@ describe("agent reach provider artifact adapter", () => { ]); }); + it("does not turn social display names into named actors without a strong identifier", () => { + const filepath = tempFile("agent-reach-display-name-only.json"); + fs.writeFileSync( + filepath, + JSON.stringify({ + provider: "agent-reach", + schema_version: "agent-reach.external-discovery.v1", + provider_run_id: "run-display-name-only", + generated_at: "2026-06-30T00:00:00.000Z", + query: { keyword: "agents sdk" }, + platforms: ["x_twitter"], + status: "ok", + items: [ + { + event_id: "evt-display-name-only", + platform: "x_twitter", + raw_event_kind: "discussion", + derived_signal_kinds: ["evidence"], + scope: "project", + target_type: "project", + target_key: "openai/agents-sdk", + actor: { + actor_type: "institution", + display_name: "OpenAI", + provider_tier_hint: "core", + }, + observed_at: "2026-06-30T00:00:00.000Z", + raw_ref: "provider:event:display-name-only", + }, + ], + }), + "utf-8", + ); + + const result = readAgentReachProviderArtifact(filepath, { + explicitInput: true, + entityRegistry: [ + { + entity_id: "entity-openai", + display_name: "OpenAI", + actor_type: "institution", + tier: "core", + handles: ["openai"], + profile_urls: ["https://x.com/openai"], + updated_at: "2026-06-30T00:00:00.000Z", + }, + ], + }); + const aggregate = buildDailyExternalAggregate({ + date: "2026-06-30", + generated_at: "2026-06-30T01:00:00.000Z", + provider_result: result, + }); + + expect(result.warnings[0]?.reason_code).toBe("registry_miss"); + expect(aggregate.project_evidence[0]?.named_registry_actors).toEqual([]); + }); + + it("keeps identity hashes out of registry actor matching", () => { + const filepath = tempFile("agent-reach-identity-hash-not-registry.json"); + fs.writeFileSync( + filepath, + JSON.stringify({ + provider: "agent-reach", + schema_version: "agent-reach.external-discovery.v1", + provider_run_id: "run-identity-hash", + generated_at: "2026-06-30T00:00:00.000Z", + query: { keyword: "agents sdk" }, + platforms: ["x_twitter"], + status: "ok", + items: [ + { + event_id: "evt-identity-hash", + platform: "x_twitter", + raw_event_kind: "discussion", + derived_signal_kinds: ["evidence"], + scope: "project", + target_type: "project", + target_key: "openai/agents-sdk", + actor: { + actor_type: "community", + identity_hash: "openai", + provider_tier_hint: "core", + }, + observed_at: "2026-06-30T00:00:00.000Z", + raw_ref: "provider:event:identity-hash", + }, + ], + }), + "utf-8", + ); + + const result = readAgentReachProviderArtifact(filepath, { + explicitInput: true, + entityRegistry: [ + { + entity_id: "entity-openai", + display_name: "OpenAI", + actor_type: "institution", + tier: "core", + handles: ["openai"], + profile_urls: ["https://x.com/openai"], + updated_at: "2026-06-30T00:00:00.000Z", + }, + ], + }); + const aggregate = buildDailyExternalAggregate({ + date: "2026-06-30", + generated_at: "2026-06-30T01:00:00.000Z", + provider_result: result, + }); + + expect(result.events[0]?.actor).toMatchObject({ + identity_hash: "openai", + tier_basis: "provider_hint", + }); + expect(result.events[0]?.actor.provider_actor_id).toBeUndefined(); + expect(result.warnings[0]?.reason_code).toBe("registry_miss"); + expect(aggregate.project_evidence[0]?.named_registry_actors).toEqual([]); + expect(aggregate.project_evidence[0]?.distinct_actor_count).toBe(1); + }); + + it("uses profile URLs and official owner context as registry-positive inputs", () => { + const filepath = tempFile("agent-reach-profile-and-owner.json"); + fs.writeFileSync( + filepath, + JSON.stringify({ + provider: "agent-reach", + schema_version: "agent-reach.external-discovery.v1", + provider_run_id: "run-profile-owner", + generated_at: "2026-06-30T00:00:00.000Z", + query: { keyword: "agents sdk" }, + platforms: ["x_twitter", "official_web"], + status: "ok", + items: [ + { + event_id: "evt-profile-url", + platform: "x_twitter", + raw_event_kind: "discussion", + derived_signal_kinds: ["evidence"], + scope: "project", + target_type: "project", + target_key: "openai/agents-sdk", + actor: { + actor_type: "institution", + platform_profile_url: "https://x.com/OpenAI", + }, + observed_at: "2026-06-30T00:00:00.000Z", + raw_ref: "provider:event:profile-url", + }, + { + event_id: "evt-github-owner", + platform: "official_web", + raw_event_kind: "official_release", + derived_signal_kinds: ["evidence"], + scope: "project", + target_type: "project", + target_key: "openai/agents-sdk", + actor: { + actor_type: "institution", + registry_entity_id: "entity-openai", + }, + target: { + repo_url: "https://github.com/openai/agents-sdk", + }, + observed_at: "2026-06-30T01:00:00.000Z", + url: "https://github.com/openai/agents-sdk/releases/tag/v1", + }, + ], + }), + "utf-8", + ); + + const result = readAgentReachProviderArtifact(filepath, { + explicitInput: true, + entityRegistry: [ + { + entity_id: "entity-openai", + display_name: "OpenAI", + actor_type: "institution", + tier: "core", + handles: ["openai"], + profile_urls: ["https://x.com/openai"], + domains: ["openai.com"], + github_owners: ["openai"], + updated_at: "2026-06-30T00:00:00.000Z", + }, + ], + }); + const aggregate = buildDailyExternalAggregate({ + date: "2026-06-30", + generated_at: "2026-06-30T01:00:00.000Z", + provider_result: result, + }); + + expect(aggregate.project_evidence[0]?.named_registry_actors[0]).toMatchObject({ + display_name: "OpenAI", + event_count: 2, + source_roles: ["social_discussant", "official_publisher", "official_owner"], + }); + }); + it("accepts nested target and identity hash fields from real AgentReach artifacts", () => { const filepath = tempFile("agent-reach-real-shape.json"); fs.writeFileSync( @@ -184,7 +387,7 @@ describe("agent reach provider artifact adapter", () => { target_key: "new agent memory framework", actor: { actor_type: "community", - provider_actor_id: "sha256:community-actor", + identity_hash: "sha256:community-actor", }, }); expect(result.events[0]?.event_id).toMatch(/^agent-reach:/); diff --git a/src/__tests__/externalDiscoveryAggregate.test.ts b/src/__tests__/externalDiscoveryAggregate.test.ts index 30478de..18fa94a 100644 --- a/src/__tests__/externalDiscoveryAggregate.test.ts +++ b/src/__tests__/externalDiscoveryAggregate.test.ts @@ -18,6 +18,7 @@ function event(overrides: Partial): ExternalSignalEvent { registry_entity_id: "entity-openai", registry_display_name: "OpenAI", registry_tier: "core", + source_roles: ["social_discussant"], }, observed_at: "2026-06-30T00:00:00.000Z", raw_ref: "provider:event:1", @@ -54,6 +55,7 @@ describe("external discovery aggregate", () => { display_name: "OpenAI", actor_type: "institution", registry_tier: "core", + source_roles: ["social_discussant"], event_count: 1, platforms: ["x_twitter"], first_seen_at: "2026-06-30T00:00:00.000Z", @@ -101,14 +103,47 @@ describe("external discovery aggregate", () => { registry_entity_id: "entity-zeta", registry_display_name: "Zeta Lab", registry_tier: "watch", + source_roles: ["official_owner"], }, }), event({ event_id: "evt-core-1" }), - event({ event_id: "evt-core-2", platform: "official_blog", observed_at: "2026-06-30T03:00:00.000Z" }), + event({ event_id: "evt-core-2", platform: "official_blog", observed_at: "2026-06-30T03:00:00.000Z", actor: { + actor_type: "institution", + effective_tier: "core", + tier_basis: "registry", + registry_entity_id: "entity-openai", + registry_display_name: "OpenAI", + registry_tier: "core", + source_roles: ["official_publisher"], + } }), ], }); expect(aggregate.project_evidence[0]?.named_registry_actors.map((actor) => actor.display_name)).toEqual(["OpenAI", "Zeta Lab"]); expect(aggregate.project_evidence[0]?.named_registry_actors[0]?.event_count).toBe(2); + expect(aggregate.project_evidence[0]?.named_registry_actors[0]?.source_roles).toEqual(["social_discussant", "official_publisher"]); + }); + + it("does not publish registry actors that miss source roles", () => { + const aggregate = buildDailyExternalAggregate({ + date: "2026-06-30", + generated_at: "2026-06-30T01:00:00.000Z", + events: [ + event({ + actor: { + actor_type: "institution", + effective_tier: "core", + tier_basis: "registry", + registry_entity_id: "entity-openai", + registry_display_name: "OpenAI", + registry_tier: "core", + }, + }), + ], + }); + + expect(aggregate.project_evidence[0]?.named_registry_actors).toEqual([]); + expect(aggregate.project_evidence[0]?.top_tier_actor_count).toBe(0); + expect(aggregate.audit.warnings[0]?.reason_code).toBe("named_actor_missing_source_roles"); }); }); diff --git a/src/__tests__/externalDiscoveryEntityRegistry.test.ts b/src/__tests__/externalDiscoveryEntityRegistry.test.ts index a6d8a92..a2b871b 100644 --- a/src/__tests__/externalDiscoveryEntityRegistry.test.ts +++ b/src/__tests__/externalDiscoveryEntityRegistry.test.ts @@ -1,5 +1,26 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; import { describe, expect, it } from "vitest"; -import { applyEntityRegistry, type ExternalEntityRegistryEntry } from "../externalDiscovery/entityRegistry.ts"; +import { applyEntityRegistry, readEntityRegistryWithWarnings, type EntityRegistryLookupContext, type ExternalEntityRegistryEntry } from "../externalDiscovery/entityRegistry.ts"; + +const socialContext: EntityRegistryLookupContext = { + platform: "x_twitter", + raw_event_kind: "discussion", + url: "https://x.com/someone/status/1", +}; + +const officialBlogContext: EntityRegistryLookupContext = { + platform: "official_blog", + raw_event_kind: "blog_post", + url: "https://openai.com/blog/agents-sdk", +}; + +const githubOwnerContext: EntityRegistryLookupContext = { + platform: "official_web", + raw_event_kind: "official_release", + url: "https://github.com/openai/agents-sdk/releases/tag/v1", +}; const registry: ExternalEntityRegistryEntry[] = [ { @@ -7,15 +28,23 @@ const registry: ExternalEntityRegistryEntry[] = [ display_name: "OpenAI", actor_type: "institution", tier: "core", + aliases: ["OpenAI"], handles: ["openai"], profile_urls: ["https://x.com/openai"], + domains: ["openai.com"], + github_owners: ["openai"], updated_at: "2026-06-30T00:00:00.000Z", }, ]; +function tempFile(name: string): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "external-registry-")); + return path.join(dir, name); +} + describe("external discovery entity registry", () => { - it("promotes only registry hits to registry-based effective tiers", () => { - const result = applyEntityRegistry({ registry_entity_id: "entity-openai", provider_actor_id: "raw-1", provider_tier_hint: "core" }, registry); + it("promotes registry entity id hits only when a source role can be resolved", () => { + const result = applyEntityRegistry({ registry_entity_id: "entity-openai", provider_actor_id: "raw-1", provider_tier_hint: "core" }, registry, socialContext); expect(result.warnings).toEqual([]); expect(result.actor).toMatchObject({ @@ -25,25 +54,57 @@ describe("external discovery entity registry", () => { registry_entity_id: "entity-openai", registry_display_name: "OpenAI", registry_tier: "core", + source_roles: ["social_discussant"], }); }); - it("matches provider actor ids against registry handles", () => { - const result = applyEntityRegistry({ actor_type: "institution", provider_actor_id: "@openai", provider_tier_hint: "core" }, registry); + it("does not publish registry entity id hits without role context", () => { + const result = applyEntityRegistry({ registry_entity_id: "entity-openai", provider_actor_id: "raw-1", provider_tier_hint: "core" }, registry); - expect(result.warnings).toEqual([]); - expect(result.actor).toMatchObject({ - actor_type: "institution", + expect(result.warnings[0]?.reason_code).toBe("named_actor_role_unresolved"); + expect(result.actor.tier_basis).toBe("provider_hint"); + expect(result.actor.registry_entity_id).toBeUndefined(); + }); + + it("matches handles and profile URLs as social discussants", () => { + const handleResult = applyEntityRegistry({ actor_type: "institution", handle: "@openai", provider_tier_hint: "core" }, registry, socialContext); + const profileResult = applyEntityRegistry({ actor_type: "institution", platform_profile_url: "https://x.com/OpenAI" }, registry, socialContext); + + expect(handleResult.warnings).toEqual([]); + expect(handleResult.actor).toMatchObject({ effective_tier: "core", tier_basis: "registry", - registry_entity_id: "entity-openai", registry_display_name: "OpenAI", - registry_tier: "core", + source_roles: ["social_discussant"], }); + expect(profileResult.actor.source_roles).toEqual(["social_discussant"]); + }); + + it("does not match display names or provider tier hints by themselves", () => { + const result = applyEntityRegistry({ actor_type: "institution", display_name: "OpenAI", provider_tier_hint: "core" }, registry, socialContext); + + expect(result.warnings[0]?.reason_code).toBe("registry_miss"); + expect(result.actor.effective_tier).toBe("ordinary"); + expect(result.actor.registry_entity_id).toBeUndefined(); + }); + + it("does not treat bare provider actor ids as registry handles", () => { + const result = applyEntityRegistry({ actor_type: "institution", provider_actor_id: "openai", provider_tier_hint: "core" }, registry, socialContext); + + expect(result.warnings[0]?.reason_code).toBe("registry_miss"); + expect(result.actor.registry_entity_id).toBeUndefined(); + }); + + it("matches official publisher and official owner roles from source context", () => { + const publisher = applyEntityRegistry({ registry_entity_id: "entity-openai" }, registry, officialBlogContext); + const owner = applyEntityRegistry({ registry_entity_id: "entity-openai" }, registry, githubOwnerContext); + + expect(publisher.actor.source_roles).toEqual(["official_publisher"]); + expect(owner.actor.source_roles).toEqual(["official_publisher", "official_owner"]); }); it("keeps provider tier hints anonymous when registry misses", () => { - const result = applyEntityRegistry({ actor_type: "person", provider_actor_id: "raw-2", provider_tier_hint: "core" }, registry); + const result = applyEntityRegistry({ actor_type: "person", provider_actor_id: "raw-2", provider_tier_hint: "core" }, registry, socialContext); expect(result.warnings[0]?.reason_code).toBe("registry_miss"); expect(result.actor.effective_tier).toBe("ordinary"); @@ -52,10 +113,34 @@ describe("external discovery entity registry", () => { }); it("allows empty registry startup without creating top-tier actors", () => { - const result = applyEntityRegistry({ actor_type: "community", provider_tier_hint: "watch" }, []); + const result = applyEntityRegistry({ actor_type: "community", provider_tier_hint: "watch" }, [], socialContext); expect(result.warnings[0]?.reason_code).toBe("registry_empty"); expect(result.actor.effective_tier).toBe("ordinary"); expect(result.actor.registry_tier).toBeUndefined(); }); + + it("reports invalid registry entries without allowing them to match", () => { + const filepath = tempFile("entity-registry.json"); + fs.writeFileSync( + filepath, + JSON.stringify([ + { + entity_id: "entity-unsafe", + display_name: "Unsafe", + actor_type: "institution", + tier: "core", + handles: ["unsafe"], + profile_urls: [], + private_diagnostics: "token should not be here", + updated_at: "2026-06-30T00:00:00.000Z", + }, + ]), + "utf-8", + ); + + const result = readEntityRegistryWithWarnings(filepath); + expect(result.entries).toEqual([]); + expect(result.warnings[0]?.reason_code).toBe("registry_invalid"); + }); }); diff --git a/src/__tests__/externalDiscoveryRedaction.test.ts b/src/__tests__/externalDiscoveryRedaction.test.ts index e098ac0..e4527ca 100644 --- a/src/__tests__/externalDiscoveryRedaction.test.ts +++ b/src/__tests__/externalDiscoveryRedaction.test.ts @@ -14,6 +14,7 @@ function safeAggregate(overrides: Record = {}): Record { display_name: "OpenAI", actor_type: "institution", registry_tier: "core", + source_roles: ["social_discussant"], event_count: 1, platforms: ["x_twitter"], first_seen_at: "2026-06-30T00:00:00.000Z", diff --git a/src/__tests__/externalDiscoveryTypeContract.test.ts b/src/__tests__/externalDiscoveryTypeContract.test.ts index 1706a66..df7a757 100644 --- a/src/__tests__/externalDiscoveryTypeContract.test.ts +++ b/src/__tests__/externalDiscoveryTypeContract.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; import { externalAggregateLatestPath, externalAggregatePath, externalEntityRegistryPath, externalRawInputPath, externalSanitizedFixtureDirPath } from "../externalDiscovery/paths.ts"; -import { EXTERNAL_PLATFORMS, EXTERNAL_TARGET_TYPES } from "../externalDiscovery/types.ts"; +import { EXTERNAL_NAMED_ACTOR_SOURCE_ROLES, EXTERNAL_PLATFORMS, EXTERNAL_TARGET_TYPES } from "../externalDiscovery/types.ts"; describe("external discovery type and path contract", () => { it("freezes the V1 platform and target enums", () => { @@ -9,6 +9,7 @@ describe("external discovery type and path contract", () => { expect(EXTERNAL_TARGET_TYPES).toEqual(["project", "paper", "product", "topic"]); expect(EXTERNAL_TARGET_TYPES).not.toContain("direction"); expect(EXTERNAL_TARGET_TYPES).not.toContain("unknown"); + expect(EXTERNAL_NAMED_ACTOR_SOURCE_ROLES).toEqual(["social_discussant", "official_publisher", "official_owner"]); }); it("keeps external discovery raw input and public aggregate paths separate", () => { diff --git a/src/__tests__/externalDiscoveryVerification.test.ts b/src/__tests__/externalDiscoveryVerification.test.ts new file mode 100644 index 0000000..1d3bf5b --- /dev/null +++ b/src/__tests__/externalDiscoveryVerification.test.ts @@ -0,0 +1,215 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { buildVerifyDailyResult } from "../action/dailyVerification.ts"; +import type { DailyReport, DailyRunSummary } from "../types.ts"; + +const roots: string[] = []; +const originalCwd = process.cwd(); +const date = "2026-06-30"; + +afterEach(() => { + process.chdir(originalCwd); + for (const root of roots.splice(0)) { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +function setupWorkspace(): string { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "external-discovery-verification-")); + roots.push(root); + fs.mkdirSync(path.join(root, "data", "reports"), { recursive: true }); + fs.mkdirSync(path.join(root, "data", "raw", "github"), { recursive: true }); + fs.mkdirSync(path.join(root, "data", "external-discovery"), { recursive: true }); + process.chdir(root); + return root; +} + +function writeJson(filepath: string, value: unknown): void { + fs.mkdirSync(path.dirname(filepath), { recursive: true }); + fs.writeFileSync(filepath, JSON.stringify(value, null, 2)); +} + +function makeSummary(): DailyRunSummary { + return { + date, + generated_at: "2026-06-30T08:00:00.000Z", + dry_run: true, + minimum_viable_run_completed: true, + completion_notes: [], + counts: { + raw_signals: 10, + normalized_projects: 4, + scored_projects: 4, + high_score_projects: 2, + anomaly_projects: 0, + new_projects: 4, + classifications: 4, + }, + source_status: [ + { + source: "github_trending", + enabled: true, + item_count: 10, + distinct_projects: 4, + status: "active", + notes: [], + }, + ], + quality: { + missing_descriptions: 0, + watchlist_hits: 0, + low_confidence_projects: 0, + medium_confidence_projects: 0, + insufficient_metrics_projects: 0, + suspicious_growth_projects: 0, + single_source_projects: 0, + single_spike_projects: 0, + emerging_projects: 4, + persistent_projects: 0, + }, + diagnostics: { + anomaly_share: 0, + uniform_star_velocity_detected: false, + metrics_source_distribution: { embedded: 0, github_api: 0, github_html: 0, github_cache: 0, unavailable: 0 }, + star_delta_source_distribution: { github_live: 0, github_snapshot: 0, signal: 0, unavailable: 0 }, + github_star_delta: { + live_delta_attempts: 0, + live_delta_success: 0, + snapshot_delta_success: 0, + token_missing: 0, + auth_invalid: 0, + rate_limit: 0, + network_blocked: 0, + }, + }, + top_projects: [], + observer_top_candidates: [], + watchouts: [], + next_focus: [], + recommended_actions: [], + freshness_sources: [], + mission_discovery_status: "degraded", + mission_degraded_reason_codes: ["no_matched_direction"], + }; +} + +function makeReport(): DailyReport { + return { + date, + generated_at: "2026-06-30T08:00:00.000Z", + enhancement_status: "rules-only", + enhancement_audit: { rejected_outputs: [] }, + personalized_relevance_applicable: false, + overall_daily_status: "数据新鲜，可直接阅读", + freshness_sources: [], + today_fresh_candidate_count: 1, + context_candidate_count: 1, + pending_confirmation_count: 0, + main_board_mode: "fresh_today_only", + today_star_projects: [], + context_only_projects: [], + new_projects: [], + high_score_projects: [], + anomaly_projects: [], + all_projects: [], + today_pulse_projects: [], + mission_match_projects: [], + explore_ribbon_projects: [], + coverage_atlas: [], + gap_ledger: [], + mission_discovery_status: "degraded", + mission_degraded_reason_codes: ["no_matched_direction"], + global_hot_projects: [], + demand_relevant_projects: [], + searched_direction_statuses: [], + } as DailyReport; +} + +function makeExternalAggregate(namedActorOverrides: Record = {}): Record { + return { + schema_version: "external-discovery.aggregate.v1", + date, + generated_at: "2026-06-30T08:00:00.000Z", + provider: "agent-reach", + status: "ok", + source_input_hash: "abc123", + public_safe: true, + redaction_policy_version: "external-discovery-redaction.v1", + contains_raw_text: false, + contains_profile_urls: false, + event_count: 1, + accepted_event_count: 1, + rejected_event_count: 0, + platform_counts: { x_twitter: 1 }, + derived_signal_kind_counts: { evidence: 1 }, + project_evidence: [ + { + evidence_id: "project:openai/agents-sdk", + event_ids: ["evt-1"], + scope: "project", + target_key: "openai/agents-sdk", + derived_signal_kinds: ["evidence"], + platforms: ["x_twitter"], + named_registry_actors: [ + { + entity_id: "entity-openai", + display_name: "OpenAI", + actor_type: "institution", + registry_tier: "core", + source_roles: ["social_discussant"], + event_count: 1, + platforms: ["x_twitter"], + first_seen_at: "2026-06-30T00:00:00.000Z", + last_seen_at: "2026-06-30T00:00:00.000Z", + ...namedActorOverrides, + }, + ], + actor_tiers: { core: 1 }, + actor_types: { institution: 1 }, + mention_count: 1, + distinct_actor_count: 1, + top_tier_actor_count: 1, + first_seen_at: "2026-06-30T00:00:00.000Z", + last_seen_at: "2026-06-30T00:00:00.000Z", + }, + ], + direction_evidence: [], + observation_candidates: [], + audit: { rejected_events: [], warnings: [] }, + }; +} + +function writeDailyInputs(root: string, externalAggregate: Record): void { + writeJson(path.join(root, "data", "reports", `${date}.run-summary.json`), makeSummary()); + writeJson(path.join(root, "data", "reports", `${date}.daily.json`), makeReport()); + writeJson(path.join(root, "data", "raw", "github", `${date}.enrichment.json`), []); + writeJson(path.join(root, "data", "external-discovery", `${date}.aggregate.json`), externalAggregate); +} + +describe("external discovery daily verification contract", () => { + it("passes public-safe external aggregates with source roles", () => { + const root = setupWorkspace(); + writeDailyInputs(root, makeExternalAggregate()); + + const result = buildVerifyDailyResult(date); + const check = result.checks.find((item) => item.name === "external_discovery_aggregate_contract"); + + expect(check?.status).toBe("pass"); + expect(check?.detail).toContain("named_actor_rows=1"); + }); + + it("fails persisted external aggregates with named actors missing source roles", () => { + const root = setupWorkspace(); + const aggregate = makeExternalAggregate({ source_roles: undefined }); + writeDailyInputs(root, aggregate); + + const result = buildVerifyDailyResult(date); + const check = result.checks.find((item) => item.name === "external_discovery_aggregate_contract"); + + expect(check?.status).toBe("fail"); + expect(check?.detail).toContain("source_roles must be non-empty"); + expect(result.status).toBe("fail"); + }); +}); diff --git a/src/action/dailyVerification.ts b/src/action/dailyVerification.ts index f2667cf..85b9507 100644 --- a/src/action/dailyVerification.ts +++ b/src/action/dailyVerification.ts @@ -1,5 +1,8 @@ -import path from "node:path"; -import { readJsonFile } from "../storage/files.ts"; +import fs from "node:fs"; +import path from "node:path"; +import { externalAggregatePath } from "../externalDiscovery/paths.ts"; +import { assertPublicSafeAggregate } from "../externalDiscovery/redaction.ts"; +import { readJsonFile } from "../storage/files.ts"; import type { DailyReport, DailyRunSummary, @@ -18,9 +21,15 @@ function githubAuditPath(date: string): string { return path.join("data", "raw", "github", `${date}.enrichment.json`); } -function dailyReportPath(date: string): string { - return path.join("data", "reports", `${date}.daily.json`); -} +function dailyReportPath(date: string): string { + return path.join("data", "reports", `${date}.daily.json`); +} + +interface OptionalJsonRead { + exists: boolean; + value?: unknown; + error?: string; +} function aggregateStatus(checks: VerificationCheck[]): VerifyDailyResult["status"] { if (checks.some((check) => check.status === "fail")) return "fail"; @@ -267,7 +276,7 @@ function githubAuditCheck( ); } -function projectSearchContractChecks(summary: DailyRunSummary, report: DailyReport | null): VerificationCheck[] { +function projectSearchContractChecks(summary: DailyRunSummary, report: DailyReport | null): VerificationCheck[] { if (!report) { return [buildCheck("project_search_daily_fields", "warn", "daily report missing; cannot verify project-search contract fields")]; } @@ -431,7 +440,114 @@ function projectSearchContractChecks(summary: DailyRunSummary, report: DailyRepo : "mission inventory audit fields are missing from run-summary", ), ]; -} +} + +function externalAggregateContractChecks(filepath: string, aggregateRead: OptionalJsonRead): VerificationCheck[] { + if (!aggregateRead.exists) { + return [ + buildCheck( + "external_discovery_aggregate_contract", + "pass", + `external aggregate not present at ${filepath}; external layer not run for this date`, + ), + ]; + } + + if (aggregateRead.error) { + return [buildCheck("external_discovery_aggregate_contract", "fail", `external aggregate unreadable: ${aggregateRead.error}`)]; + } + + const aggregate = aggregateRead.value; + const redaction = assertPublicSafeAggregate(aggregate); + const inspection = inspectExternalAggregateContract(aggregate); + const issues = [ + ...(!redaction.ok ? [`redaction=${redaction.reason_codes.join(",")}`] : []), + ...inspection.issues, + ]; + + return [ + buildCheck( + "external_discovery_aggregate_contract", + issues.length === 0 ? "pass" : "fail", + issues.length === 0 + ? `external aggregate public-safe; project_evidence=${inspection.projectEvidenceCount}; direction_evidence=${inspection.directionEvidenceCount}; named_actor_rows=${inspection.namedActorRows}` + : issues.join("; "), + ), + ]; +} + +function inspectExternalAggregateContract(value: unknown): { + projectEvidenceCount: number; + directionEvidenceCount: number; + namedActorRows: number; + issues: string[]; +} { + const issues: string[] = []; + if (!isRecord(value)) { + return { projectEvidenceCount: 0, directionEvidenceCount: 0, namedActorRows: 0, issues: ["external aggregate must be an object"] }; + } + + if (value.schema_version !== "external-discovery.aggregate.v1") { + issues.push("schema_version must be external-discovery.aggregate.v1"); + } + + const projectEvidence = evidenceArray(value.project_evidence, "project_evidence", issues); + const directionEvidence = evidenceArray(value.direction_evidence, "direction_evidence", issues); + let namedActorRows = 0; + + for (const [sectionName, evidenceRows] of [ + ["project_evidence", projectEvidence], + ["direction_evidence", directionEvidence], + ] as const) { + evidenceRows.forEach((evidence, evidenceIndex) => { + if (!isRecord(evidence)) { + issues.push(`${sectionName}[${evidenceIndex}] must be an object`); + return; + } + if (!Array.isArray(evidence.named_registry_actors)) { + issues.push(`${sectionName}[${evidenceIndex}].named_registry_actors must be an array`); + return; + } + evidence.named_registry_actors.forEach((actor, actorIndex) => { + namedActorRows += 1; + if (!isRecord(actor)) { + issues.push(`${sectionName}[${evidenceIndex}].named_registry_actors[${actorIndex}] must be an object`); + return; + } + const sourceRoles = actor.source_roles; + if (!Array.isArray(sourceRoles) || sourceRoles.length === 0) { + issues.push(`${sectionName}[${evidenceIndex}].named_registry_actors[${actorIndex}].source_roles must be non-empty`); + return; + } + const invalidRoles = sourceRoles.filter((role) => !isNamedActorSourceRole(role)); + if (invalidRoles.length > 0) { + issues.push(`${sectionName}[${evidenceIndex}].named_registry_actors[${actorIndex}].source_roles has invalid roles`); + } + }); + }); + } + + return { + projectEvidenceCount: projectEvidence.length, + directionEvidenceCount: directionEvidence.length, + namedActorRows, + issues, + }; +} + +function evidenceArray(value: unknown, name: string, issues: string[]): unknown[] { + if (Array.isArray(value)) return value; + issues.push(`${name} must be an array`); + return []; +} + +function isNamedActorSourceRole(value: unknown): boolean { + return value === "social_discussant" || value === "official_publisher" || value === "official_owner"; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} function defaultDiagnostics(): NonNullable { return { @@ -492,38 +608,59 @@ function normalizeSummaryDiagnostics(summary: DailyRunSummary): DailyRunSummary }; } -function buildChecks(summary: DailyRunSummary, githubAudit: GitHubEnrichmentAuditEntry[], report: DailyReport | null): VerificationCheck[] { - const checks = [ - ...completionChecks(summary), - ...sourceChecks(summary), - ...qualityChecks(summary), - ...llmChecks(summary), - ...freshnessChecks(summary), - ...projectSearchContractChecks(summary, report), - ]; - const githubCheck = githubAuditCheck(summary, githubAudit); - if (githubCheck) checks.push(githubCheck); - return checks; -} +function buildChecks( + summary: DailyRunSummary, + githubAudit: GitHubEnrichmentAuditEntry[], + report: DailyReport | null, + externalAggregateFilepath: string, + externalAggregate: OptionalJsonRead, +): VerificationCheck[] { + const checks = [ + ...completionChecks(summary), + ...sourceChecks(summary), + ...qualityChecks(summary), + ...llmChecks(summary), + ...freshnessChecks(summary), + ...projectSearchContractChecks(summary, report), + ...externalAggregateContractChecks(externalAggregateFilepath, externalAggregate), + ]; + const githubCheck = githubAuditCheck(summary, githubAudit); + if (githubCheck) checks.push(githubCheck); + return checks; +} + +function readOptionalJson(filepath: string): OptionalJsonRead { + if (!fs.existsSync(filepath)) return { exists: false }; + try { + return { exists: true, value: readJsonFile(filepath, null) }; + } catch (error) { + return { + exists: true, + error: error instanceof Error ? error.message : String(error), + }; + } +} /** * daily verification 负责把“这次产物能不能信”收敛成一组稳定检查项。 * 这里故意把主链路完成信号、source 健康度和数据异常诊断放在同一处，避免调用方各自拼装质检口径。 */ export function buildVerifyDailyResult(date: string): VerifyDailyResult { - const runSummaryPath = summaryPath(date); - const githubEnrichmentPath = githubAuditPath(date); - const reportPath = dailyReportPath(date); - const summary = readJsonFile(runSummaryPath, null); - const githubAudit = readJsonFile(githubEnrichmentPath, []); - const report = readJsonFile(reportPath, null); + const runSummaryPath = summaryPath(date); + const githubEnrichmentPath = githubAuditPath(date); + const reportPath = dailyReportPath(date); + const externalAggregateFilepath = externalAggregatePath(date); + const summary = readJsonFile(runSummaryPath, null); + const githubAudit = readJsonFile(githubEnrichmentPath, []); + const report = readJsonFile(reportPath, null); + const externalAggregate = readOptionalJson(externalAggregateFilepath); if (!summary) { return missingSummaryResult(date, runSummaryPath, githubEnrichmentPath); - } - - const normalizedSummary = normalizeSummaryDiagnostics(summary); - const checks = buildChecks(normalizedSummary, githubAudit, report); + } + + const normalizedSummary = normalizeSummaryDiagnostics(summary); + const checks = buildChecks(normalizedSummary, githubAudit, report, externalAggregateFilepath, externalAggregate); return { date, status: aggregateStatus(checks), diff --git a/src/externalDiscovery/agentReachProvider.ts b/src/externalDiscovery/agentReachProvider.ts index 5c7bdc2..639c395 100644 --- a/src/externalDiscovery/agentReachProvider.ts +++ b/src/externalDiscovery/agentReachProvider.ts @@ -1,6 +1,6 @@ import fs from "node:fs"; -import { applyEntityRegistry, readEntityRegistry, type ExternalEntityRegistryEntry } from "./entityRegistry.ts"; +import { applyEntityRegistry, readEntityRegistryWithWarnings, type ExternalEntityRegistryEntry } from "./entityRegistry.ts"; import { externalEntityRegistryPath } from "./paths.ts"; import { stableSourceInputHash } from "./redaction.ts"; import type { @@ -64,7 +64,7 @@ export function readAgentReachProviderArtifact(filepath: string, options: ReadOp const events: ExternalSignalEvent[] = []; const rejectedEvents: AgentReachProviderReadResult["rejected_events"] = []; const registry = resolveEntityRegistry(options); - const warnings: AgentReachProviderReadResult["warnings"] = []; + const warnings: AgentReachProviderReadResult["warnings"] = [...registry.warnings]; for (const item of artifact.items) { const event = parseEvent(item); @@ -96,36 +96,47 @@ export function readAgentReachProviderArtifact(filepath: string, options: ReadOp } function resolveEntityRegistry(options: ReadOptions): - | { - shouldApply: true; - entries: ExternalEntityRegistryEntry[]; - } - | { + | { + shouldApply: true; + entries: ExternalEntityRegistryEntry[]; + warnings: AgentReachProviderReadResult["warnings"]; + } + | { shouldApply: false; entries: []; + warnings: []; } { - if (options.entityRegistry) return { shouldApply: true, entries: options.entityRegistry }; + if (options.entityRegistry) return { shouldApply: true, entries: options.entityRegistry, warnings: [] }; const registryPath = options.entityRegistryPath ?? externalEntityRegistryPath(); - if (!fs.existsSync(registryPath)) return { shouldApply: false, entries: [] }; - return { shouldApply: true, entries: readEntityRegistry(registryPath) }; + if (!fs.existsSync(registryPath)) return { shouldApply: false, entries: [], warnings: [] }; + const registry = readEntityRegistryWithWarnings(registryPath); + return { shouldApply: true, entries: registry.entries, warnings: registry.warnings }; } function enrichEventActor( event: ExternalSignalEvent, registry: | { - shouldApply: true; - entries: ExternalEntityRegistryEntry[]; - } + shouldApply: true; + entries: ExternalEntityRegistryEntry[]; + warnings: AgentReachProviderReadResult["warnings"]; + } | { shouldApply: false; entries: []; + warnings: []; }, ): { event: ExternalSignalEvent; warnings: AgentReachProviderReadResult["warnings"] } { if (!registry.shouldApply) return { event, warnings: [] }; - const lookup = applyEntityRegistry(event.actor, registry.entries); + const lookup = applyEntityRegistry(event.actor, registry.entries, { + platform: event.platform, + raw_event_kind: event.raw_event_kind, + url: event.url, + target_url: event.target_url, + target_repo_url: event.target_repo_url, + }); return { event: { ...event, @@ -200,6 +211,10 @@ function parseEvent(value: unknown): effective_tier: isActorTier(actor.effective_tier) ? actor.effective_tier : "unknown", tier_basis: actor.tier_basis === "registry" || actor.tier_basis === "provider_hint" ? actor.tier_basis : isProviderTierHint(actor.provider_tier_hint) || isProviderTierHint(actor.tier_hint) ? "provider_hint" : "none", provider_actor_id: providerActorId(actor), + identity_hash: identityHash(actor), + display_name: typeof actor.display_name === "string" ? actor.display_name : undefined, + handle: typeof actor.handle === "string" ? actor.handle : undefined, + platform_profile_url: typeof actor.platform_profile_url === "string" ? actor.platform_profile_url : typeof actor.profile_url === "string" ? actor.profile_url : undefined, provider_tier_hint: isProviderTierHint(actor.provider_tier_hint) ? actor.provider_tier_hint : isProviderTierHint(actor.tier_hint) ? actor.tier_hint : undefined, registry_entity_id: typeof actor.registry_entity_id === "string" ? actor.registry_entity_id : undefined, registry_display_name: typeof actor.registry_display_name === "string" ? actor.registry_display_name : undefined, @@ -209,6 +224,8 @@ function parseEvent(value: unknown): source_published_at: typeof value.source_published_at === "string" ? value.source_published_at : undefined, ingested_at: typeof value.ingested_at === "string" ? value.ingested_at : undefined, url, + target_url: typeof target?.url === "string" ? target.url : undefined, + target_repo_url: typeof target?.repo_url === "string" ? target.repo_url : undefined, raw_ref: rawRef, }, }; @@ -250,7 +267,12 @@ function inferTargetKey(value: Record, target: Record): string | undefined { - const candidates = [actor.provider_actor_id, actor.identity_hash, actor.identity_id]; + const candidates = [actor.provider_actor_id]; + return candidates.find((candidate): candidate is string => typeof candidate === "string" && candidate.trim().length > 0)?.trim(); +} + +function identityHash(actor: Record): string | undefined { + const candidates = [actor.identity_hash, actor.identity_id]; return candidates.find((candidate): candidate is string => typeof candidate === "string" && candidate.trim().length > 0)?.trim(); } diff --git a/src/externalDiscovery/aggregate.ts b/src/externalDiscovery/aggregate.ts index 7bf24f5..f92b33a 100644 --- a/src/externalDiscovery/aggregate.ts +++ b/src/externalDiscovery/aggregate.ts @@ -6,6 +6,7 @@ import type { ExternalActorType, ExternalEvidence, ExternalNamedRegistryActor, + ExternalNamedActorSourceRole, ExternalPlatform, ExternalSignalEvent, ExternalSignalKind, @@ -53,7 +54,7 @@ export function buildDailyExternalAggregate(input: BuildDailyExternalAggregateIn observation_candidates: input.observation_candidates ?? [], audit: { rejected_events: rejectedEvents, - warnings: input.provider_result?.warnings ?? [], + warnings: [...(input.provider_result?.warnings ?? []), ...namedActorRoleWarnings(events)], }, }; @@ -110,6 +111,8 @@ function buildNamedRegistryActors(events: ExternalSignalEvent[]): ExternalNamedR !actor.registry_entity_id || !actor.registry_display_name || !actor.registry_tier || + !actor.source_roles || + actor.source_roles.length === 0 || (actor.actor_type !== "institution" && actor.actor_type !== "team" && actor.actor_type !== "person") ) { continue; @@ -123,6 +126,7 @@ function buildNamedRegistryActors(events: ExternalSignalEvent[]): ExternalNamedR display_name: actor.registry_display_name, actor_type: actor.actor_type, registry_tier: actor.registry_tier, + source_roles: uniqueRoles(actor.source_roles), event_count: 1, platforms: [event.platform], first_seen_at: event.observed_at, @@ -135,6 +139,7 @@ function buildNamedRegistryActors(events: ExternalSignalEvent[]): ExternalNamedR existing.actor.event_count += 1; existing.actor.platforms = unique([...existing.actor.platforms, event.platform]).sort() as ExternalPlatform[]; + existing.actor.source_roles = uniqueRoles([...existing.actor.source_roles, ...actor.source_roles]); existing.dates.push(event.observed_at); existing.dates.sort(); existing.actor.first_seen_at = existing.dates[0]!; @@ -151,6 +156,22 @@ function buildNamedRegistryActors(events: ExternalSignalEvent[]): ExternalNamedR ); } +function namedActorRoleWarnings(events: ExternalSignalEvent[]): Array<{ reason_code: string; reason_detail: string }> { + return events + .filter( + (event) => + event.actor.tier_basis === "registry" && + event.actor.registry_entity_id && + event.actor.registry_display_name && + event.actor.registry_tier && + (!event.actor.source_roles || event.actor.source_roles.length === 0), + ) + .map((event) => ({ + reason_code: "named_actor_missing_source_roles", + reason_detail: `registry actor ${event.actor.registry_entity_id} on event ${event.event_id} was not published as a named actor`, + })); +} + function countPlatforms(events: ExternalSignalEvent[]): Partial> { return countBy(events.map((event) => event.platform)); } @@ -169,18 +190,34 @@ function countBy(values: T[]): Partial> { function distinctActorIds(events: ExternalSignalEvent[]): Set { return new Set( - events.map((event, index) => event.actor.registry_entity_id ?? event.actor.provider_actor_id ?? `${event.actor.actor_type}:${index}`), + events.map((event, index) => event.actor.registry_entity_id ?? event.actor.identity_hash ?? event.actor.provider_actor_id ?? `${event.actor.actor_type}:${index}`), ); } function topTierActorIds(events: ExternalSignalEvent[]): Set { return new Set( events - .filter((event) => event.actor.tier_basis === "registry" && event.actor.registry_entity_id && event.actor.registry_tier) + .filter((event) => isPublishableNamedRegistryActor(event.actor)) .map((event) => event.actor.registry_entity_id!), ); } +function isPublishableNamedRegistryActor(actor: ExternalSignalEvent["actor"]): boolean { + return ( + actor.tier_basis === "registry" && + Boolean(actor.registry_entity_id) && + Boolean(actor.registry_display_name) && + Boolean(actor.registry_tier) && + Boolean(actor.source_roles?.length) && + (actor.actor_type === "institution" || actor.actor_type === "team" || actor.actor_type === "person") + ); +} + function unique(values: T[]): T[] { return Array.from(new Set(values)); } + +function uniqueRoles(values: ExternalNamedActorSourceRole[]): ExternalNamedActorSourceRole[] { + const order: ExternalNamedActorSourceRole[] = ["social_discussant", "official_publisher", "official_owner"]; + return order.filter((role) => values.includes(role)); +} diff --git a/src/externalDiscovery/entityRegistry.ts b/src/externalDiscovery/entityRegistry.ts index 10c69ea..a4976c8 100644 --- a/src/externalDiscovery/entityRegistry.ts +++ b/src/externalDiscovery/entityRegistry.ts @@ -2,6 +2,9 @@ import fs from "node:fs"; import type { ExternalActorType, + ExternalNamedActorSourceRole, + ExternalPlatform, + ExternalRawEventKind, ExternalRegistryTier, ExternalSignalActor, ExternalTierBasis, @@ -12,34 +15,82 @@ export interface ExternalEntityRegistryEntry { display_name: string; actor_type: "institution" | "team" | "person"; tier: ExternalRegistryTier; + aliases?: string[]; handles: string[]; profile_urls: string[]; + domains?: string[]; + github_owners?: string[]; updated_at: string; } +export type EntityRegistryWarningReason = "registry_empty" | "registry_miss" | "registry_invalid" | "named_actor_role_unresolved"; + +export interface EntityRegistryLookupContext { + platform: ExternalPlatform; + raw_event_kind: ExternalRawEventKind; + url?: string; + target_url?: string; + target_repo_url?: string; +} + export interface EntityRegistryLookupResult { actor: ExternalSignalActor; warnings: Array<{ - reason_code: "registry_empty" | "registry_miss"; + reason_code: EntityRegistryWarningReason; reason_detail: string; }>; } export function readEntityRegistry(filepath: string): ExternalEntityRegistryEntry[] { - if (!fs.existsSync(filepath)) return []; - const value = JSON.parse(fs.readFileSync(filepath, "utf-8")) as unknown; - if (!Array.isArray(value)) return []; - return value.filter(isRegistryEntry); + return readEntityRegistryWithWarnings(filepath).entries; +} + +export function readEntityRegistryWithWarnings(filepath: string): { + entries: ExternalEntityRegistryEntry[]; + warnings: EntityRegistryLookupResult["warnings"]; +} { + if (!fs.existsSync(filepath)) return { entries: [], warnings: [] }; + let value: unknown; + try { + value = JSON.parse(fs.readFileSync(filepath, "utf-8")) as unknown; + } catch { + return { + entries: [], + warnings: [{ reason_code: "registry_invalid", reason_detail: "entity registry is not valid JSON" }], + }; + } + if (!Array.isArray(value)) { + return { + entries: [], + warnings: [{ reason_code: "registry_invalid", reason_detail: "entity registry must be an array" }], + }; + } + + const entries: ExternalEntityRegistryEntry[] = []; + const warnings: EntityRegistryLookupResult["warnings"] = []; + for (const entry of value) { + if (isRegistryEntry(entry)) { + entries.push(entry); + } else { + warnings.push({ reason_code: "registry_invalid", reason_detail: "entity registry entry is invalid or unsafe" }); + } + } + return { entries, warnings }; } export function applyEntityRegistry( actor: { actor_type?: ExternalActorType; provider_actor_id?: string; + identity_hash?: string; + display_name?: string; + handle?: string; + platform_profile_url?: string; provider_tier_hint?: ExternalSignalActor["provider_tier_hint"]; registry_entity_id?: string; }, registry: ExternalEntityRegistryEntry[], + context?: EntityRegistryLookupContext, ): EntityRegistryLookupResult { if (registry.length === 0) { return { @@ -48,53 +99,131 @@ export function applyEntityRegistry( }; } - const matched = findRegistryMatch(actor, registry); - if (!matched) { + const match = findRegistryMatch(actor, registry, context); + if (!match) { return { actor: anonymousActor(actor, actor.provider_tier_hint ? "provider_hint" : "none"), warnings: [{ reason_code: "registry_miss", reason_detail: "actor did not match entity registry" }], }; } + if (match.roles.length === 0) { + return { + actor: anonymousActor(actor, actor.provider_tier_hint ? "provider_hint" : "none"), + warnings: [{ reason_code: "named_actor_role_unresolved", reason_detail: "registry actor matched but source role could not be resolved" }], + }; + } + return { actor: { - actor_type: matched.actor_type, - effective_tier: matched.tier, + actor_type: match.entry.actor_type, + effective_tier: match.entry.tier, tier_basis: "registry", provider_actor_id: actor.provider_actor_id, + identity_hash: actor.identity_hash, + display_name: actor.display_name, + handle: actor.handle, + platform_profile_url: actor.platform_profile_url, provider_tier_hint: actor.provider_tier_hint, - registry_entity_id: matched.entity_id, - registry_display_name: matched.display_name, - registry_tier: matched.tier, + registry_entity_id: match.entry.entity_id, + registry_display_name: match.entry.display_name, + registry_tier: match.entry.tier, + source_roles: match.roles, }, warnings: [], }; } +interface RegistryMatch { + entry: ExternalEntityRegistryEntry; + roles: ExternalNamedActorSourceRole[]; +} + function findRegistryMatch( actor: { provider_actor_id?: string; + handle?: string; + platform_profile_url?: string; registry_entity_id?: string; }, registry: ExternalEntityRegistryEntry[], -): ExternalEntityRegistryEntry | undefined { + context?: EntityRegistryLookupContext, +): RegistryMatch | undefined { + const matches: Array<{ entry: ExternalEntityRegistryEntry; roles: ExternalNamedActorSourceRole[] }> = []; + if (actor.registry_entity_id) { const byEntityId = registry.find((entry) => entry.entity_id === actor.registry_entity_id); - if (byEntityId) return byEntityId; + if (byEntityId) matches.push({ entry: byEntityId, roles: rolesForRegistryEntityContext(byEntityId, context) }); } - if (!actor.provider_actor_id) return undefined; - const providerActorId = actor.provider_actor_id.trim(); - if (providerActorId.length === 0) return undefined; + const handleInput = actor.handle ?? handleLikeProviderActorId(actor.provider_actor_id); + const profileInput = actor.platform_profile_url ?? profileLikeProviderActorId(actor.provider_actor_id); + + if (handleInput) { + const providerHandle = normalizeHandle(handleInput); + const byHandle = registry.find((entry) => entry.handles.map(normalizeHandle).includes(providerHandle)); + if (byHandle) matches.push({ entry: byHandle, roles: socialRolesForContext(context) }); + } - const providerHandle = normalizeHandle(providerActorId); - const providerUrl = normalizeUrl(providerActorId); - return registry.find((entry) => { - if (providerActorId === entry.entity_id) return true; - if (entry.handles.map(normalizeHandle).includes(providerHandle)) return true; - if (entry.profile_urls.map(normalizeUrl).includes(providerUrl)) return true; - return false; - }); + if (profileInput) { + const providerUrl = normalizeUrl(profileInput); + const byProfile = registry.find((entry) => entry.profile_urls.map(normalizeUrl).includes(providerUrl)); + if (byProfile) matches.push({ entry: byProfile, roles: socialRolesForContext(context) }); + } + + for (const entry of registry) { + const officialRoles = officialRolesForContext(entry, context); + if (officialRoles.length > 0) matches.push({ entry, roles: officialRoles }); + } + + const byEntity = new Map(); + for (const match of matches) { + const existing = byEntity.get(match.entry.entity_id); + const roles = uniqueRoles([...(existing?.roles ?? []), ...match.roles]); + byEntity.set(match.entry.entity_id, { entry: match.entry, roles }); + } + + return byEntity.values().next().value as RegistryMatch | undefined; +} + +function rolesForRegistryEntityContext(entry: ExternalEntityRegistryEntry, context?: EntityRegistryLookupContext): ExternalNamedActorSourceRole[] { + const registryEntityOfficialRole = + context && (context.platform === "official_web" || context.platform === "official_blog" || context.raw_event_kind === "official_release" || context.raw_event_kind === "blog_post") + ? (["official_publisher"] as ExternalNamedActorSourceRole[]) + : []; + return uniqueRoles([...socialRolesForContext(context), ...officialRolesForContext(entry, context), ...registryEntityOfficialRole]); +} + +function socialRolesForContext(context?: EntityRegistryLookupContext): ExternalNamedActorSourceRole[] { + if (!context) return []; + const socialPlatform = context.platform === "x_twitter" || context.platform === "reddit" || context.platform === "hacker_news"; + const socialKind = + context.raw_event_kind === "mention" || + context.raw_event_kind === "discussion" || + context.raw_event_kind === "question" || + context.raw_event_kind === "showcase"; + return socialPlatform && socialKind ? ["social_discussant"] : []; +} + +function officialRolesForContext(entry: ExternalEntityRegistryEntry, context?: EntityRegistryLookupContext): ExternalNamedActorSourceRole[] { + if (!context) return []; + const roles: ExternalNamedActorSourceRole[] = []; + const officialPlatform = context.platform === "official_web" || context.platform === "official_blog"; + const officialKind = context.raw_event_kind === "official_release" || context.raw_event_kind === "blog_post"; + const sourceDomain = context.url ? normalizeDomain(context.url) : undefined; + + if ((officialPlatform || officialKind) && sourceDomain && (entry.domains ?? []).map(normalizeDomainValue).includes(sourceDomain)) { + roles.push("official_publisher"); + } + + const githubOwners = [context.url, context.target_url, context.target_repo_url] + .map(githubOwnerFromUrl) + .filter((owner): owner is string => Boolean(owner)); + if (githubOwners.some((owner) => (entry.github_owners ?? []).map(normalizeGithubOwner).includes(owner))) { + roles.push("official_owner"); + } + + return uniqueRoles(roles); } function normalizeHandle(value: string): string { @@ -121,10 +250,60 @@ function normalizeUrl(value: string): string { } } +function normalizeDomain(value: string): string | undefined { + try { + const url = new URL(value.trim().toLowerCase()); + return normalizeDomainValue(url.hostname); + } catch { + return undefined; + } +} + +function normalizeDomainValue(value: string): string { + return value.trim().toLowerCase().replace(/^www\./, ""); +} + +function githubOwnerFromUrl(value: string | undefined): string | undefined { + if (!value) return undefined; + try { + const url = new URL(value.trim().toLowerCase()); + if (url.hostname !== "github.com" && url.hostname !== "www.github.com") return undefined; + return normalizeGithubOwner(url.pathname.split("/").filter(Boolean)[0] ?? ""); + } catch { + return undefined; + } +} + +function normalizeGithubOwner(value: string): string { + return value.trim().toLowerCase(); +} + +function handleLikeProviderActorId(value: string | undefined): string | undefined { + if (!value) return undefined; + const trimmed = value.trim(); + if (trimmed.startsWith("@")) return trimmed; + return undefined; +} + +function profileLikeProviderActorId(value: string | undefined): string | undefined { + if (!value) return undefined; + const trimmed = value.trim(); + return /^https?:\/\//i.test(trimmed) ? trimmed : undefined; +} + +function uniqueRoles(values: ExternalNamedActorSourceRole[]): ExternalNamedActorSourceRole[] { + const order: ExternalNamedActorSourceRole[] = ["social_discussant", "official_publisher", "official_owner"]; + return order.filter((role) => values.includes(role)); +} + function anonymousActor( actor: { actor_type?: ExternalActorType; provider_actor_id?: string; + identity_hash?: string; + display_name?: string; + handle?: string; + platform_profile_url?: string; provider_tier_hint?: ExternalSignalActor["provider_tier_hint"]; }, tierBasis: ExternalTierBasis, @@ -134,6 +313,10 @@ function anonymousActor( effective_tier: actor.actor_type === "unknown" || !actor.actor_type ? "unknown" : "ordinary", tier_basis: tierBasis, provider_actor_id: actor.provider_actor_id, + identity_hash: actor.identity_hash, + display_name: actor.display_name, + handle: actor.handle, + platform_profile_url: actor.platform_profile_url, provider_tier_hint: actor.provider_tier_hint, }; } @@ -146,8 +329,21 @@ function isRegistryEntry(value: unknown): value is ExternalEntityRegistryEntry { typeof entry.display_name === "string" && (entry.actor_type === "institution" || entry.actor_type === "team" || entry.actor_type === "person") && (entry.tier === "core" || entry.tier === "proven" || entry.tier === "watch") && + (entry.aliases === undefined || isStringArray(entry.aliases)) && Array.isArray(entry.handles) && Array.isArray(entry.profile_urls) && + (entry.domains === undefined || isStringArray(entry.domains)) && + (entry.github_owners === undefined || isStringArray(entry.github_owners)) && + !containsUnsafeRegistryFields(entry) && typeof entry.updated_at === "string" ); } + +function isStringArray(value: unknown): value is string[] { + return Array.isArray(value) && value.every((item) => typeof item === "string"); +} + +function containsUnsafeRegistryFields(entry: Partial & Record): boolean { + const serialized = JSON.stringify(entry).toLowerCase(); + return /\b(cookie|session|token|password|oauth|private_diagnostics|provider_diagnostics)\b/.test(serialized); +} diff --git a/src/externalDiscovery/types.ts b/src/externalDiscovery/types.ts index 4564606..0f40db5 100644 --- a/src/externalDiscovery/types.ts +++ b/src/externalDiscovery/types.ts @@ -9,21 +9,28 @@ export type ExternalRegistryTier = "core" | "proven" | "watch"; export type ExternalProviderTierHint = "core" | "proven" | "watch" | "ordinary" | "unknown"; export type ExternalEvidenceScope = "project" | "direction"; export type ExternalTierBasis = "registry" | "provider_hint" | "none"; +export type ExternalNamedActorSourceRole = "social_discussant" | "official_publisher" | "official_owner"; export const EXTERNAL_PLATFORMS = ["x_twitter", "reddit", "hacker_news", "official_web", "official_blog"] as const; export const EXTERNAL_TARGET_TYPES = ["project", "paper", "product", "topic"] as const; export const EXTERNAL_ACTOR_TYPES = ["institution", "team", "person", "community", "unknown"] as const; export const EXTERNAL_REGISTRY_TIERS = ["core", "proven", "watch"] as const; +export const EXTERNAL_NAMED_ACTOR_SOURCE_ROLES = ["social_discussant", "official_publisher", "official_owner"] as const; export interface ExternalSignalActor { actor_type: ExternalActorType; effective_tier: ExternalActorTier; tier_basis: ExternalTierBasis; provider_actor_id?: string; + identity_hash?: string; + display_name?: string; + handle?: string; + platform_profile_url?: string; provider_tier_hint?: ExternalProviderTierHint; registry_entity_id?: string; registry_display_name?: string; registry_tier?: ExternalRegistryTier; + source_roles?: ExternalNamedActorSourceRole[]; } export interface ExternalSignalEvent { @@ -39,6 +46,8 @@ export interface ExternalSignalEvent { source_published_at?: string; ingested_at?: string; url?: string; + target_url?: string; + target_repo_url?: string; raw_ref?: string; } @@ -47,6 +56,7 @@ export interface ExternalNamedRegistryActor { display_name: string; actor_type: "institution" | "team" | "person"; registry_tier: ExternalRegistryTier; + source_roles: ExternalNamedActorSourceRole[]; event_count: number; platforms: ExternalPlatform[]; first_seen_at: string; From 6b0d717b4342b795578e304be3178a31633b2727 Mon Sep 17 00:00:00 2001 From: Aspetta <892024285@qq.com> Date: Wed, 1 Jul 2026 12:35:03 +0800 Subject: [PATCH 3/8] fix(agentreach): filter stale social evidence --- .../externalDiscoveryAggregate.test.ts | 43 ++++++++++ .../externalDiscoveryRedaction.test.ts | 10 +++ src/externalDiscovery/aggregate.ts | 78 ++++++++++++++++++- src/externalDiscovery/redaction.ts | 6 +- 4 files changed, 132 insertions(+), 5 deletions(-) diff --git a/src/__tests__/externalDiscoveryAggregate.test.ts b/src/__tests__/externalDiscoveryAggregate.test.ts index 18fa94a..15b2075 100644 --- a/src/__tests__/externalDiscoveryAggregate.test.ts +++ b/src/__tests__/externalDiscoveryAggregate.test.ts @@ -146,4 +146,47 @@ describe("external discovery aggregate", () => { expect(aggregate.project_evidence[0]?.top_tier_actor_count).toBe(0); expect(aggregate.audit.warnings[0]?.reason_code).toBe("named_actor_missing_source_roles"); }); + + it("filters stale X and Reddit results out of the daily external aggregate", () => { + const aggregate = buildDailyExternalAggregate({ + date: "2026-06-30", + generated_at: "2026-06-30T01:00:00.000Z", + events: [ + event({ + event_id: "recent-x", + platform: "x_twitter", + source_published_at: "2026-06-20T00:00:00.000Z", + }), + event({ + event_id: "old-reddit", + platform: "reddit", + source_published_at: "2026-05-01T00:00:00.000Z", + actor: { + actor_type: "community", + effective_tier: "ordinary", + tier_basis: "none", + identity_hash: "reddit-user-1", + }, + }), + event({ + event_id: "old-official-blog", + platform: "official_blog", + raw_event_kind: "blog_post", + source_published_at: "2026-01-01T00:00:00.000Z", + }), + ], + }); + + expect(aggregate.accepted_event_count).toBe(2); + expect(aggregate.rejected_event_count).toBe(1); + expect(aggregate.platform_counts).toMatchObject({ x_twitter: 1, official_blog: 1 }); + expect(aggregate.audit.rejected_events).toEqual([ + { + event_id: "old-reddit", + reason_code: "event_outside_recent_social_window", + reason_detail: "reddit event is older than 30 days for aggregate date 2026-06-30", + }, + ]); + expect(aggregate.project_evidence[0]?.event_ids).toEqual(["old-official-blog", "recent-x"]); + }); }); diff --git a/src/__tests__/externalDiscoveryRedaction.test.ts b/src/__tests__/externalDiscoveryRedaction.test.ts index e4527ca..2f7cbef 100644 --- a/src/__tests__/externalDiscoveryRedaction.test.ts +++ b/src/__tests__/externalDiscoveryRedaction.test.ts @@ -36,6 +36,16 @@ describe("external discovery redaction", () => { expect(containsForbiddenPublicArtifactText({ handle: "@raw" })).toBe(true); expect(containsForbiddenPublicArtifactText({ provider_diagnostics: { trace: "private" } })).toBe(true); expect(containsForbiddenPublicArtifactText({ status_reason: "oauth token leaked" })).toBe(true); + expect(containsForbiddenPublicArtifactText({ status_reason: "api token: leaked" })).toBe(true); + expect(containsForbiddenPublicArtifactText({ status_reason: "token=leaked" })).toBe(true); + }); + + it("allows public project identifiers that contain token as a domain term", () => { + expect( + containsForbiddenPublicArtifactText({ + target_key: "https://github.com/swaranshu-borgaonkar/token-budget-contracts", + }), + ).toBe(false); }); it("rejects named_registry_actors that carry raw identity fields", () => { diff --git a/src/externalDiscovery/aggregate.ts b/src/externalDiscovery/aggregate.ts index f92b33a..1707cb0 100644 --- a/src/externalDiscovery/aggregate.ts +++ b/src/externalDiscovery/aggregate.ts @@ -28,9 +28,14 @@ const registryTierRank: Record(["x_twitter", "reddit"]); + export function buildDailyExternalAggregate(input: BuildDailyExternalAggregateInput): DailyExternalAggregate { - const events = input.events ?? input.provider_result?.events ?? []; - const rejectedEvents = input.provider_result?.rejected_events ?? []; + const rawEvents = input.events ?? input.provider_result?.events ?? []; + const recentEvents = filterRecentSocialEvents(rawEvents, input.date); + const events = recentEvents.events; + const rejectedEvents = [...(input.provider_result?.rejected_events ?? []), ...recentEvents.rejected_events]; const aggregate: DailyExternalAggregate = { schema_version: "external-discovery.aggregate.v1", date: input.date, @@ -54,7 +59,7 @@ export function buildDailyExternalAggregate(input: BuildDailyExternalAggregateIn observation_candidates: input.observation_candidates ?? [], audit: { rejected_events: rejectedEvents, - warnings: [...(input.provider_result?.warnings ?? []), ...namedActorRoleWarnings(events)], + warnings: [...(input.provider_result?.warnings ?? []), ...recentEvents.warnings, ...namedActorRoleWarnings(events)], }, }; @@ -66,6 +71,65 @@ export function buildDailyExternalAggregate(input: BuildDailyExternalAggregateIn return aggregate; } +function filterRecentSocialEvents(events: ExternalSignalEvent[], date: string): { + events: ExternalSignalEvent[]; + rejected_events: Array<{ event_id?: string; reason_code: string; reason_detail: string }>; + warnings: Array<{ reason_code: string; reason_detail: string }>; +} { + const cutoff = socialRecencyCutoff(date); + if (!cutoff) return { events, rejected_events: [], warnings: [] }; + + const accepted: ExternalSignalEvent[] = []; + const rejected: Array<{ event_id?: string; reason_code: string; reason_detail: string }> = []; + const invalidTimestampWarnings: Array<{ reason_code: string; reason_detail: string }> = []; + + for (const event of events) { + if (!SOCIAL_RECENCY_PLATFORMS.has(event.platform)) { + accepted.push(event); + continue; + } + + const eventTime = socialEventPublishedTime(event); + if (!eventTime) { + accepted.push(event); + invalidTimestampWarnings.push({ + reason_code: "social_event_recency_timestamp_invalid", + reason_detail: `${event.event_id} did not provide a parseable source_published_at or observed_at; kept for review`, + }); + continue; + } + + if (eventTime < cutoff) { + rejected.push({ + event_id: event.event_id, + reason_code: "event_outside_recent_social_window", + reason_detail: `${event.platform} event is older than ${SOCIAL_RECENCY_WINDOW_DAYS} days for aggregate date ${date}`, + }); + continue; + } + + accepted.push(event); + } + + return { events: accepted, rejected_events: rejected, warnings: uniqueWarnings(invalidTimestampWarnings) }; +} + +function socialRecencyCutoff(date: string): number | null { + const anchor = Date.parse(`${date}T00:00:00.000Z`); + if (!Number.isFinite(anchor)) return null; + return anchor - SOCIAL_RECENCY_WINDOW_DAYS * 24 * 60 * 60 * 1000; +} + +function socialEventPublishedTime(event: ExternalSignalEvent): number | null { + const candidates = [event.source_published_at, event.observed_at]; + for (const candidate of candidates) { + if (!candidate) continue; + const parsed = Date.parse(candidate); + if (Number.isFinite(parsed)) return parsed; + } + return null; +} + function buildEvidence(events: ExternalSignalEvent[]): ExternalEvidence[] { const grouped = new Map(); for (const event of events) { @@ -217,6 +281,14 @@ function unique(values: T[]): T[] { return Array.from(new Set(values)); } +function uniqueWarnings(warnings: T[]): T[] { + const byKey = new Map(); + for (const warning of warnings) { + byKey.set(`${warning.reason_code}:${warning.reason_detail}`, warning); + } + return [...byKey.values()]; +} + function uniqueRoles(values: ExternalNamedActorSourceRole[]): ExternalNamedActorSourceRole[] { const order: ExternalNamedActorSourceRole[] = ["social_discussant", "official_publisher", "official_owner"]; return order.filter((role) => values.includes(role)); diff --git a/src/externalDiscovery/redaction.ts b/src/externalDiscovery/redaction.ts index af5ecec..934c102 100644 --- a/src/externalDiscovery/redaction.ts +++ b/src/externalDiscovery/redaction.ts @@ -22,7 +22,9 @@ const forbiddenKeys = new Set([ "provider_diagnostics", ]); -const secretPattern = /\b(cookie|session|token|password|oauth)\b/i; +const secretPattern = /\b(cookie|session|password|oauth)\b/i; +const tokenSecretPattern = + /\b(?:access|api|bearer|oauth|refresh)\s+token\b|\btoken\s+(?:credential|leak(?:ed)?|secret|value)\b|\btoken\s*[:=]/i; const profileUrlPattern = /^https?:\/\/(?:www\.)?(?:twitter\.com|x\.com|reddit\.com|news\.ycombinator\.com)\/[^\s/]+/i; export function stableSourceInputHash(input: string | Buffer): string { @@ -62,7 +64,7 @@ function collectRedactionReasonCodes(value: unknown): string[] { reasonCodes.push(`forbidden_key:${key}`); } if (typeof currentValue === "string") { - if (secretPattern.test(currentValue)) reasonCodes.push("forbidden_secret_text"); + if (secretPattern.test(currentValue) || tokenSecretPattern.test(currentValue)) reasonCodes.push("forbidden_secret_text"); if (profileUrlPattern.test(currentValue)) reasonCodes.push("forbidden_profile_url_text"); } }); From c9728187b864e2adfafb08bd7054be07e8032b6d Mon Sep 17 00:00:00 2001 From: Aspetta <892024285@qq.com> Date: Thu, 2 Jul 2026 00:47:16 +0800 Subject: [PATCH 4/8] Add AgentReach candidate explanations backend --- .gitignore | 1 + data/README.md | 3 + ...andidate-explanation-enhancement-design.md | 578 ++++++++++++++ ...idate-explanation-enhancement.exec-plan.md | 425 +++++++++++ ...DiscoveryCandidateExplanationInput.test.ts | 104 +++ ...overyCandidateExplanationRedaction.test.ts | 142 ++++ ...rnalDiscoveryCandidateExplanations.test.ts | 123 +++ .../externalDiscoveryDailyIntegration.test.ts | 122 +++ .../externalDiscoveryStructure.test.ts | 2 + .../externalDiscoveryTypeContract.test.ts | 12 +- .../externalDiscoveryVerification.test.ts | 72 +- src/__tests__/runSummaryObserver.test.ts | 40 +- src/action/dailyVerification.ts | 116 ++- src/action/runSummary.ts | 83 +- src/cli.ts | 77 +- src/externalDiscovery/agentReachProvider.ts | 61 ++ src/externalDiscovery/dailyIntegration.ts | 178 +++++ src/externalDiscovery/explanationPrompts.ts | 36 + src/externalDiscovery/explanationRedaction.ts | 121 +++ src/externalDiscovery/explanations.ts | 722 ++++++++++++++++++ src/externalDiscovery/paths.ts | 8 + src/externalDiscovery/types.ts | 10 + src/types.ts | 48 +- src/visualConsole/readLayer.ts | 6 + 24 files changed, 3016 insertions(+), 74 deletions(-) create mode 100644 docs/specs/design-docs/agentreach-candidate-explanation-enhancement-design.md create mode 100644 docs/specs/exec-plans/agentreach-candidate-explanation-enhancement.exec-plan.md create mode 100644 src/__tests__/externalDiscoveryCandidateExplanationInput.test.ts create mode 100644 src/__tests__/externalDiscoveryCandidateExplanationRedaction.test.ts create mode 100644 src/__tests__/externalDiscoveryCandidateExplanations.test.ts create mode 100644 src/__tests__/externalDiscoveryDailyIntegration.test.ts create mode 100644 src/externalDiscovery/dailyIntegration.ts create mode 100644 src/externalDiscovery/explanationPrompts.ts create mode 100644 src/externalDiscovery/explanationRedaction.ts create mode 100644 src/externalDiscovery/explanations.ts diff --git a/.gitignore b/.gitignore index 92e8b69..c20e561 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,4 @@ data/raw/external-discovery/** !data/raw/external-discovery/ !data/raw/external-discovery/fixtures/ !data/raw/external-discovery/fixtures/** +data/external-discovery/*.candidate-explanations.json diff --git a/data/README.md b/data/README.md index f9de376..8c5b5d3 100644 --- a/data/README.md +++ b/data/README.md @@ -18,6 +18,7 @@ These files are part of the project deliverable. They let readers inspect histor - `data/upstream/` - `data/raw/external-discovery/` +- `data/external-discovery/*.candidate-explanations.json` `data/upstream/` is reserved for optional local scratch checkouts or caches, such as an explicit self-hosted `agents-radar` mirror. It is ignored by Git and is not part of the public artifact history. @@ -25,6 +26,8 @@ These files are part of the project deliverable. They let readers inspect histor Public external discovery history belongs in `data/external-discovery/*.aggregate.json`. These aggregates must be public-safe: no raw social text, no profile URLs, no raw handles, no cookie/session/token/OAuth material, and no public `*.events.jsonl` default artifact. +`data/external-discovery/*.candidate-explanations.json` is a cached display-only AgentReach explanation artifact. Local real-run files are ignored by default; only sanitized fixtures used by tests should be committed. + ## Automation behavior The daily and weekly GitHub Actions workflows update the tracked public artifacts and commit them back into this repository. This is intentional: the repo is both code and a public historical data log. diff --git a/docs/specs/design-docs/agentreach-candidate-explanation-enhancement-design.md b/docs/specs/design-docs/agentreach-candidate-explanation-enhancement-design.md new file mode 100644 index 0000000..6e28615 --- /dev/null +++ b/docs/specs/design-docs/agentreach-candidate-explanation-enhancement-design.md @@ -0,0 +1,578 @@ +# AgentReach 候选解释增强层设计 + +## 文档状态 + +- 状态：`Approved for ExecPlan` +- 存放位置：`docs/specs/design-docs/` +- 提交策略：正式设计文档，可随对应实现或 exec-plan 一并提交 +- 对应需求：`docs/specs/product-specs/外部发现与补证信号层需求分析.md` +- 设计范围：AgentReach 外部发现候选的“它是什么 / 为什么值得看”后端解释增强 +- 非目标：不做外部讨论趋势，不改 Agent-Reach 上游，不改主评分，不让前端直接调用 DeepSeek + +## 1. Summary + +当前 AgentReach 后端链路已经能产出 `DailyExternalAggregate`，包含 `observation_candidates`、`project_evidence`、`direction_evidence`、`platform_counts`、`audit`、`named_registry_actors` 等结构化字段。前端已经可以展示外部候选与证据，但用户仍难以从候选名称和平台标签判断“这个项目到底是什么”“为什么值得看”。 + +本设计新增一层后端 display-oriented 解释增强能力：在不修改外部发现核心 schema、不影响 primary score 的前提下，基于已有 aggregate 和 evidence 字段，调用现有 LLM provider 链路生成可缓存、可降级、可审计的候选解释摘要。 + +目标输出是给 UI 消费的解释字段，而不是新的事实判断： + +- `what_it_is_cn`：它是什么，用普通中文解释项目或方向线索。 +- `why_watch_cn`：为什么值得看，解释进入外部发现页的原因。 +- `summary_confidence`：摘要可信度，只表达输入信息是否足够。 +- `summary_source`：摘要来源，区分 `llm`、`rules_fallback`、`existing_project_brief`。 +- `summary_warnings`：摘要生成中的限制，例如缺少 repo description、方向未绑定项目。 + +LLM 只负责把已有结构化证据转成人话，不得凭空补充功能、机构、使用场景或趋势结论。 + +## 2. Goals + +1. 让 `/agentreach` 页面候选卡和详情页能稳定展示“它是什么”和“为什么值得看”。 +2. 对已绑定 repo / product / paper 的候选，尽量生成具体、可读的项目简介。 +3. 对未绑定明确对象的方向候选，明确写成“方向线索”，不得伪装成 GitHub 项目。 +4. 复用现有 LLM provider、DeepSeek provider、`callStructuredEnhancement`、JSON 修复和脱敏机制。 +5. LLM 失败、超时、配置缺失或输出不合格时，必须回退到当前规则解释，不影响 daily 或 aggregate 产出。 +6. 输出字段只作为 display-only enrichment，不参与主榜分数、Trendshift / GitHub 主链路判断。 + +## 3. Non-goals + +1. 不做真正多日“外部讨论趋势”总结。单日 aggregate 只能生成今日候选解释。 +2. 不扩展 Agent-Reach 上游搜索平台，不新增 X / Reddit / HN API 调用。 +3. 不把 LLM 摘要写入 `ExternalSignalEvent` 原始事件事实。 +4. 不把外部热度提升为主榜结论，不修改 `ScoreBreakdown` 或 primary score。 +5. 不在浏览器、React 组件或前端 API 中暴露 `DEEPSEEK_API_KEY`。 +6. 不承诺英文摘要第一版完整生成。英文 UI 可先使用英文规则回退，中文摘要只在中文页展示。 + +## 4. Existing Code References + +### AgentReach 外部发现后端 + +- `src/externalDiscovery/types.ts` + - 定义 `ExternalSignalEvent`、`ExternalEvidence`、`ObservationCandidate`、`DailyExternalAggregate`。 +- `src/externalDiscovery/aggregate.ts` + - 将事件聚合为 `project_evidence`、`direction_evidence`、`platform_counts`、`named_registry_actors`。 +- `src/externalDiscovery/agentReachProvider.ts` + - 读取 AgentReach 本地 artifact 并转为 canonical event。 +- `src/externalDiscovery/redaction.ts` + - 校验 public-safe aggregate，避免 raw text、profile URL 或敏感字段泄露。 +- `src/externalDiscovery/paths.ts` + - 当前已有 `externalAggregatePath(date)` 和 `externalAggregateLatestPath()`。 + +### 现有 LLM 增强能力 + +- `src/llm.ts` + - 统一 LLM 调用、并发、重试、prompt 脱敏。 +- `src/providers/deepseek.ts` + - DeepSeek provider。 +- `src/action/enhancementLlm.ts` + - `callStructuredEnhancement` 和 JSON 修复重试。 +- `src/action/projectLibraryEnhancement.ts` + - 已有 `project_brief_cn` 生成模式，可复用其“项目简介必须具体”的校验思路。 +- `src/action/dailyReport.ts` + - 已有 `project_brief_cn` / `why_today_cn` 的 prompt 约束和 fallback 经验。 + +### 前端消费位置 + +- UI worktree 中的 `/agentreach` 页面应只读取后端产物字段。 +- 后端输出建议通过 view-model adapter 暴露给 `AgentReachViewModel`，不要让 React 自己拼 LLM prompt。 + +## 5. Data Contract / Field Mapping + +### 5.1 新增展示增强模型 + +建议新增 display-only artifact，而不是直接扩展核心 `DailyExternalAggregate` 第一层字段。 + +```ts +interface ExternalCandidateExplanationArtifact { + schema_version: "external-discovery.candidate-explanations.v1"; + date: string; + generated_at: string; + provider: "deepseek" | "openai" | "anthropic" | "openrouter" | "minimax" | "github-copilot" | "rules"; + explanation_policy_version: "candidate-explanations.v1"; + aggregate_source_input_hash: string; + aggregate_generated_at: string; + input_context_hash: string; + public_safe: true; + redaction_policy_version: string; + contains_raw_text: false; + contains_profile_urls: false; + status: "ok" | "partial" | "skipped" | "failed"; + status_reason?: string; + explanations: ExternalCandidateExplanation[]; + audit: { + eligible_count: number; + attempted_count: number; + accepted_count: number; + enhanced_count: number; + rejected_count: number; + fallback_count: number; + warnings: Array<{ reason_code: string; reason_detail: string }>; + }; +} +``` + +```ts +interface ExternalCandidateExplanation { + candidate_key: string; + candidate_kind: "project" | "paper" | "product" | "direction"; + target_key: string; + explanation_scope: "bound_object" | "direction_signal" | "external_evidence_boost"; + what_it_is_cn: string; + why_watch_cn: string; + summary_confidence: "high" | "medium" | "low"; + summary_source: "llm" | "rules_fallback" | "existing_project_brief"; + evidence_ids: string[]; + platforms: ExternalPlatform[]; + caveats: string[]; + generated_at: string; +} +``` + +### 5.2 字段语义冻结 + +| 字段 | 生产者 | 消费者 | 语义 | 缺失 / 降级行为 | +| --- | --- | --- | --- | --- | +| `explanation_policy_version` | explanation generator | verify / replay | 摘要生成策略版本，V1 固定为 `candidate-explanations.v1` | 缺失时 verify fail | +| `aggregate_source_input_hash` | aggregate builder | explanation generator / verify | 关联被解释的 `DailyExternalAggregate.source_input_hash` | 缺失时 artifact 不可用于 UI merge | +| `input_context_hash` | explanation input builder | cache / replay / verify | 对排序后的 `CandidateExplanationInput[]` 做稳定 hash，用于判断摘要是否对应当前输入包 | 不匹配时 UI 不得使用该 explanation artifact | +| `public_safe` / `contains_raw_text` / `contains_profile_urls` | public-safe validator | deploy / verify / UI | 与 aggregate 保持同类公开安全标记，必须为 `true/false/false` | 缺失或不匹配时不得提交或供 UI 读取 | +| `candidate_key` | explanation input builder | explanation merge / UI view-model | 稳定候选键，格式固定为 `${candidate_kind}:${target_key}`；同一 daily aggregate 内必须唯一 | 无法生成时不得进入 LLM batch，记录 `candidate_key_missing` 并使用规则 fallback | +| `explanation_scope` | explanation input builder | prompt / validator / UI | `bound_object` 表示已绑定 repo/product/paper；`direction_signal` 表示方向线索；`external_evidence_boost` 表示已有候选获得外部补证 | 不得由 LLM 自行决定；无法判断时使用 `direction_signal` 并降为 `low` | +| `status` | explanation generator | run-summary / UI / verify | `ok`=eligible candidates 全部有可展示 explanation，且 `llm/existing_project_brief` 增强覆盖率达到 70%；`partial`=部分 LLM 调用或校验失败，或增强覆盖率不足；`skipped`=LLM 未启用或无候选；`failed`=增强流程整体失败但 aggregate 仍可用 | UI 不得把 `partial/failed/skipped` 解释成“没有外部信号” | +| `status_reason` | explanation generator | run-summary / verify | 固定 reason code，见第 9 节 | 缺失时 verify warning | +| `summary_confidence` | validator | UI | 摘要可信度，只表示解释输入是否足够，不表示项目质量 | 缺失时视为 `low` | +| `summary_source` | explanation generator | UI / verify | `existing_project_brief`=复用已有项目简介；`llm`=LLM 输出通过校验；`rules_fallback`=规则回退 | 缺失时视为 `rules_fallback` | +| `what_it_is_cn` | existing brief / LLM / rules | UI | 回答“它是什么”，不得承诺主榜结论 | 缺失时使用规则 `intro_line` | +| `why_watch_cn` | LLM / rules | UI | 回答“为什么今天值得看”，只表达外部层原因 | 缺失时使用规则 `appearance_reason` | +| `caveats` | generator / validator | UI | 展示限制和待确认项，例如需要主源确认、方向未绑定 repo | 缺失时默认追加“外部层不能作为主榜结论” | + +`summary_confidence` 判定冻结为： + +- `high`：存在可复用的 `project_brief_cn` 或 repo description，且候选已绑定明确 repo/product/paper，并至少有 1 条外部证据。 +- `medium`：存在明确 `target_url/repo_url` 或 `target_display_name`，但缺少项目简介；可说明对象存在和外部讨论原因，但不能写具体用途。 +- `low`：只有方向 topic、长标题、平台计数或未绑定对象；只能写方向线索或保守候选说明。 + +`summary_source` 优先级冻结为： + +1. `existing_project_brief` 优先用于 `what_it_is_cn`，避免重复调用 LLM 生成已有项目简介。 +2. `llm` 可用于生成 `why_watch_cn`，也可在有足够输入时生成 `what_it_is_cn`。 +3. `rules_fallback` 是所有失败、缺失、低质量输出的最终兜底。 + +### 5.3 为什么建议旁路 artifact + +`DailyExternalAggregate` 当前是 public-safe 外部发现事实聚合，核心字段不包含自由文本摘要。LLM 摘要属于解释层，具有失败、重试、缓存、质量校验等独立生命周期。旁路 artifact 有三个好处： + +1. 不污染已冻结的 external discovery aggregate 事实契约。 +2. LLM 摘要可以独立重跑、独立缓存、独立回退。 +3. UI 可以优先读取解释 artifact，缺失时仍展示原 aggregate。 + +推荐路径： + +```text +data/external-discovery/YYYY-MM-DD.candidate-explanations.json +data/external-discovery/latest.candidate-explanations.json +``` + +第一版也可以在 view-model 构建时把 explanation artifact merge 到候选展示模型中，但不要反向覆盖 `DailyExternalAggregate`。 + +### 5.4 冻结的 CandidateExplanationInput + +为了避免 LLM 只能生成“被外部讨论提到的候选”这类空泛文案，第一版必须构建独立的 public-safe prompt input。该输入不是 canonical fact schema，而是解释增强层的只读输入包。 + +```ts +interface CandidateExplanationInput { + candidate_key: string; + candidate_kind: "project" | "paper" | "product" | "direction"; + target_key: string; + display_name: string; + explanation_scope: "bound_object" | "direction_signal" | "external_evidence_boost"; + target_url?: string; + repo_url?: string; + target_display_name?: string; + existing_project_brief_cn?: string; + repo_description?: string; + public_evidence_titles: string[]; + public_source_titles: string[]; + evidence_reason_facts: string[]; + evidence_ids: string[]; + platforms: ExternalPlatform[]; + mention_count: number; + distinct_actor_count: number; + top_tier_actor_count: number; + named_registry_actor_names: string[]; + can_enter_daily: boolean; + can_enter_weekly: boolean; + cannot_be_primary_conclusion: true; + input_confidence: "high" | "medium" | "low"; +} +``` + +字段来源冻结为： + +| 输入字段 | 来源 | 规则 | +| --- | --- | --- | +| `display_name` | `ObservationCandidate.target_key`、UI display adapter 或 sanitized provider target name | 必须存在；长标题可截断展示，但 prompt input 保留完整 public-safe 文本 | +| `target_url` / `repo_url` | canonical event / provider target URL / existing project index | 只允许 repo、paper、product 官方 URL；不得放 profile URL | +| `target_display_name` | sanitized provider target name 或 repo full name | 仅用于帮助 LLM 识别对象，不作为主源确认 | +| `existing_project_brief_cn` | project library / observer / daily report existing brief | 已知 repo 命中时必须优先使用 | +| `repo_description` | existing project index / GitHub 主链路已采集 description | 仅使用已在主链路或项目库存在的描述，不新增 live GitHub 请求 | +| `public_evidence_titles` | AgentReach provider item title 经 sanitizer 后产生 | 只允许标题级短文本；禁止 raw post body、评论全文、handle、profile URL | +| `public_source_titles` | official page/blog/HN/Reddit/X item title 经 sanitizer 后产生 | 最多 5 条，每条建议不超过 160 字符 | +| `evidence_reason_facts` | 规则生成 | 例如“HN 出现外部讨论”“具名讨论者 OpenAI 命中 registry”“跨平台出现” | +| `named_registry_actor_names` | `ExternalEvidence.named_registry_actors.display_name` | 只能使用 registry 命中的 public-safe display name | + +`public_evidence_titles` / `public_source_titles` 是解决“它是什么”的关键输入。实现阶段不得只把它们留作未来扩展；如果 provider raw title 暂时不可用，必须记录 `public_title_context_missing`，并将对应候选 `input_confidence` 降为 `medium` 或 `low`。 + +### 5.5 基础 aggregate 输入字段 + +每个候选解释输入应只包含 public-safe 字段： + +- `ObservationCandidate` + - `candidate_kind` + - `target_key` + - `qualification` + - `can_enter_daily` + - `can_enter_weekly` + - `cannot_be_primary_conclusion` +- 对应的 `ExternalEvidence` + - `scope` + - `target_key` + - `derived_signal_kinds` + - `platforms` + - `mention_count` + - `distinct_actor_count` + - `top_tier_actor_count` + - `named_registry_actors.display_name` + - `actor_types` + - `actor_tiers` + - `first_seen_at` + - `last_seen_at` +- 聚合上下文 + - `status` + - `status_reason` + - `platform_counts` + - `accepted_event_count` + - `rejected_event_count` + +如果 `CandidateExplanationInput` 不包含 `existing_project_brief_cn`、`repo_description`、`public_evidence_titles` 或 `public_source_titles` 中至少一类具体语义输入，则 LLM 不得编造具体功能，只能生成保守解释；该候选的 `summary_confidence` 不得高于 `medium`。 + +## 6. Explanation Semantics + +### 6.1 项目级候选 + +当候选满足以下条件之一时，可以按项目候选解释： + +- `candidate_kind` 是 `project`、`paper` 或 `product`。 +- 对应 evidence `scope=project`。 +- `target_key` 是 repo-like key 或存在可 public-safe 展示的 repo / object URL。 + +输出语义： + +- `what_it_is_cn` 应解释“这是一个什么对象”，例如项目、论文、产品、工具。 +- 如果输入缺少 description，不得写“它可以自动完成某某任务”这类无来源功能描述。 +- 可以写“目前只能确认它是一个被外部讨论提到的 GitHub repo 候选，具体功能仍需主源补充”。 + +### 6.2 方向级候选 + +当候选满足以下条件之一时，必须按方向线索解释： + +- `candidate_kind=direction` +- evidence `scope=direction` +- `qualification=direction_observation` +- 无明确 repo / paper / product 绑定 + +输出语义： + +- `what_it_is_cn` 必须包含“方向线索”或等价含义。 +- 不得把长标题或 topic hint 写成具体项目名。 +- `why_watch_cn` 应解释该方向在外部讨论中出现，适合进入观察，不是主榜结论。 + +### 6.3 外部补证候选 + +当 evidence 主要是对已有对象的 `derived_signal_kinds` 包含 `evidence`，或上层 view-model 标记为外部补证时： + +- `what_it_is_cn` 可以优先使用已有项目简介或绑定对象描述。 +- `why_watch_cn` 应突出“已有候选获得外部补证”，而不是“新项目发现”。 +- 仍必须带上“外部层仅作次级证据”的 caveat。 + +## 7. LLM Prompt Rules + +Prompt 必须固定以下约束： + +1. 只根据输入 JSON 写摘要，不得访问互联网，不得推测未提供的功能。 +2. 输出 exactly one JSON object，不使用 markdown。 +3. 对项目候选，优先说明“它是什么 + 用户通常拿它做什么”，但只有输入支持时才能写用途。 +4. 对方向候选，必须写成“方向线索”，不得包装成项目简介。 +5. `why_watch_cn` 必须解释外部层原因，例如跨平台、具名讨论者、关注度、可进入日报/周报。 +6. 不得出现“主榜结论”“高置信推荐”“已经形成趋势”等过度结论。 +7. 若证据不足，必须显式写“仍需 GitHub / Trendshift 主链路确认”。 + +建议 LLM 输出草案： + +```json +{ + "explanations": [ + { + "candidate_key": "project:example/repo", + "what_it_is_cn": "这是一个被外部讨论提到的 GitHub repo 候选，目前可以确认对象存在，但具体功能仍需主源补充。", + "why_watch_cn": "它今天在 HN 和 Reddit 出现外部讨论，并已发现可绑定的 repo 线索，适合作为外部层候选继续观察。", + "summary_confidence": "medium", + "caveats": ["外部层不能作为主榜结论", "仍需 GitHub / Trendshift 主链路确认"] + } + ] +} +``` + +## 8. Validation Rules + +LLM 输出必须经过校验，校验失败则丢弃并回退规则文案。 + +基础校验： + +- `candidate_key` 必须能匹配输入候选。 +- `what_it_is_cn` 和 `why_watch_cn` 必须是中文非空字符串。 +- 每段长度建议 30 到 140 个中文字符。 +- 不得包含 URL、markdown 链接、API key、profile URL。 +- 不得包含“主榜结论”“高置信推荐”“确定爆发”“已经成为趋势”等过度判断。 +- 方向候选的 `what_it_is_cn` 必须包含“方向”或“线索”等弱绑定语义。 +- 未提供功能描述时，不得出现明显编造的具体能力动词，例如“自动部署”“自动生成完整应用”等，除非输入 evidence 明确支持。 + +去重校验： + +- 不同候选不得复用完全相同的 `what_it_is_cn`。 +- 多个候选若输出高度相似，应优先保留 evidence 更充分的候选，其余回退规则文案。 + +## 8.1 最小增强义务 + +解释增强层的外部依赖是 optional，但当 LLM 已启用且存在候选时，增强结果不能被实现缩水成“全部规则 fallback 也算完成”。 + +V1 最小义务冻结为： + +1. 默认 eligible candidate 覆盖上限为 `top_n=30`，由候选排序后的前 30 个候选组成；少于 30 个时覆盖全部候选。 +2. eligible candidate 排序优先级：`external_evidence_boost` 高于 `bound_object` 高于 `direction_signal`；同类内按 evidence 数、平台数、`top_tier_actor_count`、`mention_count` 降序。 +3. 有 `existing_project_brief_cn` 的候选必须生成 explanation，不需要调用 LLM 也不能跳过。 +4. 有 `repo_url/target_url` 且有 `public_evidence_titles` 或 `repo_description` 的项目候选必须尝试 LLM 或复用现有简介。 +5. 方向候选至少覆盖 eligible direction candidates 的前 5 个；若不足 5 个则覆盖全部。 +6. eligible candidate 中增强 explanation 覆盖率低于 70% 时，artifact `status` 必须是 `partial`，不得标记为 `ok`。增强 explanation 只统计 `summary_source=llm` 或 `existing_project_brief`，不统计 `rules_fallback`。 +7. eligible candidate 中增强 explanation 覆盖率为 0 且候选数大于 0 时，artifact `status` 必须是 `failed` 或 `skipped`，不得标记为 `ok`。 + +`rules_fallback` 可以作为单候选兜底，但不得被用来规避 LLM 尝试义务。若因为输入不足未调用 LLM，必须在 audit 中记录 `input_context_insufficient`。 + +## 9. Fallback Strategy + +增强层必须可降级，但降级语义必须显式可见。 + +### LLM 未启用 + +- status：`skipped` +- 原因：`llm_disabled` +- UI 使用现有规则字段：项目候选、方向线索、外部补证、需主源确认。 + +### LLM 调用失败 + +- status：`partial` 或 `failed` +- 单候选失败不影响其他候选。 +- 失败候选使用 `rules_fallback`。 +- audit 记录 `summary_generation_failed`。 + +### LLM 输出不合格 + +- 丢弃该候选 LLM 摘要。 +- 使用规则 fallback。 +- audit 记录 `summary_validation_failed` 和原因码。 + +### 输入信息不足 + +- 不调用或调用后降级为低置信摘要。 +- `summary_confidence=low` +- `caveats` 写明“缺少项目功能描述”或“尚未绑定明确 repo”。 + +### status_reason 冻结集合 + +`status_reason` 必须使用以下 reason code 之一或组合；不得写自由文本作为唯一状态原因： + +- `llm_disabled` +- `no_candidates` +- `candidate_explanation_ok` +- `partial_llm_failure` +- `summary_generation_failed` +- `summary_validation_failed` +- `input_context_insufficient` +- `public_safe_validation_failed` +- `provider_unavailable` +- `unexpected_exception` + +单候选 warning reason code 固定为： + +- `candidate_key_missing` +- `candidate_not_eligible` +- `public_title_context_missing` +- `project_brief_reused` +- `llm_output_rejected` +- `rules_fallback_used` +- `direction_candidate_low_confidence` +- `primary_source_confirmation_required` + +## 10. Runtime Placement + +第一版推荐作为 daily external aggregate 后处理步骤： + +```text +AgentReach raw artifact + -> readAgentReachProviderArtifact + -> buildDailyExternalAggregate + -> assertPublicSafeAggregate + -> buildCandidateExplanationInputs + -> assertPublicSafeCandidateExplanationInputs + -> generateExternalCandidateExplanations + -> assertPublicSafeCandidateExplanations + -> write candidate-explanations artifact + -> visual console / UI read model merge +``` + +这样可以保证： + +- 只有通过 public-safe 校验的 aggregate 与 `CandidateExplanationInput` 会进入 LLM prompt。 +- `CandidateExplanationInput` 必须先通过 public-safe 校验，再进入 LLM prompt。 +- LLM 摘要失败不会破坏 external aggregate。 +- UI 可以在没有 explanation artifact 时正常运行。 + +## 11. UI Consumption Contract + +前端 `/agentreach` 页面不直接调用 DeepSeek，只消费后端产物。 + +UI 优先级： + +1. `ExternalCandidateExplanation.what_it_is_cn` +2. 已有项目库或 observer 的 `project_brief_cn` +3. 规则 fallback `intro_line` + +候选卡： + +- 标题下展示 `why_watch_cn` 的短版，或规则 `appearance_reason`。 +- chip 仍只展示状态和来源，不承载长解释。 + +详情页： + +- `它是什么` 展示 `what_it_is_cn`。 +- `为什么值得看` 展示 `why_watch_cn` 和 caveats。 +- 方向候选显示“方向线索 / 尚未绑定明确项目”。 + +英文页： + +- 第一版不直接展示中文 LLM 摘要。 +- 若没有英文摘要字段，英文页使用英文规则 fallback。 +- 后续如需英文摘要，应新增 `what_it_is_en` / `why_watch_en`，不要机器翻译中文字段后直接混用。 + +## 12. Security & Privacy + +1. `DEEPSEEK_API_KEY` 只存在服务端环境变量，不进入浏览器 bundle。 +2. Prompt 输入只来自 public-safe aggregate 与通过校验的 `CandidateExplanationInput`，不包含 raw text、profile URL、handle、private registry metadata。 +3. 调用 `callLlm` 时继续使用现有 prompt 脱敏机制。 +4. LLM 输出不得写回 raw input，不得覆盖 canonical event。 +5. explanation artifact 公开提交前必须通过 `assertPublicSafeCandidateExplanations`。 + +### 12.1 CandidateExplanationInput public-safe 校验 + +`assertPublicSafeCandidateExplanationInputs` 必须拒绝以下内容： + +- API key、token、cookie、bearer token、private key 等敏感字符串。 +- `profile_url`、社交平台个人主页 URL、带 handle 的 profile 链接。 +- raw post body、评论全文、长文本 thread、未截断网页正文。 +- 未经 registry 命中的 actor handle 或 display name。 +- markdown link、HTML anchor、未脱敏 URL 列表。 +- 单条 `public_evidence_titles` 或 `public_source_titles` 超过 160 字符。 + +允许进入 input 的 URL 仅限： + +- `repo_url` +- `target_url` +- paper / product / official page 的对象 URL + +这些 URL 只用于对象绑定，不得作为 source actor 或讨论者身份展示。 + +### 12.2 CandidateExplanation artifact public-safe 校验 + +`assertPublicSafeCandidateExplanations` 必须拒绝以下输出： + +- 包含 API key、token、cookie、profile URL、raw handle。 +- 包含 markdown 链接或 HTML 链接。 +- 包含“主榜结论”“高置信推荐”“确定爆发”“已经成为趋势”等过度结论。 +- 方向候选摘要未包含“方向 / 线索 / 尚未绑定明确项目”等弱绑定语义。 +- `what_it_is_cn` 或 `why_watch_cn` 与其他候选完全重复。 +- LLM 输出未经输入支持的具名机构、具体功能或使用场景。 + +reason code 固定为： + +- `sensitive_text_detected` +- `profile_url_detected` +- `raw_handle_detected` +- `markdown_link_detected` +- `overclaim_detected` +- `direction_semantics_missing` +- `duplicate_summary_detected` +- `unsupported_actor_claim` +- `unsupported_function_claim` + +### 12.3 产物提交策略 + +`candidate-explanations` 是 public-safe display artifact，但默认仍按运行产物处理： + +- 本地真实运行生成的 `data/external-discovery/*.candidate-explanations.json` 默认不提交。 +- 只有经过脱敏、用于测试的 fixture 可以提交。 +- 若未来要把解释摘要作为线上页面预生成数据发布，必须在对应 release / deploy 流程中运行 public-safe 校验。 + +## 13. Test & Review Plan + +### Unit tests + +- 项目候选能生成或回退 `what_it_is_cn` / `why_watch_cn`。 +- 方向候选必须保留“方向线索”语义。 +- LLM disabled 时 artifact status 为 `skipped` 或返回 null，不影响 aggregate。 +- LLM 返回非法 JSON 时可 repair 或 fallback。 +- LLM 输出过度结论时被 validation 拒绝。 +- 未命中 repo description 时不得编造项目功能。 +- eligible candidates enhanced explanation 覆盖率低于 70% 时，artifact status 必须是 `partial`。 +- 有 `existing_project_brief_cn` 的候选必须产生 `existing_project_brief` source 的 explanation。 +- `CandidateExplanationInput` 包含 profile URL、raw handle、长正文时必须被 public-safe 校验拒绝。 + +### Integration tests + +- 使用 fixture aggregate 生成 `candidate-explanations.json`。 +- 合并 explanation artifact 后，view-model 优先展示 LLM 摘要。 +- explanation artifact 缺失时 UI 仍能展示规则 fallback。 +- `assertPublicSafeAggregate` 仍覆盖原 aggregate，新增 explanation artifact 需要对应 public-safe 检查。 +- 使用真实或近真实 fixture 检查至少 5 个候选的 `what_it_is_cn`，不得全部退化成“被外部讨论提到的候选”。 + +### Manual review + +- 选择 3 类真实候选检查： + - 已绑定 repo 项目候选。 + - 未绑定方向线索。 + - 已有候选外部补证。 +- 检查中文文案是否能回答“它是什么 / 为什么值得看”。 +- 检查是否出现把外部层写成主榜结论的表达。 + +## 14. Risks / Open Questions + +### Risks + +- 如果输入缺少 repo description 或 source title，LLM 能生成的“是什么”仍会偏保守。 +- 如果把 explanation artifact 直接混入 aggregate，后续事实契约和解释契约容易混乱。 +- 如果英文 UI 直接展示中文摘要，会破坏语言切换体验。 +- 如果 prompt 不限制，LLM 容易把方向线索写成确定项目或趋势结论。 + +### Open Questions + +1. 后续是否需要增加 `what_it_is_en` / `why_watch_en`。 +2. 线上部署时 explanation artifact 是按 daily 预生成，还是页面构建时读取 latest artifact 后合并。 + +## 15. Recommended Next Step + +本设计通过后，再生成对应 exec-plan。exec-plan 应明确： + +1. 新增文件和路径。 +2. LLM prompt 与 validation 实现。 +3. explanation artifact 写入与 latest 同步。 +4. UI view-model merge 规则。 +5. 测试命令和真实 dry-run 验证步骤。 diff --git a/docs/specs/exec-plans/agentreach-candidate-explanation-enhancement.exec-plan.md b/docs/specs/exec-plans/agentreach-candidate-explanation-enhancement.exec-plan.md new file mode 100644 index 0000000..b285d87 --- /dev/null +++ b/docs/specs/exec-plans/agentreach-candidate-explanation-enhancement.exec-plan.md @@ -0,0 +1,425 @@ +# 执行计划：AgentReach 候选解释增强层 + +## 文档状态 + +- 版本：`v0.1` +- 当前状态：`Approved for Implementation` +- 关联需求：`docs/specs/product-specs/外部发现与补证信号层需求分析.md` +- 关联设计：`docs/specs/design-docs/agentreach-candidate-explanation-enhancement-design.md` +- 计划范围：为 AgentReach 外部发现候选新增后端 display-only 解释增强层，生成“它是什么 / 为什么值得看”的 public-safe explanation artifact，并让 `/agentreach` view-model 可优先消费该产物。 +- 非目标：不做外部讨论趋势，不扩 Agent-Reach 上游，不新增平台 API，不修改主评分，不让前端直接调用 DeepSeek，不提交本地真实运行生成的 explanation artifact。 + +## 任务信息 + +| 字段 | 内容 | +| --- | --- | +| 任务名称 | AgentReach Candidate Explanation Enhancement | +| 负责人 | Codex | +| 风险等级 | `High` | +| 关联设计 | `docs/specs/design-docs/agentreach-candidate-explanation-enhancement-design.md` | +| 主要影响范围 | `src/externalDiscovery/`、`src/action/*`、`src/visualConsole/*`、`app/client/*`、`src/__tests__/*`、`docs/specs/*` | +| 主要产物 | `external-discovery.candidate-explanations.v1` artifact、候选解释 input builder、public-safe 校验、LLM 摘要生成与 fallback、view-model merge | + +## 目标 + +在现有 AgentReach external discovery aggregate 基础上，补齐候选解释增强层，让 `/agentreach` 页面能够基于后端产物展示： + +- `它是什么`：候选项目、产品、论文或方向线索的可读说明。 +- `为什么值得看`：今天进入外部发现页面的外部层原因。 +- `还需要确认什么`：外部层作为次级信号的限制与 caveat。 + +该增强层只消费 public-safe aggregate、项目库/observer 已有简介、sanitized title 等安全输入；LLM 失败或输出不合格时必须回退规则文案，且不得影响 `DailyExternalAggregate` 生成。 + +## 设计冻结摘要 + +1. 新增旁路 artifact，而不是把 LLM 摘要直接写入 `DailyExternalAggregate`。 +2. `CandidateExplanationInput` 是 LLM prompt 的唯一输入包；必须先通过 public-safe 校验。 +3. `public_evidence_titles` / `public_source_titles`、`existing_project_brief_cn`、`repo_description` 是解释“它是什么”的关键输入，不得被实现阶段跳过。 +4. 默认 eligible candidate 覆盖 `top_n=30`；增强覆盖率低于 70% 时 artifact `status=partial`。 +5. `rules_fallback` 只保证页面不空，不计入增强覆盖率。 +6. `summary_confidence` 只表示解释输入是否足够，不表示项目质量或主榜可信度。 +7. 方向候选必须显示为方向线索，不得伪装成项目。 +8. 前端不直接调用 LLM，不暴露 `DEEPSEEK_API_KEY`。 + +## 新增 / 修改模块 + +### 新增模块 + +- `src/externalDiscovery/explanations.ts` + - 定义 `ExternalCandidateExplanationArtifact`、`ExternalCandidateExplanation`、`CandidateExplanationInput` 等类型。 + - 实现 candidate key、scope、summary source、confidence、status / reason code 的基础 helper。 + - 实现 `buildCandidateExplanationInputs`。 + - 实现 `generateExternalCandidateExplanations`。 + - 实现 LLM 输出解析、校验、fallback、audit 聚合。 + +- `src/externalDiscovery/explanationPrompts.ts` + - 构建固定中文 prompt。 + - 只接收 `CandidateExplanationInput[]`。 + - 要求 JSON-only 输出，不允许 markdown。 + - 固定禁止过度结论、方向伪装项目、无依据功能编造。 + +- `src/externalDiscovery/explanationRedaction.ts` + - 实现 `assertPublicSafeCandidateExplanationInputs`。 + - 实现 `assertPublicSafeCandidateExplanations`。 + - 复用或调用 `src/externalDiscovery/redaction.ts` 中已有敏感字符串检测能力。 + - 输出稳定 reason code。 + +- `src/externalDiscovery/explanationPaths.ts` 或扩展 `src/externalDiscovery/paths.ts` + - 增加： + - `externalCandidateExplanationsPath(date)` + - `externalCandidateExplanationsLatestPath()` + - 路径固定为： + - `data/external-discovery/YYYY-MM-DD.candidate-explanations.json` + - `data/external-discovery/latest.candidate-explanations.json` + +### 修改模块 + +- `src/externalDiscovery/agentReachProvider.ts` + - 如当前 canonical event / aggregate 缺少 public-safe title 上下文，保留现有行为，不回读 raw body。 + - 若已有 provider item title 可安全使用，映射到 explanation input builder 可消费的 sanitized title context。 + - 不得把 raw post body、评论全文、handle、profile URL 写入 public aggregate 或 explanation input。 + +- `src/externalDiscovery/aggregate.ts` + - 不反向写入 LLM 摘要。 + - 可暴露 input builder 所需的 project / direction evidence lookup helper。 + +- `src/externalDiscovery/dailyIntegration.ts` + - 在 daily external aggregate 生成并通过 public-safe 校验后，调用 explanation generator。 + - 作为 explanation generation 的唯一 orchestration owner。 + - dry-run 时不得写真实 explanation artifact。 + - LLM disabled 时可返回 skipped artifact 或 null，由 exec 阶段按现有 action 风格选择。 + +- `src/action/dailyReport.ts` + - 只消费 external discovery orchestration result。 + - 不负责构建 `CandidateExplanationInput`。 + - 不直接调用 LLM 或写 explanation artifact。 + +- `src/visualConsole/agentReachViewModel.ts` + - 读取 `latest.candidate-explanations.json` 或对应日期 artifact。 + - 校验 `aggregate_source_input_hash` / `input_context_hash` 与当前 aggregate 匹配。 + - merge explanation 到候选 view-model。 + - 中文页优先展示 `what_it_is_cn` / `why_watch_cn`；英文页继续使用英文规则 fallback。 + +- `app/client/AgentReachView.tsx` + - 只消费 view-model 字段。 + - 不发起 LLM 请求。 + - 不改变当前布局大结构；只让“它是什么 / 为什么值得看”区域优先使用后端解释摘要。 + +- `src/visualConsole/types.ts` + - 如 UI worktree 当前已有 `intro_line`、`appearance_reason`、`why_watch_reasons`，新增 explanation 字段时保持兼容。 + - 建议新增 display 字段： + - `what_it_is` + - `why_watch` + - `summary_confidence` + - `summary_source` + - `explanation_caveats` + +## 阶段进度 + +| 阶段 | 状态 | 目标 | 完成标志 | +| --- | --- | --- | --- | +| Phase 0：实现前护栏 | `Done` | 锁定分支、输入文档、禁止触碰主评分和产品生态工作 | 结构检查确认只在 AgentReach 后端/UI 相关边界内改动 | +| Phase 1：类型、路径与 public-safe 契约 | `Done` | 建立 explanation artifact 类型、路径、reason code 和安全校验 | 类型测试与 redaction 测试通过 | +| Phase 2：CandidateExplanationInput 构建 | `Done` | 从 aggregate、项目库/observer、sanitized title context 构建 LLM 输入包 | input builder 测试覆盖项目、方向、外部补证、缺 title 降级 | +| Phase 3：LLM 生成、校验与 fallback | `Done` | 复用 `callStructuredEnhancement` 生成中文摘要，并按规则拒绝/回退 | 测试覆盖 existing brief、全 fallback 不得 ok、LLM disabled | +| Phase 4：daily orchestration 与 artifact 写入 | `Done` | aggregate 生成后写 candidate explanations 与 latest 指针 | integration smoke 覆盖 sanitized fixture 写入 | +| Phase 5：Visual Console / UI merge | `Done` | `/agentreach` 中文页优先展示后端解释；英文页规则 fallback | 当前 worktree 无 React AgentReach 页面，已提供 readLayer 消费入口与 verification 契约 | +| Phase 6：验证、真实 dry-run 与清理 | `Done` | focused tests、typecheck、preflight、真实样本检查 | focused tests、external discovery regression、typecheck、preflight check 已通过 | + +## 实施阶段 + +### Phase 0：实现前护栏 + +1. 确认当前工作区和分支： + - 后端分支：`codex/agentreach-backend-named-actors` 或本任务专用 AgentReach 分支。 + - UI worktree 如需改展示，只触碰 AgentReach view-model / view 文件。 +2. 阅读并记录设计依据： + - `docs/specs/design-docs/agentreach-candidate-explanation-enhancement-design.md` + - `docs/specs/design-docs/agent-reach-external-discovery-and-evidence-design.md` + - `docs/specs/product-specs/外部发现与补证信号层需求分析.md` +3. 结构护栏： + - 不改 `ScoreBreakdown`。 + - 不改 `RawSignal` 主链路语义。 + - 不新增 live GitHub / X / Reddit / HN API 请求。 + - 不让 React / browser 直接读取 `DEEPSEEK_API_KEY`。 +4. 若发现当前分支混入产品生态任务改动，停止并先整理工作区。 + +### Phase 1：类型、路径与 public-safe 契约 + +1. 新增或扩展类型： + - `ExternalCandidateExplanationArtifact` + - `ExternalCandidateExplanation` + - `CandidateExplanationInput` + - `ExternalCandidateExplanationStatus` + - reason code 类型： + - `llm_disabled` + - `no_candidates` + - `candidate_explanation_ok` + - `partial_llm_failure` + - `summary_generation_failed` + - `summary_validation_failed` + - `input_context_insufficient` + - `public_safe_validation_failed` + - `provider_unavailable` + - `unexpected_exception` + - `candidate_key_missing` + - `candidate_not_eligible` + - `public_title_context_missing` + - `project_brief_reused` + - `llm_output_rejected` + - `rules_fallback_used` + - `direction_candidate_low_confidence` + - `primary_source_confirmation_required` +2. 增加路径 helper： + - `externalCandidateExplanationsPath(date)` + - `externalCandidateExplanationsLatestPath()` +3. 实现 public-safe 校验： + - input 校验拒绝 token、profile URL、raw handle、长正文、markdown link、未脱敏 URL 列表。 + - output 校验拒绝 token、profile URL、raw handle、markdown link、过度结论、方向语义缺失、重复摘要、无依据机构/功能声明。 +4. 测试： + - unsafe input title 被拒绝。 + - unsafe explanation 被拒绝。 + - artifact 缺少 `public_safe=true` 或 `contains_raw_text=false` 时失败。 + +### Phase 2：CandidateExplanationInput 构建 + +1. 冻结 sanitized title context contract： + - `agentReachProvider.ts` 只提取 provider item 中可 public-safe 使用的 `title` / target name，不保留 raw body、评论全文、handle、profile URL。 + - title context 不反写 `DailyExternalAggregate`。 + - title context 只作为 `buildCandidateExplanationInputs` 的第二输入：`aggregate + titleContext`。 + - title context 必须参与 `input_context_hash`。 + - 如果只有 `DailyExternalAggregate` 而没有 title context，input builder 必须继续运行，但对应候选记录 `public_title_context_missing`，并将 `input_confidence` 降为 `medium` 或 `low`。 + - title context 结构建议为： + +```ts +interface CandidateExplanationTitleContext { + event_id?: string; + target_key: string; + target_display_name?: string; + public_evidence_title?: string; + public_source_title?: string; + source_platform: ExternalPlatform; +} +``` + +2. 从 `DailyExternalAggregate` 构建候选索引： + - `observation_candidates` 作为候选基准。 + - `project_evidence` / `direction_evidence` 按 `target_key` 关联。 + - 无法关联 evidence 的候选保留，但 `input_confidence=low`。 +3. 生成 `candidate_key`： + - 格式固定为 `${candidate_kind}:${target_key}`。 + - 同一 aggregate 内冲突时记录 warning 并稳定去重。 +4. 判定 `explanation_scope`： + - `external_evidence_boost`：已有候选补证语义或 evidence `derived_signal_kinds` 包含 `evidence` 且对象已知。 + - `bound_object`：项目 / 产品 / 论文候选有 repo/object URL 或已知 project brief。 + - `direction_signal`：direction candidate、direction evidence、无明确对象绑定。 +5. 收集语义输入： + - `existing_project_brief_cn`：从 project library / observer / daily report 复用。 + - `repo_description`：只从已有主链路项目数据读取，不新增 live GitHub 请求。 + - `public_evidence_titles` / `public_source_titles`：只使用 sanitized title context；最多 5 条，每条不超过 160 字符。 + - `evidence_reason_facts`：规则生成外部层原因，如平台、跨平台、具名讨论者、可进日报/周报、需主源确认。 +6. 排序和 eligible selection： + - 默认 `top_n=30`。 + - 排序：`external_evidence_boost` > `bound_object` > `direction_signal`。 + - 同类按 evidence 数、平台数、`top_tier_actor_count`、`mention_count` 降序。 + - direction eligible 至少前 5 个。 +7. 测试： + - 项目候选含 repo URL 和 title 时 `input_confidence` 至少 `medium`。 + - 有 `project_brief_cn` 的候选 `input_confidence=high`。 + - 方向候选 scope 固定为 `direction_signal`。 + - 缺 title / description 时记录 `public_title_context_missing`。 + - 只有 aggregate、没有 title context 时仍可生成 fallback input，但不得标记 high confidence。 + +### Phase 3：LLM 生成、校验与 fallback + +1. 复用 `callStructuredEnhancement`： + - 不新增新的 provider 注册方式。 + - 读取现有 `config.llm` 和 `LLM_PROVIDER=deepseek` 等配置。 +2. Prompt 规则： + - 输入为 `CandidateExplanationInput[]`。 + - 输出 JSON object：`{ explanations: [...] }`。 + - 禁止 markdown。 + - 禁止访问互联网、禁止推测未提供功能。 + - 方向候选必须写“方向线索”。 + - 证据不足必须写“仍需 GitHub / Trendshift 主链路确认”。 +3. 输出校验： + - candidate key 必须匹配。 + - 中文字段非空，建议 30-140 中文字符。 + - 不得包含 URL / markdown / token / profile URL。 + - 不得包含主榜结论、确定爆发、趋势已成等表达。 + - 未提供功能描述时不得编造具体能力。 + - 重复摘要拒绝或回退。 +4. Fallback： + - `existing_project_brief_cn` 直接生成 `summary_source=existing_project_brief`。 + - LLM disabled：artifact `status=skipped` 或 generator 返回 null，按 action 风格在 Phase 4 冻结。 + - 单候选失败：`rules_fallback`。 + - 增强覆盖率低于 70%：artifact `status=partial`。 + - 增强覆盖率为 0 且候选数大于 0：artifact `status=failed` 或 `skipped`。 +5. 测试： + - LLM stub 成功返回项目简介。 + - LLM stub 返回非法 JSON，repair 或 fallback。 + - LLM 输出“已经成为趋势”被拒绝。 + - 方向候选不含“方向/线索”被拒绝。 + - 全部 fallback 不得标记为 `ok`。 + +### Phase 4：daily orchestration 与 artifact 写入 + +1. 在 daily external aggregate 生成并通过 `assertPublicSafeAggregate` 后执行： + - `buildCandidateExplanationInputs` + - `assertPublicSafeCandidateExplanationInputs` + - `generateExternalCandidateExplanations` + - `assertPublicSafeCandidateExplanations` + - 写入 date artifact 和 latest artifact +2. 写入策略： + - 非 dry-run 写： + - `data/external-discovery/YYYY-MM-DD.candidate-explanations.json` + - `data/external-discovery/latest.candidate-explanations.json` + - dry-run 不写盘，只返回 planned writes。 + - 本地真实运行产物默认不提交。 +3. `input_context_hash`： + - 对排序后的 `CandidateExplanationInput[]` 做稳定 hash。 + - UI merge 时必须匹配。 +4. run-summary / verification： + - 记录 explanation status、eligible、attempted、enhanced、fallback、warnings。 + - explanation failed 不阻断主 daily。 +5. 测试： + - dry-run 不写 explanation artifact。 + - latest 指针写入正确。 + - source hash / input context hash 不匹配时 verify warning 或 fail。 + +### Phase 5：Visual Console / UI merge + +1. view-model 读取： + - 当前 date 对应 explanation artifact 优先。 + - fallback 到 `latest.candidate-explanations.json` 时必须与当前 aggregate hash 匹配。 +2. merge 规则： + - `what_it_is_cn` -> 中文页 `它是什么`。 + - `why_watch_cn` -> 中文页 `为什么值得看`。 + - `caveats` -> `还需要确认什么`。 + - `summary_source` / `summary_confidence` 可作为低调 chip 或调试字段，不抢主内容。 +3. 英文页： + - 不展示中文 LLM 摘要。 + - 继续使用英文规则 fallback。 + - 后续英文摘要需新增 `what_it_is_en` / `why_watch_en`。 +4. UI 约束： + - 不改变当前 `/agentreach` 页面大布局。 + - 不新增搜索框。 + - 不把外部层写成主榜结论。 +5. 测试： + - 中文页展示 LLM `它是什么` 和 `为什么值得看`。 + - 英文页不泄漏中文摘要。 + - artifact 缺失时规则 fallback 正常。 + - hash 不匹配时不使用 stale explanation。 + +### Phase 6：验证、真实 dry-run 与清理 + +1. focused tests： + +```powershell +corepack pnpm exec vitest run src/__tests__/externalDiscoveryCandidateExplanations.test.ts src/__tests__/externalDiscoveryCandidateExplanationRedaction.test.ts src/__tests__/externalDiscoveryCandidateExplanationInput.test.ts src/__tests__/visualConsoleAgentReachViewModel.test.ts src/__tests__/visualConsoleAgentReachPresentation.test.ts +``` + +2. existing external discovery regression： + +```powershell +corepack pnpm exec vitest run src/__tests__/externalDiscoveryAdapter.test.ts src/__tests__/externalDiscoveryAggregate.test.ts src/__tests__/externalDiscoveryRedaction.test.ts src/__tests__/externalDiscoveryVerification.test.ts +``` + +3. typecheck： + +```powershell +corepack pnpm run typecheck +``` + +4. implementation preflight： + +```powershell +corepack pnpm run code-implementation:preflight -- --write +corepack pnpm run code-implementation:preflight -- --check +``` + +5. 真实或近真实 dry-run： + +```powershell +corepack pnpm exec tsx src/cli.ts run-daily --date --external-discovery-input data/raw/external-discovery/.agent-reach.json --dry-run +``` + +dry-run 验收检查： + +- planned writes 中包含 date explanation artifact 和 latest explanation artifact。 +- dry-run 不写入真实 `*.candidate-explanations.json`。 +- 输出包含 explanation status、eligible、attempted、enhanced、fallback、warnings。 + +6. non-dry local smoke： + +```powershell +corepack pnpm exec tsx src/cli.ts run-daily --date --external-discovery-input data/raw/external-discovery/fixtures/.agent-reach.json +``` + +non-dry local smoke 只允许使用 sanitized fixture，不允许使用包含 raw social text、profile URL、handle、token 或 private diagnostics 的真实 raw input。 + +non-dry 验收检查： + +- 生成 `data/external-discovery/.candidate-explanations.json`。 +- 生成或更新 `data/external-discovery/latest.candidate-explanations.json`。 +- `aggregate_source_input_hash` 和 `input_context_hash` 存在。 +- artifact 通过 `assertPublicSafeCandidateExplanations`。 +- 至少 5 个真实候选的 `what_it_is_cn` 不得全部退化为“被外部讨论提到的候选”。 +- 有 `project_brief_cn` 的候选必须使用 `existing_project_brief`。 +- 方向候选必须保留“方向线索”语义。 +- explanation artifact 不包含 raw handle、profile URL、token、markdown link、长正文。 +- LLM 失败不影响 external aggregate。 + +7. 清理： + - 不提交 `data/external-discovery/*.candidate-explanations.json` 本地真实运行产物。 + - non-dry local smoke 后删除本次生成的 date explanation artifact；如 latest 指针被更新，应恢复到测试前状态或删除测试 latest。 + - 只保留 sanitized fixture 和测试需要的小样本。 + +## 新增测试文件建议 + +- `src/__tests__/externalDiscoveryCandidateExplanations.test.ts` +- `src/__tests__/externalDiscoveryCandidateExplanationInput.test.ts` +- `src/__tests__/externalDiscoveryCandidateExplanationRedaction.test.ts` +- `src/__tests__/externalDiscoveryCandidateExplanationPaths.test.ts` +- 更新： + - `src/__tests__/externalDiscoveryVerification.test.ts` + - `src/__tests__/visualConsoleAgentReachViewModel.test.ts` + - `src/__tests__/visualConsoleAgentReachPresentation.test.ts` + +## 验收标准 + +1. 生成 `external-discovery.candidate-explanations.v1` artifact。 +2. explanation artifact 包含 `public_safe=true`、`contains_raw_text=false`、`contains_profile_urls=false`。 +3. `CandidateExplanationInput` 通过 public-safe 校验后才进入 LLM prompt。 +4. 启用 LLM 且存在候选时，eligible top 30 的增强覆盖率低于 70% 不得标记 `ok`。 +5. `rules_fallback` 不计入增强覆盖率。 +6. 有 `existing_project_brief_cn` 的候选必须生成 `summary_source=existing_project_brief`。 +7. 方向候选不得被写成明确项目。 +8. LLM 输出过度结论、重复摘要、无依据功能声明时必须被拒绝。 +9. 前端中文页优先展示后端解释摘要；英文页不泄漏中文摘要。 +10. 外部解释增强不改变主 score、`RawSignal`、`DailyExternalAggregate` 事实契约或 primary conclusion。 + +## 回滚策略 + +1. 如果 explanation generator 出现问题： + - 禁用 LLM 或跳过 explanation artifact 生成。 + - `/agentreach` UI 回退现有规则字段。 + - external aggregate 和 daily 主产物继续可用。 +2. 如果 public-safe 校验误伤： + - 保留 explanation generation skipped / partial 状态。 + - 不放宽 raw text / profile URL / token 禁止项。 +3. 如果 UI merge 出现 stale artifact： + - 通过 `aggregate_source_input_hash` / `input_context_hash` 禁用该 explanation artifact。 + - 回退规则 fallback。 +4. 回滚不得删除或修改 `DailyExternalAggregate` 既有字段。 + +## 验证记录 + +| 日期 | 命令 | 结果 | 备注 | +| --- | --- | --- | --- | +| 2026-07-01 | `ExecPlan created` | `Pending review` | 仅生成计划，尚未执行代码实现 | +| 2026-07-01 | `ExecPlan review` | `Approved with nits` | 已确认可进入代码实现；实现时优先扩展 `paths.ts`，并在 `dailyIntegration` 归一 LLM disabled 为 `skipped` | +| 2026-07-01 | `corepack pnpm exec vitest run src/__tests__/externalDiscoveryCandidateExplanationInput.test.ts src/__tests__/externalDiscoveryCandidateExplanationRedaction.test.ts src/__tests__/externalDiscoveryCandidateExplanations.test.ts src/__tests__/externalDiscoveryDailyIntegration.test.ts src/__tests__/externalDiscoveryTypeContract.test.ts src/__tests__/externalDiscoveryStructure.test.ts src/__tests__/externalDiscoveryVerification.test.ts` | `Passed` | 7 files / 17 tests | +| 2026-07-01 | `corepack pnpm run typecheck` | `Passed` | TypeScript noEmit 通过 | +| 2026-07-01 | `corepack pnpm exec vitest run src/__tests__/externalDiscoveryAdapter.test.ts src/__tests__/externalDiscoveryAggregate.test.ts src/__tests__/externalDiscoveryRedaction.test.ts src/__tests__/externalDiscoveryVerification.test.ts` | `Passed` | 4 files / 23 tests | +| 2026-07-01 | `corepack pnpm run code-implementation:preflight -- --check` | `Passed` | skill and exec-plan receipt verified | diff --git a/src/__tests__/externalDiscoveryCandidateExplanationInput.test.ts b/src/__tests__/externalDiscoveryCandidateExplanationInput.test.ts new file mode 100644 index 0000000..a5e6a1c --- /dev/null +++ b/src/__tests__/externalDiscoveryCandidateExplanationInput.test.ts @@ -0,0 +1,104 @@ +import { describe, expect, it } from "vitest"; +import { buildCandidateExplanationInputs } from "../externalDiscovery/explanations.ts"; +import type { DailyExternalAggregate, ExternalSignalEvent } from "../externalDiscovery/types.ts"; +import { buildDailyExternalAggregate } from "../externalDiscovery/aggregate.ts"; +import type { ProjectLibraryEnhancementArtifact, ScoredProject } from "../types.ts"; + +function event(overrides: Partial = {}): ExternalSignalEvent { + return { + event_id: "evt-1", + platform: "hacker_news", + raw_event_kind: "discussion", + derived_signal_kinds: ["evidence"], + scope: "project", + target_type: "project", + target_key: "openai/agents-sdk", + actor: { actor_type: "community", effective_tier: "ordinary", tier_basis: "none" }, + observed_at: "2026-06-30T00:00:00.000Z", + raw_ref: "provider:event:1", + ...overrides, + }; +} + +function aggregate(events: ExternalSignalEvent[]): DailyExternalAggregate { + return buildDailyExternalAggregate({ + date: "2026-06-30", + generated_at: "2026-06-30T01:00:00.000Z", + events, + }); +} + +describe("external candidate explanation input builder", () => { + it("builds project inputs from aggregate evidence and existing project brief", () => { + const projectLibraryArtifact: ProjectLibraryEnhancementArtifact = { + date: "2026-06-30", + generated_at: "2026-06-30T01:00:00.000Z", + provider: "deepseek", + entries: [ + { + repo_full_name: "openai/agents-sdk", + project_brief_cn: "OpenAI Agents SDK 是用于构建和编排智能体工作流的开发工具包。", + source: "agent", + provider: "deepseek", + generated_at: "2026-06-30T01:00:00.000Z", + }, + ], + }; + const scored = [ + { + project: { + repo_full_name: "openai/agents-sdk", + repo_url: "https://github.com/openai/agents-sdk", + project_name: "Agents SDK", + description: "Build agentic workflows with tools and handoffs.", + }, + }, + ] as ScoredProject[]; + + const result = buildCandidateExplanationInputs({ + aggregate: aggregate([event()]), + titleContext: [ + { + event_id: "evt-1", + target_key: "openai/agents-sdk", + target_display_name: "Agents SDK", + public_evidence_title: "Launch HN: Agents SDK", + source_platform: "hacker_news", + }, + ], + scoredProjects: scored, + projectLibraryArtifact, + }); + + expect(result.inputs[0]).toMatchObject({ + candidate_key: "project:openai/agents-sdk", + explanation_scope: "external_evidence_boost", + input_confidence: "high", + existing_project_brief_cn: "OpenAI Agents SDK 是用于构建和编排智能体工作流的开发工具包。", + repo_description: "Build agentic workflows with tools and handoffs.", + }); + expect(result.input_context_hash).toHaveLength(64); + }); + + it("keeps direction evidence as direction_signal and lowers confidence when title context is missing", () => { + const result = buildCandidateExplanationInputs({ + aggregate: aggregate([ + event({ + event_id: "direction-1", + scope: "direction", + target_type: "topic", + target_key: "agent memory evaluation workflows", + derived_signal_kinds: ["discovery"], + }), + ]), + }); + + expect(result.inputs[0]).toMatchObject({ + candidate_key: "direction:agent memory evaluation workflows", + explanation_scope: "direction_signal", + input_confidence: "low", + }); + expect(result.warnings.map((item) => item.reason_code)).toContain("public_title_context_missing"); + expect(result.warnings.map((item) => item.reason_code)).toContain("direction_candidate_low_confidence"); + }); +}); diff --git a/src/__tests__/externalDiscoveryCandidateExplanationRedaction.test.ts b/src/__tests__/externalDiscoveryCandidateExplanationRedaction.test.ts new file mode 100644 index 0000000..d6a1a6f --- /dev/null +++ b/src/__tests__/externalDiscoveryCandidateExplanationRedaction.test.ts @@ -0,0 +1,142 @@ +import { describe, expect, it } from "vitest"; +import { + assertPublicSafeCandidateExplanations, + assertPublicSafeCandidateExplanationInputs, +} from "../externalDiscovery/explanationRedaction.ts"; +import type { CandidateExplanationInput, ExternalCandidateExplanationArtifact } from "../externalDiscovery/explanations.ts"; + +function input(overrides: Partial = {}): CandidateExplanationInput { + return { + candidate_key: "project:openai/agents-sdk", + candidate_kind: "project", + target_key: "openai/agents-sdk", + display_name: "Agents SDK", + explanation_scope: "bound_object", + repo_url: "https://github.com/openai/agents-sdk", + public_evidence_titles: ["Launch HN: Agents SDK"], + public_source_titles: [], + evidence_reason_facts: ["来源平台：HN", "仍需主源确认，不能作为主榜结论"], + evidence_ids: ["project:openai/agents-sdk"], + platforms: ["hacker_news"], + mention_count: 1, + distinct_actor_count: 1, + top_tier_actor_count: 0, + named_registry_actor_names: [], + can_enter_daily: true, + can_enter_weekly: false, + cannot_be_primary_conclusion: true, + input_confidence: "medium", + input_warnings: [], + ...overrides, + }; +} + +function artifact(overrides: Partial = {}): ExternalCandidateExplanationArtifact { + return { + schema_version: "external-discovery.candidate-explanations.v1", + date: "2026-06-30", + generated_at: "2026-06-30T01:00:00.000Z", + provider: "rules", + explanation_policy_version: "candidate-explanations.v1", + aggregate_source_input_hash: "hash", + aggregate_generated_at: "2026-06-30T01:00:00.000Z", + input_context_hash: "hash", + public_safe: true, + redaction_policy_version: "external-discovery-explanation-redaction.v1", + contains_raw_text: false, + contains_profile_urls: false, + status: "ok", + status_reason: "candidate_explanation_ok", + explanations: [ + { + candidate_key: "direction:agent memory", + candidate_kind: "direction", + target_key: "agent memory", + explanation_scope: "direction_signal", + what_it_is_cn: "agent memory 是今天外部讨论中出现的方向线索，尚未绑定明确 GitHub 项目。", + why_watch_cn: "HN 出现相关讨论，适合先进入方向观察，但不能替代主链路判断。", + summary_confidence: "low", + summary_source: "rules_fallback", + evidence_ids: ["direction:agent memory"], + platforms: ["hacker_news"], + caveats: ["外部层不能作为主榜结论，仍需 GitHub / Trendshift 主链路确认。"], + generated_at: "2026-06-30T01:00:00.000Z", + }, + ], + audit: { + eligible_count: 1, + attempted_count: 0, + accepted_count: 1, + enhanced_count: 0, + rejected_count: 0, + fallback_count: 1, + warnings: [], + }, + ...overrides, + }; +} + +describe("external candidate explanation redaction", () => { + it("rejects unsafe title context before LLM input", () => { + const result = assertPublicSafeCandidateExplanationInputs([ + input({ public_evidence_titles: ["please inspect @raw_handle profile"] }), + ]); + + expect(result.ok).toBe(false); + expect(result.reason_codes).toContain("raw_handle_detected"); + }); + + it("rejects overclaimed explanation output", () => { + const result = assertPublicSafeCandidateExplanations( + artifact({ + explanations: [ + { + ...artifact().explanations[0]!, + what_it_is_cn: "agent memory 是已经形成趋势的主榜结论。", + }, + ], + }), + ); + + expect(result.ok).toBe(false); + expect(result.reason_codes).toContain("overclaim_detected"); + }); + + it("requires direction explanations to keep direction semantics", () => { + const result = assertPublicSafeCandidateExplanations( + artifact({ + explanations: [ + { + ...artifact().explanations[0]!, + what_it_is_cn: "agent memory 是一个明确 GitHub 项目。", + why_watch_cn: "今天被外部来源提到，可以继续查看。", + }, + ], + }), + ); + + expect(result.ok).toBe(false); + expect(result.reason_codes).toContain("direction_semantics_missing"); + }); + + it("rejects project explanations that describe the candidate as a direction clue", () => { + const result = assertPublicSafeCandidateExplanations( + artifact({ + explanations: [ + { + ...artifact().explanations[0]!, + candidate_key: "project:openai/agents-sdk", + candidate_kind: "project", + target_key: "openai/agents-sdk", + explanation_scope: "bound_object", + what_it_is_cn: "OpenAI Agents SDK 是今天外部讨论中出现的方向线索，尚未绑定明确 GitHub 项目。", + why_watch_cn: "HN 出现该对象讨论，仍需 GitHub / Trendshift 主链路确认。", + }, + ], + }), + ); + + expect(result.ok).toBe(false); + expect(result.reason_codes).toContain("project_semantics_conflict"); + }); +}); diff --git a/src/__tests__/externalDiscoveryCandidateExplanations.test.ts b/src/__tests__/externalDiscoveryCandidateExplanations.test.ts new file mode 100644 index 0000000..2611346 --- /dev/null +++ b/src/__tests__/externalDiscoveryCandidateExplanations.test.ts @@ -0,0 +1,123 @@ +import { describe, expect, it } from "vitest"; +import { buildDailyExternalAggregate } from "../externalDiscovery/aggregate.ts"; +import { + buildCandidateExplanationInputs, + generateExternalCandidateExplanations, +} from "../externalDiscovery/explanations.ts"; +import type { AppConfig } from "../config.ts"; +import type { ExternalSignalEvent } from "../externalDiscovery/types.ts"; +import type { ProjectLibraryEnhancementArtifact } from "../types.ts"; + +function config(enabled: boolean): AppConfig { + return { + llm: { + enabled, + mode: enabled ? "semantic-classification" : "rules-only", + provider: enabled ? "deepseek" : "none", + }, + } as AppConfig; +} + +function event(overrides: Partial = {}): ExternalSignalEvent { + return { + event_id: "evt-1", + platform: "hacker_news", + raw_event_kind: "discussion", + derived_signal_kinds: ["evidence"], + scope: "project", + target_type: "project", + target_key: "openai/agents-sdk", + actor: { actor_type: "community", effective_tier: "ordinary", tier_basis: "none" }, + observed_at: "2026-06-30T00:00:00.000Z", + raw_ref: "provider:event:1", + ...overrides, + }; +} + +describe("external candidate explanations", () => { + it("uses existing project brief as enhanced source without calling LLM", async () => { + const aggregate = buildDailyExternalAggregate({ + date: "2026-06-30", + generated_at: "2026-06-30T01:00:00.000Z", + events: [event()], + }); + const projectLibraryArtifact: ProjectLibraryEnhancementArtifact = { + date: "2026-06-30", + generated_at: "2026-06-30T01:00:00.000Z", + provider: "deepseek", + entries: [ + { + repo_full_name: "openai/agents-sdk", + project_brief_cn: "OpenAI Agents SDK 是用于构建和编排智能体工作流的开发工具包。", + source: "agent", + provider: "deepseek", + generated_at: "2026-06-30T01:00:00.000Z", + }, + ], + }; + const inputBuild = buildCandidateExplanationInputs({ aggregate, projectLibraryArtifact }); + + const artifact = await generateExternalCandidateExplanations({ + aggregate, + inputBuild, + config: config(true), + generatedAt: "2026-06-30T02:00:00.000Z", + }); + + expect(artifact.status).toBe("ok"); + expect(artifact.audit.enhanced_count).toBe(1); + expect(artifact.explanations[0]).toMatchObject({ + summary_source: "existing_project_brief", + what_it_is_cn: "OpenAI Agents SDK 是用于构建和编排智能体工作流的开发工具包。", + }); + }); + + it("does not mark all-rules fallback output as ok", async () => { + const aggregate = buildDailyExternalAggregate({ + date: "2026-06-30", + generated_at: "2026-06-30T01:00:00.000Z", + events: [ + event({ + event_id: "direction-1", + scope: "direction", + target_type: "topic", + target_key: "agent memory evaluation workflows", + derived_signal_kinds: ["discovery"], + }), + ], + }); + const inputBuild = buildCandidateExplanationInputs({ aggregate }); + + const artifact = await generateExternalCandidateExplanations({ + aggregate, + inputBuild, + config: config(true), + generatedAt: "2026-06-30T02:00:00.000Z", + }); + + expect(artifact.status).toBe("failed"); + expect(artifact.audit.enhanced_count).toBe(0); + expect(artifact.audit.fallback_count).toBe(1); + expect(artifact.explanations[0]?.what_it_is_cn).toContain("方向线索"); + }); + + it("marks LLM disabled artifacts as skipped while keeping fallback explanations", async () => { + const aggregate = buildDailyExternalAggregate({ + date: "2026-06-30", + generated_at: "2026-06-30T01:00:00.000Z", + events: [event()], + }); + const inputBuild = buildCandidateExplanationInputs({ aggregate }); + + const artifact = await generateExternalCandidateExplanations({ + aggregate, + inputBuild, + config: config(false), + generatedAt: "2026-06-30T02:00:00.000Z", + }); + + expect(artifact.status).toBe("skipped"); + expect(artifact.status_reason).toBe("llm_disabled"); + expect(artifact.explanations[0]?.summary_source).toBe("rules_fallback"); + }); +}); diff --git a/src/__tests__/externalDiscoveryDailyIntegration.test.ts b/src/__tests__/externalDiscoveryDailyIntegration.test.ts new file mode 100644 index 0000000..fac8df2 --- /dev/null +++ b/src/__tests__/externalDiscoveryDailyIntegration.test.ts @@ -0,0 +1,122 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { runDailyExternalDiscoveryIntegration } from "../externalDiscovery/dailyIntegration.ts"; +import type { AppConfig } from "../config.ts"; +import type { ProjectLibraryEnhancementArtifact } from "../types.ts"; + +const roots: string[] = []; +const originalCwd = process.cwd(); + +afterEach(() => { + process.chdir(originalCwd); + for (const root of roots.splice(0)) { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +function setupWorkspace(): string { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "external-discovery-daily-integration-")); + roots.push(root); + fs.mkdirSync(path.join(root, "data", "raw", "external-discovery", "fixtures"), { recursive: true }); + process.chdir(root); + return root; +} + +function config(): AppConfig { + return { + llm: { + enabled: false, + mode: "rules-only", + provider: "none", + }, + } as AppConfig; +} + +function writeSampleAgentReachInput(inputPath: string): void { + fs.writeFileSync( + inputPath, + JSON.stringify({ + provider: "agent-reach", + schema_version: "agent-reach.external-discovery.v1", + provider_run_id: "run-sanitized", + generated_at: "2026-06-30T00:00:00.000Z", + query: { keyword: "agent sdk" }, + platforms: ["hacker_news"], + status: "ok", + items: [ + { + event_id: "evt-1", + platform: "hacker_news", + raw_event_kind: "discussion", + derived_signal_kinds: ["evidence"], + scope: "project", + target_type: "project", + target_key: "openai/agents-sdk", + title: "Launch HN: OpenAI Agents SDK", + target: { + name: "OpenAI Agents SDK", + repo_url: "https://github.com/openai/agents-sdk", + }, + actor: { + actor_type: "community", + effective_tier: "ordinary", + tier_basis: "none", + }, + observed_at: "2026-06-30T00:00:00.000Z", + raw_ref: "provider:event:1", + }, + ], + }), + "utf-8", + ); +} + +describe("external discovery daily integration", () => { + it("writes aggregate and candidate explanation artifacts from sanitized AgentReach input", async () => { + const root = setupWorkspace(); + const inputPath = path.join(root, "data", "raw", "external-discovery", "fixtures", "sample.agent-reach.json"); + writeSampleAgentReachInput(inputPath); + + const result = await runDailyExternalDiscoveryIntegration({ + date: "2026-06-30", + generatedAt: "2026-06-30T01:00:00.000Z", + config: config(), + inputPath, + explicitInput: true, + dryRun: false, + }); + + expect(result.aggregate.accepted_event_count).toBe(1); + expect(result.input_build.eligible_count).toBe(1); + expect(result.explanations.status).toBe("skipped"); + expect(fs.existsSync(path.join(root, "data", "external-discovery", "2026-06-30.aggregate.json"))).toBe(true); + expect(fs.existsSync(path.join(root, "data", "external-discovery", "2026-06-30.candidate-explanations.json"))).toBe(true); + expect(result.explanations.explanations[0]?.why_watch_cn).toContain("HN"); + }); + + it("keeps daily integration non-blocking when candidate explanation build fails", async () => { + const root = setupWorkspace(); + const inputPath = path.join(root, "data", "raw", "external-discovery", "fixtures", "sample.agent-reach.json"); + writeSampleAgentReachInput(inputPath); + + const result = await runDailyExternalDiscoveryIntegration({ + date: "2026-06-30", + generatedAt: "2026-06-30T01:00:00.000Z", + config: config(), + inputPath, + explicitInput: true, + dryRun: false, + projectLibraryArtifact: { + entries: [{ repo_full_name: "openai/agents-sdk", project_brief_cn: undefined }], + } as unknown as ProjectLibraryEnhancementArtifact, + }); + + expect(result.aggregate.accepted_event_count).toBe(1); + expect(result.explanations.status).toBe("failed"); + expect(result.explanations.status_reason).toBe("candidate_explanation_generation_failed"); + expect(fs.existsSync(path.join(root, "data", "external-discovery", "2026-06-30.aggregate.json"))).toBe(true); + expect(fs.existsSync(path.join(root, "data", "external-discovery", "2026-06-30.candidate-explanations.json"))).toBe(true); + }); +}); diff --git a/src/__tests__/externalDiscoveryStructure.test.ts b/src/__tests__/externalDiscoveryStructure.test.ts index 07c1dc1..77a78cc 100644 --- a/src/__tests__/externalDiscoveryStructure.test.ts +++ b/src/__tests__/externalDiscoveryStructure.test.ts @@ -12,9 +12,11 @@ describe("external discovery structure boundaries", () => { expect(gitignore).toContain("data/raw/external-discovery/**"); expect(gitignore).toContain("!data/raw/external-discovery/fixtures/**"); + expect(gitignore).toContain("data/external-discovery/*.candidate-explanations.json"); expect(dataReadme).toContain("data/raw/external-discovery/"); expect(dataReadme).toContain("local-only"); expect(dataReadme).toContain("data/external-discovery/*.aggregate.json"); + expect(dataReadme).toContain("data/external-discovery/*.candidate-explanations.json"); expect(dataReadme).toContain("no public `*.events.jsonl` default artifact"); }); diff --git a/src/__tests__/externalDiscoveryTypeContract.test.ts b/src/__tests__/externalDiscoveryTypeContract.test.ts index df7a757..40d314f 100644 --- a/src/__tests__/externalDiscoveryTypeContract.test.ts +++ b/src/__tests__/externalDiscoveryTypeContract.test.ts @@ -1,5 +1,13 @@ import { describe, expect, it } from "vitest"; -import { externalAggregateLatestPath, externalAggregatePath, externalEntityRegistryPath, externalRawInputPath, externalSanitizedFixtureDirPath } from "../externalDiscovery/paths.ts"; +import { + externalAggregateLatestPath, + externalAggregatePath, + externalCandidateExplanationsLatestPath, + externalCandidateExplanationsPath, + externalEntityRegistryPath, + externalRawInputPath, + externalSanitizedFixtureDirPath, +} from "../externalDiscovery/paths.ts"; import { EXTERNAL_NAMED_ACTOR_SOURCE_ROLES, EXTERNAL_PLATFORMS, EXTERNAL_TARGET_TYPES } from "../externalDiscovery/types.ts"; describe("external discovery type and path contract", () => { @@ -16,6 +24,8 @@ describe("external discovery type and path contract", () => { expect(slash(externalRawInputPath("2026-06-30"))).toBe("data/raw/external-discovery/2026-06-30.agent-reach.json"); expect(slash(externalAggregatePath("2026-06-30"))).toBe("data/external-discovery/2026-06-30.aggregate.json"); expect(slash(externalAggregateLatestPath())).toBe("data/external-discovery/latest.aggregate.json"); + expect(slash(externalCandidateExplanationsPath("2026-06-30"))).toBe("data/external-discovery/2026-06-30.candidate-explanations.json"); + expect(slash(externalCandidateExplanationsLatestPath())).toBe("data/external-discovery/latest.candidate-explanations.json"); expect(slash(externalEntityRegistryPath())).toBe("data/external-discovery/entity-registry.json"); expect(slash(externalSanitizedFixtureDirPath())).toBe("data/raw/external-discovery/fixtures"); }); diff --git a/src/__tests__/externalDiscoveryVerification.test.ts b/src/__tests__/externalDiscoveryVerification.test.ts index 1d3bf5b..d6f3021 100644 --- a/src/__tests__/externalDiscoveryVerification.test.ts +++ b/src/__tests__/externalDiscoveryVerification.test.ts @@ -181,11 +181,59 @@ function makeExternalAggregate(namedActorOverrides: Record = {} }; } -function writeDailyInputs(root: string, externalAggregate: Record): void { +function makeCandidateExplanations(overrides: Record = {}): Record { + return { + schema_version: "external-discovery.candidate-explanations.v1", + date, + generated_at: "2026-06-30T08:00:00.000Z", + provider: "rules", + explanation_policy_version: "candidate-explanations.v1", + aggregate_source_input_hash: "abc123", + aggregate_generated_at: "2026-06-30T08:00:00.000Z", + input_context_hash: "input-hash", + public_safe: true, + redaction_policy_version: "external-discovery-explanation-redaction.v1", + contains_raw_text: false, + contains_profile_urls: false, + status: "partial", + status_reason: "partial_llm_failure", + explanations: [ + { + candidate_key: "project:openai/agents-sdk", + candidate_kind: "project", + target_key: "openai/agents-sdk", + explanation_scope: "external_evidence_boost", + what_it_is_cn: "OpenAI Agents SDK 是已有候选获得外部补证的对象，目前只确认它在外部来源中再次出现。", + why_watch_cn: "X 出现该对象的外部讨论，可作为日报或周报的次级证据，适合继续查看证据来源。", + summary_confidence: "medium", + summary_source: "rules_fallback", + evidence_ids: ["project:openai/agents-sdk"], + platforms: ["x_twitter"], + caveats: ["外部层不能作为主榜结论，仍需 GitHub / Trendshift 主链路确认。"], + generated_at: "2026-06-30T08:00:00.000Z", + }, + ], + audit: { + eligible_count: 1, + attempted_count: 0, + accepted_count: 1, + enhanced_count: 0, + rejected_count: 0, + fallback_count: 1, + warnings: [], + }, + ...overrides, + }; +} + +function writeDailyInputs(root: string, externalAggregate: Record, candidateExplanations?: Record): void { writeJson(path.join(root, "data", "reports", `${date}.run-summary.json`), makeSummary()); writeJson(path.join(root, "data", "reports", `${date}.daily.json`), makeReport()); writeJson(path.join(root, "data", "raw", "github", `${date}.enrichment.json`), []); writeJson(path.join(root, "data", "external-discovery", `${date}.aggregate.json`), externalAggregate); + if (candidateExplanations) { + writeJson(path.join(root, "data", "external-discovery", `${date}.candidate-explanations.json`), candidateExplanations); + } } describe("external discovery daily verification contract", () => { @@ -212,4 +260,26 @@ describe("external discovery daily verification contract", () => { expect(check?.detail).toContain("source_roles must be non-empty"); expect(result.status).toBe("fail"); }); + + it("fails when candidate explanations are missing for an aggregate with accepted events", () => { + const root = setupWorkspace(); + writeDailyInputs(root, makeExternalAggregate()); + + const result = buildVerifyDailyResult(date); + const check = result.checks.find((item) => item.name === "external_candidate_explanations_contract"); + + expect(check?.status).toBe("fail"); + expect(check?.detail).toContain("candidate explanations missing"); + }); + + it("fails stale candidate explanations with mismatched aggregate hash", () => { + const root = setupWorkspace(); + writeDailyInputs(root, makeExternalAggregate(), makeCandidateExplanations({ aggregate_source_input_hash: "stale-hash" })); + + const result = buildVerifyDailyResult(date); + const check = result.checks.find((item) => item.name === "external_candidate_explanations_contract"); + + expect(check?.status).toBe("fail"); + expect(check?.detail).toContain("aggregate_source_input_hash does not match"); + }); }); diff --git a/src/__tests__/runSummaryObserver.test.ts b/src/__tests__/runSummaryObserver.test.ts index 27ebf70..e913444 100644 --- a/src/__tests__/runSummaryObserver.test.ts +++ b/src/__tests__/runSummaryObserver.test.ts @@ -56,11 +56,28 @@ describe("renderDailyRunSummary observer section", () => { rate_limit: 0, network_blocked: 0, }, - }, - top_projects: [], - observer_status: { - ecosystem_focus: "active", - }, + }, + top_projects: [], + external_discovery: { + aggregate_status: "ok", + accepted_event_count: 2, + rejected_event_count: 0, + observation_candidate_count: 1, + project_evidence_count: 1, + direction_evidence_count: 1, + explanation_status: "partial", + explanation_status_reason: "partial_llm_failure", + explanation_eligible_count: 1, + explanation_attempted_count: 1, + explanation_enhanced_count: 0, + explanation_fallback_count: 1, + explanation_rejected_count: 1, + warning_count: 1, + warnings: ["summary_generation_failed:project:openai/agents-sdk"], + }, + observer_status: { + ecosystem_focus: "active", + }, observer_candidate_count: 1, observer_ecosystem_counts: { "multi-agent-coordination": 1, @@ -113,8 +130,11 @@ describe("renderDailyRunSummary observer section", () => { const rendered = renderDailyRunSummary(summary); expect(rendered).toContain("incubating_direction multi-agent-coordination"); - expect(rendered).toContain("observer_promotion_candidate multi-agent-coordination"); - expect(rendered).toContain("observer_promotion_candidate_count: 1"); - expect(rendered).toContain("pressure_state_distribution"); - }); -}); + expect(rendered).toContain("observer_promotion_candidate multi-agent-coordination"); + expect(rendered).toContain("observer_promotion_candidate_count: 1"); + expect(rendered).toContain("pressure_state_distribution"); + expect(rendered).toContain("## AgentReach 外部发现"); + expect(rendered).toContain("explanation_status: partial"); + expect(rendered).toContain("summary_generation_failed:project:openai/agents-sdk"); + }); +}); diff --git a/src/action/dailyVerification.ts b/src/action/dailyVerification.ts index 85b9507..87bd6c9 100644 --- a/src/action/dailyVerification.ts +++ b/src/action/dailyVerification.ts @@ -1,7 +1,9 @@ import fs from "node:fs"; import path from "node:path"; -import { externalAggregatePath } from "../externalDiscovery/paths.ts"; +import { externalAggregatePath, externalCandidateExplanationsPath } from "../externalDiscovery/paths.ts"; import { assertPublicSafeAggregate } from "../externalDiscovery/redaction.ts"; +import { assertPublicSafeCandidateExplanations } from "../externalDiscovery/explanationRedaction.ts"; +import type { ExternalCandidateExplanationArtifact } from "../externalDiscovery/explanations.ts"; import { readJsonFile } from "../storage/files.ts"; import type { DailyReport, @@ -476,6 +478,103 @@ function externalAggregateContractChecks(filepath: string, aggregateRead: Option ]; } +function externalCandidateExplanationContractChecks( + filepath: string, + aggregateRead: OptionalJsonRead, + explanationsRead: OptionalJsonRead, +): VerificationCheck[] { + if (!explanationsRead.exists) { + const needsExplanationArtifact = aggregateNeedsCandidateExplanations(aggregateRead.value); + return [ + buildCheck( + "external_candidate_explanations_contract", + needsExplanationArtifact ? "fail" : "pass", + needsExplanationArtifact + ? `candidate explanations missing at ${filepath}; external aggregate has accepted events or candidates` + : `candidate explanations not present at ${filepath}; external explanation layer not run for this date`, + ), + ]; + } + + if (explanationsRead.error) { + return [buildCheck("external_candidate_explanations_contract", "fail", `candidate explanations unreadable: ${explanationsRead.error}`)]; + } + + const artifact = explanationsRead.value as ExternalCandidateExplanationArtifact; + const redaction = assertPublicSafeCandidateExplanations(artifact); + const inspection = inspectExternalCandidateExplanationContract(artifact, aggregateRead.value); + const issues = [ + ...(!redaction.ok ? [`redaction=${redaction.reason_codes.join(",")}`] : []), + ...inspection.issues, + ]; + + return [ + buildCheck( + "external_candidate_explanations_contract", + issues.length === 0 ? "pass" : "fail", + issues.length === 0 + ? `candidate explanations public-safe; status=${inspection.status}; eligible=${inspection.eligibleCount}; enhanced=${inspection.enhancedCount}; fallback=${inspection.fallbackCount}` + : issues.join("; "), + ), + ]; +} + +function aggregateNeedsCandidateExplanations(aggregate: unknown): boolean { + if (!isRecord(aggregate)) return false; + const acceptedEventCount = typeof aggregate.accepted_event_count === "number" ? aggregate.accepted_event_count : 0; + const candidates = Array.isArray(aggregate.observation_candidates) ? aggregate.observation_candidates : []; + return acceptedEventCount > 0 || candidates.length > 0; +} + +function inspectExternalCandidateExplanationContract(value: unknown, aggregate: unknown): { + status: string; + eligibleCount: number; + enhancedCount: number; + fallbackCount: number; + issues: string[]; +} { + const issues: string[] = []; + if (!isRecord(value)) { + return { status: "unknown", eligibleCount: 0, enhancedCount: 0, fallbackCount: 0, issues: ["candidate explanations must be an object"] }; + } + + if (value.schema_version !== "external-discovery.candidate-explanations.v1") { + issues.push("schema_version must be external-discovery.candidate-explanations.v1"); + } + if (value.public_safe !== true || value.contains_raw_text !== false || value.contains_profile_urls !== false) { + issues.push("candidate explanations must carry public_safe=true, contains_raw_text=false, contains_profile_urls=false"); + } + if (isRecord(aggregate) && typeof aggregate.source_input_hash === "string" && value.aggregate_source_input_hash !== aggregate.source_input_hash) { + issues.push("aggregate_source_input_hash does not match external aggregate source_input_hash"); + } + if (typeof value.input_context_hash !== "string" || value.input_context_hash.length === 0) { + issues.push("input_context_hash missing"); + } + + const explanations = Array.isArray(value.explanations) ? value.explanations : []; + const audit = isRecord(value.audit) ? value.audit : {}; + const eligibleCount = Number(audit.eligible_count ?? 0); + const enhancedCount = Number(audit.enhanced_count ?? 0); + const fallbackCount = Number(audit.fallback_count ?? 0); + const status = typeof value.status === "string" ? value.status : "unknown"; + const fallbackCountFromRows = explanations.filter((item) => isRecord(item) && item.summary_source === "rules_fallback").length; + + if (fallbackCount !== fallbackCountFromRows) { + issues.push("fallback_count does not match rules_fallback rows"); + } + if (status === "ok" && eligibleCount > 0 && enhancedCount / eligibleCount < 0.7) { + issues.push("status ok requires enhanced coverage >= 70%"); + } + + return { + status, + eligibleCount, + enhancedCount, + fallbackCount, + issues, + }; +} + function inspectExternalAggregateContract(value: unknown): { projectEvidenceCount: number; directionEvidenceCount: number; @@ -614,6 +713,8 @@ function buildChecks( report: DailyReport | null, externalAggregateFilepath: string, externalAggregate: OptionalJsonRead, + externalCandidateExplanationsFilepath: string, + externalCandidateExplanations: OptionalJsonRead, ): VerificationCheck[] { const checks = [ ...completionChecks(summary), @@ -623,6 +724,7 @@ function buildChecks( ...freshnessChecks(summary), ...projectSearchContractChecks(summary, report), ...externalAggregateContractChecks(externalAggregateFilepath, externalAggregate), + ...externalCandidateExplanationContractChecks(externalCandidateExplanationsFilepath, externalAggregate, externalCandidateExplanations), ]; const githubCheck = githubAuditCheck(summary, githubAudit); if (githubCheck) checks.push(githubCheck); @@ -650,17 +752,27 @@ export function buildVerifyDailyResult(date: string): VerifyDailyResult { const githubEnrichmentPath = githubAuditPath(date); const reportPath = dailyReportPath(date); const externalAggregateFilepath = externalAggregatePath(date); + const externalCandidateExplanationsFilepath = externalCandidateExplanationsPath(date); const summary = readJsonFile(runSummaryPath, null); const githubAudit = readJsonFile(githubEnrichmentPath, []); const report = readJsonFile(reportPath, null); const externalAggregate = readOptionalJson(externalAggregateFilepath); + const externalCandidateExplanations = readOptionalJson(externalCandidateExplanationsFilepath); if (!summary) { return missingSummaryResult(date, runSummaryPath, githubEnrichmentPath); } const normalizedSummary = normalizeSummaryDiagnostics(summary); - const checks = buildChecks(normalizedSummary, githubAudit, report, externalAggregateFilepath, externalAggregate); + const checks = buildChecks( + normalizedSummary, + githubAudit, + report, + externalAggregateFilepath, + externalAggregate, + externalCandidateExplanationsFilepath, + externalCandidateExplanations, + ); return { date, status: aggregateStatus(checks), diff --git a/src/action/runSummary.ts b/src/action/runSummary.ts index 8719739..8ea5d6d 100644 --- a/src/action/runSummary.ts +++ b/src/action/runSummary.ts @@ -571,9 +571,9 @@ export function buildDailyRunSummary( classificationsCount: number; llmDiagnostics?: LlmRunDiagnostics; githubStarDelta?: GitHubStarDeltaSummary; - observer?: { - status: ObserverStatus; - candidateCount: number; + observer?: { + status: ObserverStatus; + candidateCount: number; ecosystemCounts: Record; incubatingDirections?: DailyRunSummary["observer_incubating_directions"]; promotionCandidates?: DailyRunSummary["observer_promotion_candidates"]; @@ -605,12 +605,13 @@ export function buildDailyRunSummary( | "judge_source" | "matched_by" | "source_notes" - > - >; - }; - missionInventoryAudit?: { - rolling_30d_searchable_catalog_count: number; - rolling_30d_vertical_or_task_oriented_count: number; + > + >; + }; + externalDiscovery?: DailyRunSummary["external_discovery"]; + missionInventoryAudit?: { + rolling_30d_searchable_catalog_count: number; + rolling_30d_vertical_or_task_oriented_count: number; rolling_7d_qualified_non_head_count: number; rolling_30d_direction_qualified_counts: Record; }; @@ -667,18 +668,19 @@ export function buildDailyRunSummary( } : undefined; summary.completion_notes = completionNotes(counts, summarySources, quality, opts.dryRun); - if (opts.observer) { - summary.observer_status = { ecosystem_focus: opts.observer.status }; - summary.observer_candidate_count = opts.observer.candidateCount; + if (opts.observer) { + summary.observer_status = { ecosystem_focus: opts.observer.status }; + summary.observer_candidate_count = opts.observer.candidateCount; summary.observer_ecosystem_counts = opts.observer.ecosystemCounts; summary.observer_incubating_directions = opts.observer.incubatingDirections; summary.observer_promotion_candidates = opts.observer.promotionCandidates; - summary.candidate_catalog_additions = opts.observer.promotionCandidates; - summary.observer_top_candidates = opts.observer.topCandidates; - } - summary.mission_metrics = buildMissionMetrics(summary); - if (opts.missionInventoryAudit) { - summary.mission_metrics = { + summary.candidate_catalog_additions = opts.observer.promotionCandidates; + summary.observer_top_candidates = opts.observer.topCandidates; + } + summary.external_discovery = opts.externalDiscovery; + summary.mission_metrics = buildMissionMetrics(summary); + if (opts.missionInventoryAudit) { + summary.mission_metrics = { ...summary.mission_metrics, rolling_30d_searchable_catalog_count: opts.missionInventoryAudit.rolling_30d_searchable_catalog_count, rolling_30d_vertical_or_task_oriented_count: opts.missionInventoryAudit.rolling_30d_vertical_or_task_oriented_count, @@ -825,12 +827,27 @@ function renderObserverSummary(summary: DailyRunSummary): string[] { `- candidate_catalog_addition ${item.direction_key}: evidence=${item.evidence.join(" | ") || "none"}; unmet_gates=${item.unmet_gates.join(" | ") || "none"}`, ) : []), - ]; -} - -function renderMissionSummary(summary: DailyRunSummary): string[] { - const coverageAtlas = summary.coverage_atlas ?? []; - const gapLedger = summary.gap_ledger ?? []; + ]; +} + +function renderExternalDiscoverySummary(summary: DailyRunSummary): string[] { + const external = summary.external_discovery; + if (!external) return ["- external_discovery: unavailable"]; + const warnings = Array.isArray(external.warnings) ? external.warnings : []; + + return [ + `- aggregate_status: ${external.aggregate_status}${external.aggregate_status_reason ? ` (${external.aggregate_status_reason})` : ""}`, + `- accepted_events: ${external.accepted_event_count}; rejected_events=${external.rejected_event_count}; candidates=${external.observation_candidate_count}`, + `- evidence: project=${external.project_evidence_count}; direction=${external.direction_evidence_count}`, + `- explanation_status: ${external.explanation_status}${external.explanation_status_reason ? ` (${external.explanation_status_reason})` : ""}`, + `- explanations: eligible=${external.explanation_eligible_count}; attempted=${external.explanation_attempted_count}; enhanced=${external.explanation_enhanced_count}; fallback=${external.explanation_fallback_count}; rejected=${external.explanation_rejected_count}`, + ...(warnings.length > 0 ? warnings.slice(0, 8).map((warning) => `- warning: ${warning}`) : ["- warnings: none"]), + ]; +} + +function renderMissionSummary(summary: DailyRunSummary): string[] { + const coverageAtlas = summary.coverage_atlas ?? []; + const gapLedger = summary.gap_ledger ?? []; const missionMetrics = summary.mission_metrics; return [ @@ -898,13 +915,17 @@ export function renderDailyRunSummary(summary: DailyRunSummary): string { "", "## 数据源状态", "", - ...summary.source_status.map( - (source) => `- ${source.source}: ${statusLabel(source.status)} | enabled=${source.enabled} | items=${source.item_count} | projects=${source.distinct_projects}`, - ), - "", - "## Mission Discovery", - "", - ...renderMissionSummary(summary), + ...summary.source_status.map( + (source) => `- ${source.source}: ${statusLabel(source.status)} | enabled=${source.enabled} | items=${source.item_count} | projects=${source.distinct_projects}`, + ), + "", + "## AgentReach 外部发现", + "", + ...renderExternalDiscoverySummary(summary), + "", + "## Mission Discovery", + "", + ...renderMissionSummary(summary), "", "## Observer Status", "", diff --git a/src/cli.ts b/src/cli.ts index 3af87b9..fea8a2c 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -27,7 +27,8 @@ import { searchGithubRepositoriesForDirection } from "./signal/githubRepositoryS import { captureTrackedRepoStarSnapshots } from "./signal/githubTrackedStars.ts"; import { buildMissionInventoryAudit } from "./signal/missionInventoryAudit.ts"; import { runMissionDeepDiscovery } from "./signal/missionDeepDiscovery.ts"; -import { runMissionScoutDiscovery } from "./signal/missionScoutDiscovery.ts"; +import { runMissionScoutDiscovery } from "./signal/missionScoutDiscovery.ts"; +import { runDailyExternalDiscoveryIntegration } from "./externalDiscovery/dailyIntegration.ts"; import { ensureDataDirs, readJsonFile, @@ -76,11 +77,12 @@ interface CliOptions { project?: string; slug?: string; trendKey?: string; - sourceView?: "overview" | "projects" | "weekly"; - anchorDate?: string; - recordAgentMemory?: boolean; - inputPath?: string; -} + sourceView?: "overview" | "projects" | "weekly"; + anchorDate?: string; + recordAgentMemory?: boolean; + inputPath?: string; + externalDiscoveryInputPath?: string; +} type FlagHandler = (opts: CliOptions, argv: string[], index: number) => number; @@ -147,11 +149,15 @@ const FLAG_HANDLERS: Record = { if (argv[index + 1]) opts.anchorDate = argv[index + 1]; return index + 1; }, - "--input": (opts, argv, index) => { - if (argv[index + 1]) opts.inputPath = argv[index + 1]; - return index + 1; - }, -}; + "--input": (opts, argv, index) => { + if (argv[index + 1]) opts.inputPath = argv[index + 1]; + return index + 1; + }, + "--external-discovery-input": (opts, argv, index) => { + if (argv[index + 1]) opts.externalDiscoveryInputPath = argv[index + 1]; + return index + 1; + }, +}; function parseArgs(argv: string[]): { command: string; opts: CliOptions } { const command = argv[2] ?? "run-daily"; @@ -694,13 +700,33 @@ export async function runDaily(opts: CliOptions): Promise { recent_daily_reports: readRecentDailyReports(opts.date, 5), }), ); - await generateProjectLibraryEnhancements({ + const projectLibraryArtifact = await generateProjectLibraryEnhancements({ date: opts.date, scored, config, observerEntries: observer.artifact.entries, dryRun, }); + const externalDiscovery = await runDailyExternalDiscoveryIntegration({ + date: opts.date, + generatedAt, + config, + dryRun, + inputPath: opts.externalDiscoveryInputPath, + explicitInput: Boolean(opts.externalDiscoveryInputPath), + scoredProjects: scored, + projectLibraryArtifact, + }); + logger.info("external discovery integration completed", { + date: opts.date, + status: externalDiscovery.aggregate.status, + acceptedEvents: externalDiscovery.aggregate.accepted_event_count, + candidates: externalDiscovery.input_build.eligible_count, + explanationStatus: externalDiscovery.explanations.status, + enhancedExplanations: externalDiscovery.explanations.audit.enhanced_count, + fallbackExplanations: externalDiscovery.explanations.audit.fallback_count, + dryRun, + }); reportWithFreshness.llm_diagnostics = { enabled: reportWithFreshness.llm_diagnostics?.enabled ?? llmDiagnostics.enabled, provider: reportWithFreshness.llm_diagnostics?.provider ?? llmDiagnostics.provider, @@ -768,9 +794,30 @@ export async function runDaily(opts: CliOptions): Promise { matched_by: entry.matched_by, source_notes: entry.source_notes, })), - }, - missionInventoryAudit, - }); + }, + externalDiscovery: { + aggregate_status: externalDiscovery.aggregate.status, + aggregate_status_reason: externalDiscovery.aggregate.status_reason, + accepted_event_count: externalDiscovery.aggregate.accepted_event_count, + rejected_event_count: externalDiscovery.aggregate.rejected_event_count, + observation_candidate_count: externalDiscovery.aggregate.observation_candidates.length, + project_evidence_count: externalDiscovery.aggregate.project_evidence.length, + direction_evidence_count: externalDiscovery.aggregate.direction_evidence.length, + explanation_status: externalDiscovery.explanations.status, + explanation_status_reason: externalDiscovery.explanations.status_reason, + explanation_eligible_count: externalDiscovery.explanations.audit.eligible_count, + explanation_attempted_count: externalDiscovery.explanations.audit.attempted_count, + explanation_enhanced_count: externalDiscovery.explanations.audit.enhanced_count, + explanation_fallback_count: externalDiscovery.explanations.audit.fallback_count, + explanation_rejected_count: externalDiscovery.explanations.audit.rejected_count, + warning_count: externalDiscovery.aggregate.audit.warnings.length + externalDiscovery.explanations.audit.warnings.length, + warnings: [ + ...externalDiscovery.aggregate.audit.warnings.map((warning) => `${warning.reason_code}:${warning.reason_detail}`), + ...externalDiscovery.explanations.audit.warnings.map((warning) => `${warning.reason_code}:${warning.reason_detail}`), + ].slice(0, 20), + }, + missionInventoryAudit, + }); writeJsonFile(runSummaryJsonPath(opts.date), runSummary, dryRun); writeJsonFile(path.join("data", "reports", "latest.run-summary.json"), runSummary, dryRun); writeTextFile(runSummaryMarkdownPath(opts.date), renderDailyRunSummary(runSummary), dryRun); diff --git a/src/externalDiscovery/agentReachProvider.ts b/src/externalDiscovery/agentReachProvider.ts index 639c395..5207dda 100644 --- a/src/externalDiscovery/agentReachProvider.ts +++ b/src/externalDiscovery/agentReachProvider.ts @@ -5,6 +5,7 @@ import { externalEntityRegistryPath } from "./paths.ts"; import { stableSourceInputHash } from "./redaction.ts"; import type { AgentReachProviderReadResult, + ExternalCandidateExplanationTitleContext, ExternalPlatform, ExternalProviderStatus, ExternalRawEventKind, @@ -62,6 +63,7 @@ export function readAgentReachProviderArtifact(filepath: string, options: ReadOp const status = artifact.status; const platforms = artifact.platforms; const events: ExternalSignalEvent[] = []; + const titleContext: ExternalCandidateExplanationTitleContext[] = []; const rejectedEvents: AgentReachProviderReadResult["rejected_events"] = []; const registry = resolveEntityRegistry(options); const warnings: AgentReachProviderReadResult["warnings"] = [...registry.warnings]; @@ -71,6 +73,8 @@ export function readAgentReachProviderArtifact(filepath: string, options: ReadOp if (event.ok) { const enriched = enrichEventActor(event.value, registry); events.push(enriched.event); + const context = extractTitleContext(item, enriched.event); + if (context) titleContext.push(context); warnings.push(...enriched.warnings); } else { rejectedEvents.push(event.rejected); @@ -92,6 +96,7 @@ export function readAgentReachProviderArtifact(filepath: string, options: ReadOp rejected_events: rejectedEvents, warnings: uniqueWarnings(warnings), source_input_hash: stableSourceInputHash(raw), + title_context: uniqueTitleContext(titleContext), }; } @@ -231,6 +236,61 @@ function parseEvent(value: unknown): }; } +function extractTitleContext(value: unknown, event: ExternalSignalEvent): ExternalCandidateExplanationTitleContext | undefined { + if (!isRecord(value)) return undefined; + const target = isRecord(value.target) ? value.target : undefined; + const targetDisplayName = firstSafeShortText([target?.name, target?.display_name, value.target_display_name]); + const publicEvidenceTitle = firstSafeShortText([value.title, value.name, target?.title]); + const publicSourceTitle = firstSafeShortText([value.source_title, value.page_title]); + + if (!targetDisplayName && !publicEvidenceTitle && !publicSourceTitle) return undefined; + return { + event_id: event.event_id, + target_key: event.target_key, + target_display_name: targetDisplayName, + public_evidence_title: publicEvidenceTitle, + public_source_title: publicSourceTitle, + source_platform: event.platform, + }; +} + +function firstSafeShortText(values: unknown[]): string | undefined { + for (const value of values) { + const normalized = safeShortText(value); + if (normalized) return normalized; + } + return undefined; +} + +function safeShortText(value: unknown): string | undefined { + if (typeof value !== "string") return undefined; + const normalized = value.replace(/\s+/g, " ").trim(); + if (normalized.length === 0 || normalized.length > 160) return undefined; + if (/https?:\/\//i.test(normalized)) return undefined; + if (/\[[^\]]+\]\([^)]+\)/.test(normalized)) return undefined; + if (/(^|\s)@[\w.-]{2,}/.test(normalized)) return undefined; + if (/\b(cookie|session|oauth|bearer|token|api[_ -]?key|password)\b/i.test(normalized)) return undefined; + return normalized; +} + +function uniqueTitleContext(contexts: ExternalCandidateExplanationTitleContext[]): ExternalCandidateExplanationTitleContext[] { + const byKey = new Map(); + for (const context of contexts) { + byKey.set( + [ + context.event_id ?? "", + context.target_key, + context.source_platform, + context.target_display_name ?? "", + context.public_evidence_title ?? "", + context.public_source_title ?? "", + ].join("\u0000"), + context, + ); + } + return [...byKey.values()]; +} + function generatedEventId(input: { rawRef?: string; url?: string; observedAt?: string }): string | undefined { const seed = input.rawRef ?? input.url; if (!seed || !input.observedAt) return undefined; @@ -287,6 +347,7 @@ function emptyResult(status: ExternalProviderStatus, statusReason: string, sourc rejected_events: [], warnings: [], source_input_hash: sourceInputHash, + title_context: [], }; } diff --git a/src/externalDiscovery/dailyIntegration.ts b/src/externalDiscovery/dailyIntegration.ts new file mode 100644 index 0000000..0d316a1 --- /dev/null +++ b/src/externalDiscovery/dailyIntegration.ts @@ -0,0 +1,178 @@ +import { buildDailyExternalAggregate } from "./aggregate.ts"; +import { readAgentReachProviderArtifact } from "./agentReachProvider.ts"; +import { + externalAggregateLatestPath, + externalAggregatePath, + externalCandidateExplanationsLatestPath, + externalCandidateExplanationsPath, + externalRawInputPath, +} from "./paths.ts"; +import { assertPublicSafeAggregate, stableSourceInputHash } from "./redaction.ts"; +import { + buildCandidateExplanationInputs, + generateExternalCandidateExplanations, + type CandidateExplanationBuildResult, + type ExternalCandidateExplanationArtifact, +} from "./explanations.ts"; +import { assertPublicSafeCandidateExplanations } from "./explanationRedaction.ts"; +import { writeJsonFile } from "../storage/files.ts"; +import type { AppConfig } from "../config.ts"; +import type { ProjectLibraryEnhancementArtifact, ScoredProject } from "../types.ts"; +import type { DailyExternalAggregate } from "./types.ts"; + +export interface DailyExternalDiscoveryIntegrationResult { + aggregate: DailyExternalAggregate; + explanations: ExternalCandidateExplanationArtifact; + input_build: CandidateExplanationBuildResult; + paths: { + aggregate: string; + aggregate_latest: string; + explanations: string; + explanations_latest: string; + }; +} + +function emptyCandidateExplanationBuildResult(reasonCode: string, reasonDetail: string, hashSeed: string): CandidateExplanationBuildResult { + return { + inputs: [], + input_context_hash: stableSourceInputHash(hashSeed), + warnings: [{ reason_code: reasonCode, reason_detail: reasonDetail }], + eligible_count: 0, + }; +} + +function failedCandidateExplanations(args: { + aggregate: DailyExternalAggregate; + inputBuild: CandidateExplanationBuildResult; + generatedAt: string; + provider: string; + reasonCode: string; + reasonDetail: string; +}): ExternalCandidateExplanationArtifact { + return { + schema_version: "external-discovery.candidate-explanations.v1", + date: args.aggregate.date, + generated_at: args.generatedAt, + provider: args.provider, + explanation_policy_version: "candidate-explanations.v1", + aggregate_source_input_hash: args.aggregate.source_input_hash, + aggregate_generated_at: args.aggregate.generated_at, + input_context_hash: args.inputBuild.input_context_hash, + public_safe: true, + redaction_policy_version: "external-discovery-explanation-redaction.v1", + contains_raw_text: false, + contains_profile_urls: false, + status: "failed", + status_reason: args.reasonCode, + explanations: [], + audit: { + eligible_count: args.inputBuild.eligible_count, + attempted_count: 0, + accepted_count: 0, + enhanced_count: 0, + rejected_count: args.inputBuild.inputs.length, + fallback_count: 0, + warnings: [ + ...args.inputBuild.warnings, + { + reason_code: args.reasonCode, + reason_detail: args.reasonDetail, + }, + ], + }, + }; +} + +function safeErrorDetail(error: unknown): string { + return error instanceof Error ? error.name : typeof error; +} + +export async function runDailyExternalDiscoveryIntegration(args: { + date: string; + generatedAt: string; + config: AppConfig; + dryRun?: boolean; + inputPath?: string; + explicitInput?: boolean; + scoredProjects?: ScoredProject[]; + projectLibraryArtifact?: ProjectLibraryEnhancementArtifact | null; +}): Promise { + const inputPath = args.inputPath ?? externalRawInputPath(args.date); + const providerResult = readAgentReachProviderArtifact(inputPath, { + explicitInput: args.explicitInput ?? Boolean(args.inputPath), + }); + const aggregate = buildDailyExternalAggregate({ + date: args.date, + generated_at: args.generatedAt, + provider_result: providerResult, + source_input_hash: providerResult.source_input_hash || stableSourceInputHash(`${providerResult.status}:${providerResult.status_reason ?? ""}:${inputPath}`), + }); + + const aggregateSafety = assertPublicSafeAggregate(aggregate); + if (!aggregateSafety.ok) { + throw new Error(`external aggregate is not public-safe: ${aggregateSafety.reason_codes.join(",")}`); + } + + const aggregatePath = externalAggregatePath(args.date); + const aggregateLatest = externalAggregateLatestPath(); + writeJsonFile(aggregatePath, aggregate, args.dryRun); + writeJsonFile(aggregateLatest, aggregate, args.dryRun); + + let inputBuild = emptyCandidateExplanationBuildResult( + "candidate_explanation_not_started", + "candidate explanation build has not started", + `${aggregate.source_input_hash}:candidate-explanation-not-started`, + ); + let explanations: ExternalCandidateExplanationArtifact; + try { + inputBuild = buildCandidateExplanationInputs({ + aggregate, + titleContext: providerResult.title_context, + scoredProjects: args.scoredProjects ?? [], + projectLibraryArtifact: args.projectLibraryArtifact ?? null, + }); + const generated = await generateExternalCandidateExplanations({ + aggregate, + inputBuild, + config: args.config, + generatedAt: args.generatedAt, + }); + const explanationSafety = assertPublicSafeCandidateExplanations(generated); + explanations = explanationSafety.ok + ? generated + : failedCandidateExplanations({ + aggregate, + inputBuild, + generatedAt: args.generatedAt, + provider: generated.provider, + reasonCode: "candidate_explanation_public_safe_failed", + reasonDetail: explanationSafety.reason_codes.join(","), + }); + } catch (error) { + explanations = failedCandidateExplanations({ + aggregate, + inputBuild, + generatedAt: args.generatedAt, + provider: args.config.llm?.provider ?? "unknown", + reasonCode: "candidate_explanation_generation_failed", + reasonDetail: safeErrorDetail(error), + }); + } + + const explanationsPath = externalCandidateExplanationsPath(args.date); + const explanationsLatest = externalCandidateExplanationsLatestPath(); + writeJsonFile(explanationsPath, explanations, args.dryRun); + writeJsonFile(explanationsLatest, explanations, args.dryRun); + + return { + aggregate, + explanations, + input_build: inputBuild, + paths: { + aggregate: aggregatePath, + aggregate_latest: aggregateLatest, + explanations: explanationsPath, + explanations_latest: explanationsLatest, + }, + }; +} diff --git a/src/externalDiscovery/explanationPrompts.ts b/src/externalDiscovery/explanationPrompts.ts new file mode 100644 index 0000000..53a24a4 --- /dev/null +++ b/src/externalDiscovery/explanationPrompts.ts @@ -0,0 +1,36 @@ +import type { CandidateExplanationInput } from "./explanations.ts"; + +export function buildCandidateExplanationPrompt(inputs: CandidateExplanationInput[]): string { + return [ + "你是 AgentRadar 的 AgentReach 外部发现解释层。", + "请只根据输入 JSON，把外部发现候选解释成普通用户能读懂的中文。", + "返回 exactly one JSON object and nothing else。", + "不要使用 markdown，不要输出 URL，不要访问互联网，不要补充输入中没有的功能、机构或趋势结论。", + "", + "输出 JSON schema:", + "{", + ' "explanations": [', + " {", + ' "candidate_key": string,', + ' "what_it_is_cn": string,', + ' "why_watch_cn": string,', + ' "summary_confidence": "high" | "medium" | "low",', + ' "caveats": string[]', + " }", + " ]", + "}", + "", + "硬性规则:", + "- what_it_is_cn 回答“它是什么”，优先使用 existing_project_brief_cn、repo_description、public_evidence_titles、public_source_titles。", + "- why_watch_cn 回答“为什么今天值得看”，只能表达外部层原因，例如平台讨论、跨平台出现、具名讨论者、可作为日报/周报次级证据。", + "- direction_signal 必须写成“方向线索”，不得写成明确 GitHub 项目。", + "- bound_object / external_evidence_boost / candidate_kind=project 必须写成“项目候选”或“已有候选获得外部补证”，不得写成“方向线索”或“尚未绑定明确 GitHub 项目”。", + "- 证据不足时必须写“仍需 GitHub / Trendshift 主链路确认”。", + "- 不得出现“主榜结论”“高置信推荐”“确定爆发”“已经形成趋势”等过度结论。", + "- 没有 description 或 brief 时，不得编造具体功能；只能说明当前可确认对象存在或作为方向线索。", + "- 每个中文字段建议 30-140 个中文字符，不同候选不要复用同一句话。", + "", + "输入 JSON:", + JSON.stringify({ candidates: inputs }, null, 2), + ].join("\n"); +} diff --git a/src/externalDiscovery/explanationRedaction.ts b/src/externalDiscovery/explanationRedaction.ts new file mode 100644 index 0000000..e12b1b6 --- /dev/null +++ b/src/externalDiscovery/explanationRedaction.ts @@ -0,0 +1,121 @@ +import { containsForbiddenPublicArtifactText, type RedactionCheckResult } from "./redaction.ts"; +import type { CandidateExplanationInput, ExternalCandidateExplanationArtifact } from "./explanations.ts"; + +const markdownLinkPattern = /\[[^\]]+\]\([^)]+\)/; +const rawHandlePattern = /(^|\s)@[\w.-]{2,}/; +const urlPattern = /https?:\/\//i; +const profileUrlPattern = /https?:\/\/(?:www\.)?(?:twitter\.com|x\.com|reddit\.com|news\.ycombinator\.com)\/[^\s/]+/i; +const unsupportedFunctionPattern = /自动部署|自动生成完整应用|自动完成端到端|一键生成|自主修复|无需人工/i; +const overclaimPattern = /高置信推荐|确定爆发|已经形成趋势|趋势已成|必然成为|可以替代主链路/i; +const mainConclusionPattern = /主榜结论/i; +const mainConclusionNegationPattern = /不能作为主榜结论|不作为主榜结论|不得作为主榜结论|不是主榜结论/i; +const directionOnlySemanticsPattern = /方向线索|尚未绑定明确项目|尚未绑定明确 GitHub 项目|进入方向观察|作为方向观察/; + +export function assertPublicSafeCandidateExplanationInputs(inputs: CandidateExplanationInput[]): RedactionCheckResult { + const reasonCodes: string[] = []; + + if (!Array.isArray(inputs)) reasonCodes.push("input_not_array"); + if (containsForbiddenPublicArtifactText(inputs)) reasonCodes.push("sensitive_text_detected"); + + for (const input of inputs) { + for (const text of [ + input.display_name, + input.target_display_name, + input.existing_project_brief_cn, + input.repo_description, + ...input.public_evidence_titles, + ...input.public_source_titles, + ...input.evidence_reason_facts, + ...input.named_registry_actor_names, + ]) { + if (!text) continue; + reasonCodes.push(...textReasonCodes(text, { allowGithubUrl: false })); + } + + for (const title of [...input.public_evidence_titles, ...input.public_source_titles]) { + if (title.length > 160) reasonCodes.push("long_public_title_detected"); + } + + for (const url of [input.target_url, input.repo_url]) { + if (!url) continue; + if (profileUrlPattern.test(url)) reasonCodes.push("profile_url_detected"); + if (markdownLinkPattern.test(url)) reasonCodes.push("markdown_link_detected"); + } + } + + return result(reasonCodes); +} + +export function assertPublicSafeCandidateExplanations(artifact: ExternalCandidateExplanationArtifact): RedactionCheckResult { + const reasonCodes: string[] = []; + + if (!isRecord(artifact)) { + reasonCodes.push("artifact_not_object"); + return result(reasonCodes); + } + if (artifact.public_safe !== true) reasonCodes.push("public_safe_not_true"); + if (artifact.contains_raw_text !== false) reasonCodes.push("contains_raw_text_not_false"); + if (artifact.contains_profile_urls !== false) reasonCodes.push("contains_profile_urls_not_false"); + if (containsForbiddenPublicArtifactText(artifact)) reasonCodes.push("sensitive_text_detected"); + + const seenEnhancedSummaries = new Set(); + for (const explanation of artifact.explanations ?? []) { + for (const text of [explanation.what_it_is_cn, explanation.why_watch_cn, ...explanation.caveats]) { + reasonCodes.push(...textReasonCodes(text, { allowGithubUrl: false })); + if (unsupportedFunctionPattern.test(text)) reasonCodes.push("unsupported_function_claim"); + if (overclaimPattern.test(text) || (mainConclusionPattern.test(text) && !mainConclusionNegationPattern.test(text))) { + reasonCodes.push("overclaim_detected"); + } + } + + if ( + (explanation.explanation_scope === "direction_signal" || explanation.candidate_kind === "direction") && + !/(方向|线索|尚未绑定明确项目|尚未绑定明确 GitHub 项目)/.test(`${explanation.what_it_is_cn}${explanation.why_watch_cn}`) + ) { + reasonCodes.push("direction_semantics_missing"); + } + if ( + explanation.explanation_scope !== "direction_signal" && + explanation.candidate_kind !== "direction" && + directionOnlySemanticsPattern.test(`${explanation.what_it_is_cn}${explanation.why_watch_cn}`) + ) { + reasonCodes.push("project_semantics_conflict"); + } + + if (explanation.summary_source !== "rules_fallback") { + const summaryKey = normalizeSummary(`${explanation.what_it_is_cn}${explanation.why_watch_cn}`); + if (seenEnhancedSummaries.has(summaryKey)) { + reasonCodes.push("duplicate_summary_detected"); + } + seenEnhancedSummaries.add(summaryKey); + } + } + + return result(reasonCodes); +} + +function textReasonCodes(text: string, options: { allowGithubUrl: boolean }): string[] { + const reasonCodes: string[] = []; + if (!options.allowGithubUrl && urlPattern.test(text)) reasonCodes.push("url_detected"); + if (profileUrlPattern.test(text)) reasonCodes.push("profile_url_detected"); + if (rawHandlePattern.test(text)) reasonCodes.push("raw_handle_detected"); + if (markdownLinkPattern.test(text)) reasonCodes.push("markdown_link_detected"); + if (/\b(cookie|session|oauth|bearer|token|api[_ -]?key|password)\b/i.test(text)) reasonCodes.push("sensitive_text_detected"); + if (text.length > 600) reasonCodes.push("long_raw_text_detected"); + return reasonCodes; +} + +function normalizeSummary(value: string): string { + return value.replace(/\s+/g, "").replace(/[，。、“”"'；：:,.!?！？()（）\-\s]/g, "").toLowerCase(); +} + +function result(reasonCodes: string[]): RedactionCheckResult { + return { + ok: reasonCodes.length === 0, + reason_codes: [...new Set(reasonCodes)].sort(), + }; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/src/externalDiscovery/explanations.ts b/src/externalDiscovery/explanations.ts new file mode 100644 index 0000000..7dfcda9 --- /dev/null +++ b/src/externalDiscovery/explanations.ts @@ -0,0 +1,722 @@ +import type { AppConfig } from "../config.ts"; +import type { ProjectLibraryEnhancementArtifact, ScoredProject } from "../types.ts"; +import { callStructuredEnhancement, isEnhancementEnabled } from "../action/enhancementLlm.ts"; +import { stableSourceInputHash } from "./redaction.ts"; +import { buildCandidateExplanationPrompt } from "./explanationPrompts.ts"; +import { assertPublicSafeCandidateExplanations, assertPublicSafeCandidateExplanationInputs } from "./explanationRedaction.ts"; +import type { + DailyExternalAggregate, + ExternalCandidateExplanationTitleContext, + ExternalEvidence, + ExternalPlatform, + ObservationCandidate, +} from "./types.ts"; + +export type ExternalCandidateExplanationStatus = "ok" | "partial" | "skipped" | "failed"; +export type ExternalCandidateExplanationScope = "bound_object" | "direction_signal" | "external_evidence_boost"; +export type ExternalCandidateExplanationConfidence = "high" | "medium" | "low"; +export type ExternalCandidateExplanationSource = "llm" | "rules_fallback" | "existing_project_brief"; + +export interface CandidateExplanationInput { + candidate_key: string; + candidate_kind: ObservationCandidate["candidate_kind"]; + target_key: string; + display_name: string; + explanation_scope: ExternalCandidateExplanationScope; + target_url?: string; + repo_url?: string; + target_display_name?: string; + existing_project_brief_cn?: string; + repo_description?: string; + public_evidence_titles: string[]; + public_source_titles: string[]; + evidence_reason_facts: string[]; + evidence_ids: string[]; + platforms: ExternalPlatform[]; + mention_count: number; + distinct_actor_count: number; + top_tier_actor_count: number; + named_registry_actor_names: string[]; + can_enter_daily: boolean; + can_enter_weekly: boolean; + cannot_be_primary_conclusion: true; + input_confidence: ExternalCandidateExplanationConfidence; + input_warnings: string[]; +} + +export interface CandidateExplanationBuildResult { + inputs: CandidateExplanationInput[]; + input_context_hash: string; + warnings: Array<{ reason_code: string; reason_detail: string }>; + eligible_count: number; +} + +export interface ExternalCandidateExplanation { + candidate_key: string; + candidate_kind: ObservationCandidate["candidate_kind"]; + target_key: string; + explanation_scope: ExternalCandidateExplanationScope; + what_it_is_cn: string; + why_watch_cn: string; + summary_confidence: ExternalCandidateExplanationConfidence; + summary_source: ExternalCandidateExplanationSource; + evidence_ids: string[]; + platforms: ExternalPlatform[]; + caveats: string[]; + generated_at: string; +} + +export interface ExternalCandidateExplanationArtifact { + schema_version: "external-discovery.candidate-explanations.v1"; + date: string; + generated_at: string; + provider: string; + explanation_policy_version: "candidate-explanations.v1"; + aggregate_source_input_hash: string; + aggregate_generated_at: string; + input_context_hash: string; + public_safe: true; + redaction_policy_version: string; + contains_raw_text: false; + contains_profile_urls: false; + status: ExternalCandidateExplanationStatus; + status_reason?: string; + explanations: ExternalCandidateExplanation[]; + audit: { + eligible_count: number; + attempted_count: number; + accepted_count: number; + enhanced_count: number; + rejected_count: number; + fallback_count: number; + warnings: Array<{ reason_code: string; reason_detail: string }>; + }; +} + +type BuildInputsArgs = { + aggregate: DailyExternalAggregate; + titleContext?: ExternalCandidateExplanationTitleContext[]; + scoredProjects?: ScoredProject[]; + projectLibraryArtifact?: ProjectLibraryEnhancementArtifact | null; + topN?: number; +}; + +type GenerateArgs = { + aggregate: DailyExternalAggregate; + inputBuild: CandidateExplanationBuildResult; + config: AppConfig; + generatedAt: string; +}; + +type LlmExplanationDraft = { + candidate_key: string; + what_it_is_cn?: string; + why_watch_cn?: string; + summary_confidence?: ExternalCandidateExplanationConfidence; + caveats?: string[]; +}; + +const POLICY_VERSION = "candidate-explanations.v1"; +const DEFAULT_TOP_N = 30; +const ENHANCED_COVERAGE_THRESHOLD = 0.7; + +const platformNames: Record = { + x_twitter: "X", + reddit: "Reddit", + hacker_news: "HN", + official_web: "官方网页", + official_blog: "官方博客", +}; + +export function buildCandidateExplanationInputs(args: BuildInputsArgs): CandidateExplanationBuildResult { + const evidenceByTarget = buildEvidenceLookup(args.aggregate); + const candidates = readCandidateBase(args.aggregate); + const titleByTarget = groupTitleContext(args.titleContext ?? []); + const projectBriefs = buildProjectBriefLookup(args.projectLibraryArtifact); + const scoredProjects = buildScoredProjectLookup(args.scoredProjects ?? []); + const warnings: Array<{ reason_code: string; reason_detail: string }> = []; + + const inputs = candidates.map((candidate) => { + const evidenceRows = evidenceByTarget.get(candidate.target_key) ?? []; + const titleRows = titleByTarget.get(candidate.target_key) ?? []; + const evidenceIds = unique(evidenceRows.map((evidence) => evidence.evidence_id)).sort(); + const platforms = unique(evidenceRows.flatMap((evidence) => evidence.platforms)).sort() as ExternalPlatform[]; + const scored = findScoredProject(scoredProjects, candidate.target_key); + const projectBrief = findProjectBrief(projectBriefs, candidate.target_key, scored?.project.repo_full_name); + const repoUrl = scored?.project.repo_url ?? repoUrlFromTargetKey(candidate.target_key); + const repoDescription = scored?.project.description?.trim() || undefined; + const targetUrl = candidate.target_key.startsWith("http") ? candidate.target_key : repoUrl; + const publicEvidenceTitles = unique(titleRows.map((item) => item.public_evidence_title).filter(isNonEmptyString)).slice(0, 5); + const publicSourceTitles = unique(titleRows.map((item) => item.public_source_title).filter(isNonEmptyString)).slice(0, 5); + const targetDisplayName = firstNonEmpty([ + ...titleRows.map((item) => item.target_display_name), + scored?.project.project_name, + displayNameFromTargetKey(candidate.target_key), + ]); + const explanationScope = explanationScopeFor(candidate, evidenceRows, Boolean(repoUrl || projectBrief)); + const inputWarnings = inputWarningsFor({ + projectBrief, + repoDescription, + publicEvidenceTitles, + publicSourceTitles, + explanationScope, + }); + + for (const reasonCode of inputWarnings) { + warnings.push({ + reason_code: reasonCode, + reason_detail: `${candidate.candidate_kind}:${candidate.target_key}`, + }); + } + + return { + candidate_key: `${candidate.candidate_kind}:${candidate.target_key}`, + candidate_kind: candidate.candidate_kind, + target_key: candidate.target_key, + display_name: targetDisplayName ?? candidate.target_key, + explanation_scope: explanationScope, + target_url: targetUrl, + repo_url: repoUrl, + target_display_name: targetDisplayName, + existing_project_brief_cn: projectBrief, + repo_description: repoDescription, + public_evidence_titles: publicEvidenceTitles, + public_source_titles: publicSourceTitles, + evidence_reason_facts: evidenceReasonFacts(candidate, evidenceRows), + evidence_ids: evidenceIds, + platforms, + mention_count: sum(evidenceRows.map((evidence) => evidence.mention_count)), + distinct_actor_count: sum(evidenceRows.map((evidence) => evidence.distinct_actor_count)), + top_tier_actor_count: sum(evidenceRows.map((evidence) => evidence.top_tier_actor_count)), + named_registry_actor_names: unique( + evidenceRows.flatMap((evidence) => evidence.named_registry_actors.map((actor) => actor.display_name)), + ).slice(0, 8), + can_enter_daily: candidate.can_enter_daily, + can_enter_weekly: candidate.can_enter_weekly, + cannot_be_primary_conclusion: true as const, + input_confidence: inputConfidence({ + projectBrief, + repoDescription, + repoUrl, + publicEvidenceTitles, + publicSourceTitles, + explanationScope, + evidenceRows, + }), + input_warnings: inputWarnings, + }; + }); + + const sortedInputs = sortInputs(inputs).slice(0, Math.max(1, args.topN ?? DEFAULT_TOP_N)); + const redaction = assertPublicSafeCandidateExplanationInputs(sortedInputs); + if (!redaction.ok) { + warnings.push({ + reason_code: "public_safe_validation_failed", + reason_detail: redaction.reason_codes.join(","), + }); + } + + return { + inputs: sortedInputs, + input_context_hash: stableSourceInputHash(JSON.stringify(sortedInputs)), + warnings: uniqueWarnings(warnings), + eligible_count: sortedInputs.length, + }; +} + +export async function generateExternalCandidateExplanations(args: GenerateArgs): Promise { + const enabled = isEnhancementEnabled(args.config); + const provider = enabled ? args.config.llm.provider : "rules"; + const inputs = args.inputBuild.inputs; + const warnings = [...args.inputBuild.warnings]; + const inputSafety = assertPublicSafeCandidateExplanationInputs(inputs); + if (!inputSafety.ok) { + return artifact(args, { + provider, + status: "failed", + statusReason: "public_safe_validation_failed", + explanations: [], + attemptedCount: 0, + rejectedCount: inputs.length, + warnings: [ + ...warnings, + { + reason_code: "public_safe_validation_failed", + reason_detail: inputSafety.reason_codes.join(","), + }, + ], + }); + } + + if (inputs.length === 0) { + return artifact(args, { + provider, + status: "skipped", + statusReason: "no_candidates", + explanations: [], + attemptedCount: 0, + rejectedCount: 0, + warnings, + }); + } + + const existing = inputs + .filter((input) => Boolean(input.existing_project_brief_cn)) + .map((input) => explanationFromExistingBrief(input, args.generatedAt)); + const existingKeys = new Set(existing.map((item) => item.candidate_key)); + const llmInputs = inputs.filter((input) => !existingKeys.has(input.candidate_key) && input.input_confidence !== "low"); + const fallbackInputs = new Map(inputs.filter((input) => !existingKeys.has(input.candidate_key)).map((input) => [input.candidate_key, input] as const)); + const llmExplanations: ExternalCandidateExplanation[] = []; + let attemptedCount = 0; + let rejectedCount = 0; + + if (!enabled) { + warnings.push({ reason_code: "llm_disabled", reason_detail: "LLM is disabled; using rules fallback explanations" }); + } else if (llmInputs.length === 0) { + warnings.push({ reason_code: "input_context_insufficient", reason_detail: "No candidate had enough context for LLM explanation" }); + } else { + attemptedCount = llmInputs.length; + const raw = await callStructuredEnhancement(buildCandidateExplanationPrompt(llmInputs), args.config, { + maxTokens: 5000, + }); + const drafts = buildDraftMap(raw); + + for (const input of llmInputs) { + const draft = drafts.get(input.candidate_key); + if (!draft) { + rejectedCount += 1; + warnings.push({ reason_code: "summary_generation_failed", reason_detail: input.candidate_key }); + continue; + } + const accepted = explanationFromDraft(input, draft, args.generatedAt); + if (!accepted) { + rejectedCount += 1; + warnings.push({ reason_code: "summary_validation_failed", reason_detail: input.candidate_key }); + continue; + } + llmExplanations.push(accepted); + fallbackInputs.delete(input.candidate_key); + } + } + + const fallback = [...fallbackInputs.values()].map((input) => rulesFallbackExplanation(input, args.generatedAt)); + const explanations = [...existing, ...llmExplanations, ...fallback].sort((left, right) => + left.candidate_key.localeCompare(right.candidate_key), + ); + const enhancedCount = explanations.filter((item) => item.summary_source === "llm" || item.summary_source === "existing_project_brief").length; + const status = explanationStatus({ + enabled, + candidateCount: inputs.length, + enhancedCount, + fallbackCount: fallback.length, + }); + const statusReason = + status === "ok" + ? "candidate_explanation_ok" + : !enabled + ? "llm_disabled" + : enhancedCount === 0 + ? "summary_generation_failed" + : "partial_llm_failure"; + const candidateArtifact = artifact(args, { + provider, + status, + statusReason, + explanations, + attemptedCount, + rejectedCount, + warnings, + }); + const redaction = assertPublicSafeCandidateExplanations(candidateArtifact); + if (redaction.ok) return candidateArtifact; + + return artifact(args, { + provider, + status: "failed", + statusReason: "public_safe_validation_failed", + explanations: inputs.map((input) => rulesFallbackExplanation(input, args.generatedAt)), + attemptedCount, + rejectedCount: inputs.length, + warnings: [ + ...warnings, + { + reason_code: "public_safe_validation_failed", + reason_detail: redaction.reason_codes.join(","), + }, + ], + }); +} + +function artifact( + args: GenerateArgs, + values: { + provider: string; + status: ExternalCandidateExplanationStatus; + statusReason: string; + explanations: ExternalCandidateExplanation[]; + attemptedCount: number; + rejectedCount: number; + warnings: Array<{ reason_code: string; reason_detail: string }>; + }, +): ExternalCandidateExplanationArtifact { + const enhancedCount = values.explanations.filter((item) => item.summary_source === "llm" || item.summary_source === "existing_project_brief").length; + const fallbackCount = values.explanations.filter((item) => item.summary_source === "rules_fallback").length; + return { + schema_version: "external-discovery.candidate-explanations.v1", + date: args.aggregate.date, + generated_at: args.generatedAt, + provider: values.provider, + explanation_policy_version: POLICY_VERSION, + aggregate_source_input_hash: args.aggregate.source_input_hash, + aggregate_generated_at: args.aggregate.generated_at, + input_context_hash: args.inputBuild.input_context_hash, + public_safe: true, + redaction_policy_version: "external-discovery-explanation-redaction.v1", + contains_raw_text: false, + contains_profile_urls: false, + status: values.status, + status_reason: values.statusReason, + explanations: values.explanations, + audit: { + eligible_count: args.inputBuild.eligible_count, + attempted_count: values.attemptedCount, + accepted_count: values.explanations.length, + enhanced_count: enhancedCount, + rejected_count: values.rejectedCount, + fallback_count: fallbackCount, + warnings: uniqueWarnings(values.warnings), + }, + }; +} + +function explanationStatus(args: { + enabled: boolean; + candidateCount: number; + enhancedCount: number; + fallbackCount: number; +}): ExternalCandidateExplanationStatus { + if (!args.enabled) return "skipped"; + if (args.candidateCount === 0) return "skipped"; + if (args.enhancedCount === 0) return "failed"; + return args.enhancedCount / args.candidateCount >= ENHANCED_COVERAGE_THRESHOLD ? "ok" : "partial"; +} + +function explanationFromExistingBrief(input: CandidateExplanationInput, generatedAt: string): ExternalCandidateExplanation { + return { + candidate_key: input.candidate_key, + candidate_kind: input.candidate_kind, + target_key: input.target_key, + explanation_scope: input.explanation_scope, + what_it_is_cn: input.existing_project_brief_cn!, + why_watch_cn: whyWatchFallback(input), + summary_confidence: "high", + summary_source: "existing_project_brief", + evidence_ids: input.evidence_ids, + platforms: input.platforms, + caveats: caveatsFor(input), + generated_at: generatedAt, + }; +} + +function explanationFromDraft( + input: CandidateExplanationInput, + draft: LlmExplanationDraft, + generatedAt: string, +): ExternalCandidateExplanation | null { + const what = draft.what_it_is_cn?.trim(); + const why = draft.why_watch_cn?.trim(); + if (!what || !why) return null; + if (input.explanation_scope === "direction_signal" && !/(方向|线索|尚未绑定明确项目|尚未绑定明确 GitHub 项目)/.test(`${what}${why}`)) { + return null; + } + if (input.explanation_scope !== "direction_signal" && input.candidate_kind !== "direction" && hasDirectionOnlySemantics(`${what}${why}`)) { + return null; + } + return { + candidate_key: input.candidate_key, + candidate_kind: input.candidate_kind, + target_key: input.target_key, + explanation_scope: input.explanation_scope, + what_it_is_cn: what, + why_watch_cn: why, + summary_confidence: draft.summary_confidence ?? input.input_confidence, + summary_source: "llm", + evidence_ids: input.evidence_ids, + platforms: input.platforms, + caveats: normalizeCaveats(draft.caveats, input), + generated_at: generatedAt, + }; +} + +function hasDirectionOnlySemantics(value: string): boolean { + return /方向线索|尚未绑定明确项目|尚未绑定明确 GitHub 项目|进入方向观察|作为方向观察/.test(value); +} + +function rulesFallbackExplanation(input: CandidateExplanationInput, generatedAt: string): ExternalCandidateExplanation { + return { + candidate_key: input.candidate_key, + candidate_kind: input.candidate_kind, + target_key: input.target_key, + explanation_scope: input.explanation_scope, + what_it_is_cn: whatItIsFallback(input), + why_watch_cn: whyWatchFallback(input), + summary_confidence: input.input_confidence === "high" ? "medium" : input.input_confidence, + summary_source: "rules_fallback", + evidence_ids: input.evidence_ids, + platforms: input.platforms, + caveats: caveatsFor(input), + generated_at: generatedAt, + }; +} + +function whatItIsFallback(input: CandidateExplanationInput): string { + if (input.explanation_scope === "direction_signal" || input.candidate_kind === "direction") { + return `${input.display_name} 是今天外部讨论中出现的方向线索，尚未绑定明确 GitHub 项目。`; + } + if (input.explanation_scope === "external_evidence_boost") { + return `${input.display_name} 是已有候选获得外部补证的对象，目前只确认它在外部来源中再次出现。`; + } + if (input.repo_description) { + return `${input.display_name} 是已绑定 repo 的项目候选，已有描述显示：${trimSentence(input.repo_description)}。`; + } + return `${input.display_name} 是已绑定对象的外部发现候选，当前仍需主链路补充具体功能说明。`; +} + +function whyWatchFallback(input: CandidateExplanationInput): string { + const platformText = input.platforms.length > 0 ? input.platforms.map((platform) => platformNames[platform]).join("、") : "外部来源"; + const actorText = + input.named_registry_actor_names.length > 0 + ? `，并命中 ${input.named_registry_actor_names.slice(0, 3).join("、")} 等公开讨论者` + : ""; + const entryText = input.can_enter_daily || input.can_enter_weekly ? "，可作为日报或周报的次级证据" : ""; + if (input.explanation_scope === "direction_signal") { + return `${platformText} 出现相关讨论${actorText}，适合先进入方向观察${entryText}，但不能替代主链路判断。`; + } + return `${platformText} 出现该对象的外部讨论或官方信号${actorText}${entryText}，适合继续查看证据来源。`; +} + +function caveatsFor(input: CandidateExplanationInput): string[] { + const caveats = ["外部层不能作为主榜结论，仍需 GitHub / Trendshift 主链路确认。"]; + if (input.explanation_scope === "direction_signal") caveats.push("该候选尚未绑定明确 GitHub 项目。"); + if (!input.repo_description && !input.existing_project_brief_cn) caveats.push("当前缺少稳定项目简介，具体用途不做推断。"); + return unique(caveats); +} + +function normalizeCaveats(caveats: unknown, input: CandidateExplanationInput): string[] { + const fromDraft = Array.isArray(caveats) + ? caveats.map((item) => (typeof item === "string" ? item.trim() : "")).filter(Boolean) + : []; + return unique([...fromDraft, ...caveatsFor(input)]).slice(0, 5); +} + +function buildDraftMap(raw: unknown): Map { + if (!raw || typeof raw !== "object") return new Map(); + const record = raw as Record; + const explanations = Array.isArray(record.explanations) ? record.explanations : []; + const map = new Map(); + for (const item of explanations) { + if (!item || typeof item !== "object") continue; + const draft = item as Record; + const candidateKey = typeof draft.candidate_key === "string" ? draft.candidate_key.trim() : ""; + if (!candidateKey) continue; + map.set(candidateKey, { + candidate_key: candidateKey, + what_it_is_cn: typeof draft.what_it_is_cn === "string" ? draft.what_it_is_cn.trim() : undefined, + why_watch_cn: typeof draft.why_watch_cn === "string" ? draft.why_watch_cn.trim() : undefined, + summary_confidence: isConfidence(draft.summary_confidence) ? draft.summary_confidence : undefined, + caveats: Array.isArray(draft.caveats) ? draft.caveats.filter((entry): entry is string => typeof entry === "string") : undefined, + }); + } + return map; +} + +function readCandidateBase(aggregate: DailyExternalAggregate): ObservationCandidate[] { + if (aggregate.observation_candidates.length > 0) return aggregate.observation_candidates; + const fromProject = aggregate.project_evidence.map((evidence): ObservationCandidate => ({ + candidate_kind: "project", + target_key: evidence.target_key, + qualification: "needs_primary_confirmation", + can_enter_daily: true, + can_enter_weekly: evidence.platforms.length > 1 || evidence.mention_count >= 2, + cannot_be_primary_conclusion: true, + })); + const fromDirection = aggregate.direction_evidence.map((evidence): ObservationCandidate => ({ + candidate_kind: "direction", + target_key: evidence.target_key, + qualification: "direction_observation", + can_enter_daily: true, + can_enter_weekly: evidence.platforms.length > 1 || evidence.mention_count >= 3, + cannot_be_primary_conclusion: true, + })); + return uniqueCandidates([...fromProject, ...fromDirection]); +} + +function buildEvidenceLookup(aggregate: DailyExternalAggregate): Map { + const map = new Map(); + for (const evidence of [...aggregate.project_evidence, ...aggregate.direction_evidence]) { + map.set(evidence.target_key, [...(map.get(evidence.target_key) ?? []), evidence]); + } + return map; +} + +function groupTitleContext(context: ExternalCandidateExplanationTitleContext[]): Map { + const map = new Map(); + for (const item of context) { + map.set(item.target_key, [...(map.get(item.target_key) ?? []), item]); + } + return map; +} + +function buildProjectBriefLookup(artifact: ProjectLibraryEnhancementArtifact | null | undefined): Map { + const map = new Map(); + for (const entry of artifact?.entries ?? []) { + map.set(entry.repo_full_name.toLowerCase(), entry.project_brief_cn.trim()); + } + return map; +} + +function buildScoredProjectLookup(projects: ScoredProject[]): Map { + const map = new Map(); + for (const project of projects) { + map.set(project.project.repo_full_name.toLowerCase(), project); + map.set(project.project.repo_url.toLowerCase(), project); + } + return map; +} + +function findScoredProject(projects: Map, targetKey: string): ScoredProject | undefined { + return projects.get(targetKey.toLowerCase()) ?? projects.get(normalizedRepoKey(targetKey)); +} + +function findProjectBrief(briefs: Map, targetKey: string, repoFullName?: string): string | undefined { + return briefs.get((repoFullName ?? "").toLowerCase()) ?? briefs.get(normalizedRepoKey(targetKey)); +} + +function normalizedRepoKey(value: string): string { + const repo = /^https?:\/\/github\.com\/([^/\s]+\/[^/\s#?]+)/i.exec(value.trim())?.[1] ?? value.trim(); + return repo.replace(/\.git$/i, "").toLowerCase(); +} + +function repoUrlFromTargetKey(value: string): string | undefined { + if (/^https?:\/\/github\.com\/[^/\s]+\/[^/\s#?]+/i.test(value)) return value; + return /^[\w.-]+\/[\w.-]+$/.test(value) ? `https://github.com/${value}` : undefined; +} + +function displayNameFromTargetKey(value: string): string { + if (/^https?:\/\/github\.com\//i.test(value)) return normalizedRepoKey(value); + if (/^https?:\/\//i.test(value)) return "外部对象"; + return value; +} + +function explanationScopeFor( + candidate: ObservationCandidate, + evidenceRows: ExternalEvidence[], + hasBoundObjectContext: boolean, +): ExternalCandidateExplanationScope { + if (candidate.candidate_kind === "direction" || candidate.qualification === "direction_observation") return "direction_signal"; + if (evidenceRows.some((evidence) => evidence.derived_signal_kinds.includes("evidence"))) return "external_evidence_boost"; + return hasBoundObjectContext ? "bound_object" : "direction_signal"; +} + +function inputConfidence(args: { + projectBrief?: string; + repoDescription?: string; + repoUrl?: string; + publicEvidenceTitles: string[]; + publicSourceTitles: string[]; + explanationScope: ExternalCandidateExplanationScope; + evidenceRows: ExternalEvidence[]; +}): ExternalCandidateExplanationConfidence { + if (args.projectBrief) return "high"; + if (args.explanationScope === "direction_signal") return args.publicEvidenceTitles.length + args.publicSourceTitles.length > 0 ? "medium" : "low"; + if (args.repoDescription && args.repoUrl && args.evidenceRows.length > 0) return "high"; + if (args.repoUrl || args.publicEvidenceTitles.length > 0 || args.publicSourceTitles.length > 0) return "medium"; + return "low"; +} + +function inputWarningsFor(args: { + projectBrief?: string; + repoDescription?: string; + publicEvidenceTitles: string[]; + publicSourceTitles: string[]; + explanationScope: ExternalCandidateExplanationScope; +}): string[] { + const warnings: string[] = []; + if (!args.projectBrief && !args.repoDescription && args.publicEvidenceTitles.length === 0 && args.publicSourceTitles.length === 0) { + warnings.push("public_title_context_missing"); + } + if (args.explanationScope === "direction_signal") warnings.push("direction_candidate_low_confidence"); + return warnings; +} + +function evidenceReasonFacts(candidate: ObservationCandidate, evidenceRows: ExternalEvidence[]): string[] { + const platforms = unique(evidenceRows.flatMap((evidence) => evidence.platforms)); + const namedActors = unique(evidenceRows.flatMap((evidence) => evidence.named_registry_actors.map((actor) => actor.display_name))); + const facts = [ + platforms.length > 0 ? `来源平台：${platforms.map((platform) => platformNames[platform]).join("、")}` : "来源平台：unknown", + platforms.length > 1 ? "跨平台出现" : undefined, + namedActors.length > 0 ? `具名讨论者：${namedActors.slice(0, 5).join("、")}` : undefined, + candidate.can_enter_daily ? "可进入日报作为次级证据" : undefined, + candidate.can_enter_weekly ? "可进入周报作为次级证据" : undefined, + candidate.cannot_be_primary_conclusion ? "仍需主源确认，不能作为主榜结论" : undefined, + ]; + return facts.filter(isNonEmptyString); +} + +function sortInputs(inputs: CandidateExplanationInput[]): CandidateExplanationInput[] { + const rank: Record = { + external_evidence_boost: 0, + bound_object: 1, + direction_signal: 2, + }; + return [...inputs].sort( + (left, right) => + rank[left.explanation_scope] - rank[right.explanation_scope] || + right.evidence_ids.length - left.evidence_ids.length || + right.platforms.length - left.platforms.length || + right.top_tier_actor_count - left.top_tier_actor_count || + right.mention_count - left.mention_count || + left.candidate_key.localeCompare(right.candidate_key), + ); +} + +function trimSentence(value: string): string { + const normalized = value.replace(/\s+/g, " ").trim(); + return normalized.length > 80 ? `${normalized.slice(0, 80)}...` : normalized; +} + +function firstNonEmpty(values: Array): string | undefined { + return values.find(isNonEmptyString); +} + +function isNonEmptyString(value: unknown): value is string { + return typeof value === "string" && value.trim().length > 0; +} + +function isConfidence(value: unknown): value is ExternalCandidateExplanationConfidence { + return value === "high" || value === "medium" || value === "low"; +} + +function sum(values: number[]): number { + return values.reduce((total, value) => total + value, 0); +} + +function unique(values: T[]): T[] { + return [...new Set(values)]; +} + +function uniqueCandidates(candidates: ObservationCandidate[]): ObservationCandidate[] { + const byKey = new Map(); + for (const candidate of candidates) { + byKey.set(`${candidate.candidate_kind}:${candidate.target_key}`, candidate); + } + return [...byKey.values()]; +} + +function uniqueWarnings(warnings: T[]): T[] { + const byKey = new Map(); + for (const warning of warnings) { + byKey.set(`${warning.reason_code}:${warning.reason_detail}`, warning); + } + return [...byKey.values()]; +} diff --git a/src/externalDiscovery/paths.ts b/src/externalDiscovery/paths.ts index bf992b7..97468e0 100644 --- a/src/externalDiscovery/paths.ts +++ b/src/externalDiscovery/paths.ts @@ -12,6 +12,14 @@ export function externalAggregateLatestPath(): string { return path.join("data", "external-discovery", "latest.aggregate.json"); } +export function externalCandidateExplanationsPath(date: string): string { + return path.join("data", "external-discovery", `${date}.candidate-explanations.json`); +} + +export function externalCandidateExplanationsLatestPath(): string { + return path.join("data", "external-discovery", "latest.candidate-explanations.json"); +} + export function externalEntityRegistryPath(): string { return path.join("data", "external-discovery", "entity-registry.json"); } diff --git a/src/externalDiscovery/types.ts b/src/externalDiscovery/types.ts index 0f40db5..c1a87d5 100644 --- a/src/externalDiscovery/types.ts +++ b/src/externalDiscovery/types.ts @@ -131,6 +131,15 @@ export interface ProviderRejectedEvent { reason_detail: string; } +export interface ExternalCandidateExplanationTitleContext { + event_id?: string; + target_key: string; + target_display_name?: string; + public_evidence_title?: string; + public_source_title?: string; + source_platform: ExternalPlatform; +} + export interface AgentReachProviderReadResult { provider: "agent-reach"; schema_version: "agent-reach.external-discovery.v1"; @@ -143,4 +152,5 @@ export interface AgentReachProviderReadResult { rejected_events: ProviderRejectedEvent[]; warnings: ExternalDiscoveryAudit["warnings"]; source_input_hash: string; + title_context: ExternalCandidateExplanationTitleContext[]; } diff --git a/src/types.ts b/src/types.ts index bcda298..1acdf61 100644 --- a/src/types.ts +++ b/src/types.ts @@ -774,10 +774,10 @@ export interface DailyRunSummaryDiagnostics { }; } -export interface DailyRunSummaryTopProject { - project_name: string; - repo_url: string; - total_score: number; +export interface DailyRunSummaryTopProject { + project_name: string; + repo_url: string; + total_score: number; base_final_rank?: number; final_rank?: number; confidence: ScoreBreakdown["confidence"]; @@ -788,12 +788,31 @@ export interface DailyRunSummaryTopProject { summary_source?: EnhancementSource; judge_source?: EnhancementSource; why_selected: string[]; - risks: string[]; -} - -export interface DailyRunSummary { - date: string; - generated_at: string; + risks: string[]; +} + +export interface DailyRunSummaryExternalDiscovery { + aggregate_status: "ok" | "skipped" | "partial" | "failed"; + aggregate_status_reason?: string; + accepted_event_count: number; + rejected_event_count: number; + observation_candidate_count: number; + project_evidence_count: number; + direction_evidence_count: number; + explanation_status: "ok" | "partial" | "skipped" | "failed"; + explanation_status_reason?: string; + explanation_eligible_count: number; + explanation_attempted_count: number; + explanation_enhanced_count: number; + explanation_fallback_count: number; + explanation_rejected_count: number; + warning_count: number; + warnings: string[]; +} + +export interface DailyRunSummary { + date: string; + generated_at: string; dry_run: boolean; minimum_viable_run_completed: boolean; completion_notes: string[]; @@ -828,10 +847,11 @@ export interface DailyRunSummary { source_status: DailyRunSummarySourceStatus[]; quality: DailyRunSummaryQuality; diagnostics: DailyRunSummaryDiagnostics; - top_projects: DailyRunSummaryTopProject[]; - observer_status?: { - ecosystem_focus: ObserverStatus; - }; + top_projects: DailyRunSummaryTopProject[]; + external_discovery?: DailyRunSummaryExternalDiscovery; + observer_status?: { + ecosystem_focus: ObserverStatus; + }; observer_candidate_count?: number; observer_ecosystem_counts?: Record; observer_incubating_directions?: ObserverIncubatingDirection[]; diff --git a/src/visualConsole/readLayer.ts b/src/visualConsole/readLayer.ts index 4c1426b..d595b33 100644 --- a/src/visualConsole/readLayer.ts +++ b/src/visualConsole/readLayer.ts @@ -14,6 +14,7 @@ import type { WeeklyReport, } from "../types.ts"; import type { MissionScoutEnhancementArtifact } from "../signal/missionScoutEnhancement.ts"; +import type { ExternalCandidateExplanationArtifact } from "../externalDiscovery/explanations.ts"; import { getFilesystemStateSignature, readCachedDirectoryEntries, readCachedJsonFile, readCachedTextFile } from "./fileCache.ts"; import { parseWeeklyMarkdown } from "./weeklyMarkdown.ts"; import type { DailyTimeNavigatorPreview, ReadResult, TopLevelViewStatus, WeeklyTimeNavigatorPreview } from "./types.ts"; @@ -166,6 +167,11 @@ export function getProjectLibraryEnhancementArtifact(date: string): ReadResult

(filepath); } +export function getExternalCandidateExplanationsArtifact(date: string): ReadResult { + const filepath = path.join("data", "external-discovery", `${date}.candidate-explanations.json`); + return validateDateInput(date, filepath) ?? readJsonStrict(filepath); +} + export function getMissionScoutArtifact(date: string): ReadResult<{ raw_signals?: RawSignal[] }> { const filepath = path.join("data", "discovery", "mission-scout", `${date}.json`); return validateDateInput(date, filepath) ?? readJsonStrict<{ raw_signals?: RawSignal[] }>(filepath); From 7c40808594446ba19b4d2354315be339cf45cccb Mon Sep 17 00:00:00 2001 From: Aspetta <892024285@qq.com> Date: Sat, 4 Jul 2026 17:39:48 +0800 Subject: [PATCH 5/8] feat(agentreach): add external discussion trend window --- data/README.md | 4 + ...external-discussion-trend-window-v0.1.json | 8 + ...external-discussion-trend-window-design.md | 852 ++++++++++++++++++ ...-discussion-trend-window-v0.1.exec-plan.md | 604 +++++++++++++ docs/specs/services/action-output.md | 11 + docs/specs/services/scoring-engine.md | 1 + .../externalDiscoveryDailyIntegration.test.ts | 5 + ...lDiscussionTrendWindowActionOutput.test.ts | 88 ++ ...ternalDiscussionTrendWindowBuilder.test.ts | 136 +++ ...DiscussionTrendWindowDirectionGate.test.ts | 120 +++ ...xternalDiscussionTrendWindowDryRun.test.ts | 45 + .../externalDiscussionTrendWindowFixtures.ts | 89 ++ .../externalDiscussionTrendWindowPath.test.ts | 14 + ...nalDiscussionTrendWindowReadStatus.test.ts | 82 ++ ...rnalDiscussionTrendWindowRedaction.test.ts | 75 ++ ...externalDiscussionTrendWindowRules.test.ts | 95 ++ ...lDiscussionTrendWindowTypeContract.test.ts | 106 +++ ...lDiscussionTrendWindowVerification.test.ts | 195 ++++ ...xternalDiscussionTrendWindowWeekly.test.ts | 129 +++ ...externalDiscussionTrendWindowWrite.test.ts | 121 +++ src/__tests__/runSummaryObserver.test.ts | 9 + src/action/dailyVerification.ts | 84 +- src/action/runSummary.ts | 5 + src/action/weeklyEnhancement.ts | 65 ++ src/action/weeklyReport.ts | 41 +- src/cli.ts | 10 +- src/externalDiscovery/dailyIntegration.ts | 13 + src/externalDiscovery/paths.ts | 8 + src/externalDiscovery/redaction.ts | 22 + src/externalDiscovery/trendWindow.ts | 695 ++++++++++++++ .../trendWindowIntegration.ts | 148 +++ src/externalDiscovery/types.ts | 128 +++ src/types.ts | 95 +- 33 files changed, 4078 insertions(+), 25 deletions(-) create mode 100644 docs/specs/agent-work/code-implementation-preflight.agentreach-external-discussion-trend-window-v0.1.json create mode 100644 docs/specs/design-docs/agentreach-external-discussion-trend-window-design.md create mode 100644 docs/specs/exec-plans/agentreach-external-discussion-trend-window-v0.1.exec-plan.md create mode 100644 src/__tests__/externalDiscussionTrendWindowActionOutput.test.ts create mode 100644 src/__tests__/externalDiscussionTrendWindowBuilder.test.ts create mode 100644 src/__tests__/externalDiscussionTrendWindowDirectionGate.test.ts create mode 100644 src/__tests__/externalDiscussionTrendWindowDryRun.test.ts create mode 100644 src/__tests__/externalDiscussionTrendWindowFixtures.ts create mode 100644 src/__tests__/externalDiscussionTrendWindowPath.test.ts create mode 100644 src/__tests__/externalDiscussionTrendWindowReadStatus.test.ts create mode 100644 src/__tests__/externalDiscussionTrendWindowRedaction.test.ts create mode 100644 src/__tests__/externalDiscussionTrendWindowRules.test.ts create mode 100644 src/__tests__/externalDiscussionTrendWindowTypeContract.test.ts create mode 100644 src/__tests__/externalDiscussionTrendWindowVerification.test.ts create mode 100644 src/__tests__/externalDiscussionTrendWindowWeekly.test.ts create mode 100644 src/__tests__/externalDiscussionTrendWindowWrite.test.ts create mode 100644 src/externalDiscovery/trendWindow.ts create mode 100644 src/externalDiscovery/trendWindowIntegration.ts diff --git a/data/README.md b/data/README.md index 8c5b5d3..849796b 100644 --- a/data/README.md +++ b/data/README.md @@ -28,6 +28,10 @@ Public external discovery history belongs in `data/external-discovery/*.aggregat `data/external-discovery/*.candidate-explanations.json` is a cached display-only AgentReach explanation artifact. Local real-run files are ignored by default; only sanitized fixtures used by tests should be committed. +`data/external-discovery/windows/*.discussion-trend-window.json` is the derived AgentReach external discussion trend window. It is public-safe by contract and is generated from sanitized daily aggregates, not from raw provider input. It must not contain raw social text, profile URLs, raw handles, cookies, sessions, OAuth material, private runner diagnostics, or platform tokens. + +Local real-run trend window artifacts should be reviewed before committing. The source raw input under `data/raw/external-discovery/` remains local-only even when the derived window is safe to inspect. + ## Automation behavior The daily and weekly GitHub Actions workflows update the tracked public artifacts and commit them back into this repository. This is intentional: the repo is both code and a public historical data log. diff --git a/docs/specs/agent-work/code-implementation-preflight.agentreach-external-discussion-trend-window-v0.1.json b/docs/specs/agent-work/code-implementation-preflight.agentreach-external-discussion-trend-window-v0.1.json new file mode 100644 index 0000000..f376904 --- /dev/null +++ b/docs/specs/agent-work/code-implementation-preflight.agentreach-external-discussion-trend-window-v0.1.json @@ -0,0 +1,8 @@ +{ + "skill_path": "docs/specs/agent-work/CodeImplementation_Skill.md", + "skill_sha256": "a15bb25d65ff2b12867a3255812d109d0821d6838a89b232392cbfaa70b7a95d", + "exec_plan_path": "docs/specs/exec-plans/agentreach-external-discussion-trend-window-v0.1.exec-plan.md", + "exec_plan_sha256": "16853f2fe8bb0e5be97cde0794c7f898ed0e3a401eec82d84bb82446d1a42af7", + "generated_at": "2026-07-03T13:04:21.289Z", + "acknowledged": true +} diff --git a/docs/specs/design-docs/agentreach-external-discussion-trend-window-design.md b/docs/specs/design-docs/agentreach-external-discussion-trend-window-design.md new file mode 100644 index 0000000..4cfb3bc --- /dev/null +++ b/docs/specs/design-docs/agentreach-external-discussion-trend-window-design.md @@ -0,0 +1,852 @@ +# AgentReach 外部讨论趋势窗口设计 + +## 文档状态 + +- 状态：`Approved for ExecPlan` +- 日期：`2026-07-03` +- 范围：`AgentReach external discussion trend window` +- 修订记录：`2026-07-03` 根据设计审核补齐 canonical artifact、字段派生、方向级 weekly gate 和趋势阈值硬约束。 +- 复审记录：`2026-07-03` Design Review 复审通过，可生成对应 exec-plan。 +- 对应需求：`docs/specs/product-specs/外部发现与补证信号层需求分析.md` +- 关联设计： + - `docs/specs/design-docs/agent-reach-external-discovery-and-evidence-design.md` + - `docs/specs/design-docs/agentreach-candidate-explanation-enhancement-design.md` +- 关联现有实现： + - `src/externalDiscovery/types.ts` + - `src/externalDiscovery/aggregate.ts` + - `src/externalDiscovery/dailyIntegration.ts` + - `src/externalDiscovery/agentReachProvider.ts` + - `src/externalDiscovery/entityRegistry.ts` + - `src/externalDiscovery/explanations.ts` + - `src/action/weeklyEnhancement.ts` + - `src/action/weeklyReport.ts` + - `src/visualConsole/build.ts` + +本文只冻结设计，不修改代码，不生成 exec-plan，不清理运行产物。 + +## 1. Summary + +AgentReach 目前已经能够把上游外部信号转换为单日 `DailyExternalAggregate`，并产出 `observation_candidates`、`project_evidence`、`direction_evidence`、`platform_counts`、`named_registry_actors`、`audit` 等字段。这个能力可以回答“今天外部层发现了什么”和“哪些候选获得了外部证据”，但还不能稳定回答需求文档中的另一个核心问题：这波外部讨论是偶发噪声，还是正在形成持续趋势。 + +本设计新增一个只消费最近 7 天 `DailyExternalAggregate[]` 的结构化趋势窗口：`ExternalDiscussionTrendWindow`。它不重新调用 AgentReach 上游，不读取 raw provider input，不改变主榜评分，不让 LLM 直接裁决趋势。它只把已有按日外部聚合进一步整理为项目级和方向级的外部讨论趋势事实，供 weekly、run-summary 和 `/agentreach` UI 只读展示。 + +设计结论： + +1. 外部讨论趋势必须基于 7 日按日窗口，而不是单日 aggregate 或一次 LLM 摘要。 +2. 趋势判断使用 rules-first 组件：讨论量、持续天数、跨平台确认、讨论者层级、绑定可信度、噪声风险。 +3. LLM 只能解释结构化趋势事实，不能决定 `momentum` 或 `verdict`。 +4. 趋势窗口只能作为次级证据，不参与 `ScoreBreakdown.total_score`、`discussion_score`、主源多源确认或主榜排名。 +5. 项目级趋势和方向级趋势必须分开表达；未绑定明确 repo / paper / product 的方向讨论不能伪装成项目结论。 + +## 2. Goals / Non-goals + +### Goals + +- 支持“外部讨论强度与持续性”的 7 日窗口表达。 +- 区分项目级外部补证趋势与方向级观察趋势。 +- 判断单平台单日爆发、跨平台确认、多日持续、头部讨论者参与之间的差异。 +- 为 weekly 提供外部讨论强化证据，帮助回答“某个方向是否在外部讨论层面也在强化”。 +- 为 `/agentreach` UI 提供稳定只读字段，用于展示“今日快照”和“近 7 日讨论趋势”。 +- 保留 `ok / partial / failed / insufficient / skipped` 等状态，避免把缺失误写成无信号。 +- 继续满足 public-safe、redaction、dry-run 不写盘和 raw/private 输入隔离要求。 + +### Non-goals + +- 不新增 AgentReach 上游平台，不扩大到 YouTube、Bilibili、播客、公众号或长内容平台。 +- 不让本仓库直接调用 X / Reddit / HN / 官方网页平台 API。 +- 不改变 `RawSignal[]`、`NormalizedProject[]`、`ScoreBreakdown` 或主评分公式。 +- 不把外部讨论趋势写成主榜高置信结论、主趋势结论或最终推荐。 +- 不设计真正多月历史趋势、季节性趋势或统计显著性模型。 +- 不承诺展示具体账号、机构或个人名称，除非 `named_registry_actors` 已经提供 public-safe registry 命中实体。 +- 不在 UI 层重新计算趋势；UI 只能消费趋势窗口 artifact。 + +## 3. Existing Code References + +### AgentReach 已有能力 + +当前 `src/externalDiscovery/types.ts` 已有以下核心字段： + +- `DailyExternalAggregate` +- `ExternalEvidence` +- `ObservationCandidate` +- `ExternalNamedRegistryActor` +- `platform_counts` +- `project_evidence` +- `direction_evidence` +- `observation_candidates` +- `audit.rejected_events` +- `audit.warnings` + +当前 `src/externalDiscovery/aggregate.ts` 已能基于单日事件聚合： + +- `mention_count` +- `distinct_actor_count` +- `top_tier_actor_count` +- `actor_tiers` +- `actor_types` +- `platforms` +- `named_registry_actors` +- `first_seen_at` +- `last_seen_at` + +这说明趋势窗口不需要重新设计底层事件 schema，第一版可以直接复用 `DailyExternalAggregate` 作为输入。 + +### 主榜 / 周报可借鉴模式 + +`docs/specs/services/scoring-engine.md` 和 `docs/specs/services/action-output.md` 冻结了几个可以借鉴的原则： + +- 评分和判断必须 rules-first、evidence-first。 +- Action / report 层不能重新计算分数，只展示结构化事实和证据。 +- weekly 必须回答趋势抽象问题，不能退化成项目列表。 +- 缺少证据时不能编造，应明确降级。 + +AgentReach 趋势窗口应借鉴这些原则，但不能借用主榜的 `total_score`、`final_rank` 或主源多源确认语义。 + +## 4. Current Gap + +当前 AgentReach 后端链路的主要缺口不是“没有外部信号”，而是： + +1. 只有单日 aggregate，无法判断讨论是否持续。 +2. UI 可以展示候选和证据，但无法说明这是单日噪声、跨平台确认，还是 7 日持续增强。 +3. weekly 还没有一个稳定的外部讨论窗口 artifact 可以消费。 +4. 当前 `platform_counts`、`mention_count`、`distinct_actor_count` 分散在单日证据里，缺少跨天归并后的趋势项。 +5. LLM 生成项目介绍可以改善“看不懂是什么”，但不能解决“趋势是否成立”的结构化判断。 + +因此，本设计新增的是一个 external discovery 内部的趋势窗口，不是 UI 美化，也不是主榜评分扩权。 + +## 5. System Position + +新增趋势窗口位于 single-day aggregate 之后、weekly / UI 之前： + +```text +AgentReach local artifact + -> provider / adapter + -> ExternalSignalEvent[] + -> DailyExternalAggregate + -> ExternalDiscussionTrendWindow + -> weekly secondary evidence + -> /agentreach read-only trend display +``` + +主链路保持不变： + +```text +Primary sources + -> RawSignal[] + -> NormalizedProject[] + -> ScoredProject[] + -> DailyReport / WeeklyReport / Visual Console +``` + +趋势窗口可以被 weekly 引用为 secondary evidence，但不得反向污染主 scoring、主 source count 或主榜排名。 + +## 6. Data Contract + +### 6.1 输入 + +趋势窗口只读取最近 7 天的 public-safe aggregate： + +```text +data/external-discovery/YYYY-MM-DD.aggregate.json +``` + +窗口锚点为 `anchor_date`，读取范围为： + +```text +window_start = anchor_date - 6 days +window_end = anchor_date +window_days = 7 +``` + +输入约束： + +- 只读取 `DailyExternalAggregate`。 +- 不读取 `data/raw/external-discovery/**`。 +- 不调用 AgentReach CLI 或平台 API。 +- 不读取 provider raw `text`、handle、profile URL 或私有 diagnostics。 +- 缺失某一天 aggregate 时记录为 missing day，不把 missing day 当作无讨论。 + +### 6.2 输出 artifact + +趋势窗口 canonical artifact 固定为： + +```text +data/external-discovery/windows/YYYY-MM-DD.discussion-trend-window.json +data/external-discovery/windows/latest.discussion-trend-window.json +``` + +实现阶段不得再改为 `data/external-discovery/YYYY-MM-DD.discussion-trend-window.json` 或其他平铺路径。原因是趋势窗口是 aggregate 的二级派生 artifact，放入 `windows/` 可以避免与单日 aggregate 混淆。 + +Writer / reader 规则固定如下： + +- 唯一 writer：external discovery trend window builder。 +- 允许 reader：weekly secondary evidence builder、run-summary / verify、visual console read model。 +- 禁止 reader：主 scoring、normalization、primary source freshness gate。 +- `latest.discussion-trend-window.json` 只能指向最近一次成功写入的 public-safe trend window。 +- 当 date-specific trend window 缺失时，UI / weekly 必须表达 `not_found` 或 `insufficient`，不得回退读取 raw input。 +- 当 `latest` 存在但 date-specific 文件不存在时，date route 不能静默读取 latest；只能在用户显式请求 `latest` 时读取 latest。 +- `dry-run` 只能报告 planned write path，不得写入 date-specific 文件或 latest 指针。 + +### 6.3 `ExternalDiscussionTrendWindow` + +```ts +interface ExternalDiscussionTrendWindow { + schema_version: "external-discussion-trend-window.v1"; + anchor_date: string; + window_start: string; + window_end: string; + window_days: 7; + generated_at: string; + + status: ExternalTrendWindowStatus; + status_reason?: string; + + project_trends: ExternalTrendItem[]; + direction_trends: ExternalTrendItem[]; + + coverage: ExternalTrendCoverage; + audit: ExternalTrendAudit; + + public_safe: true; + redaction_policy_version: string; + contains_raw_text: false; + contains_profile_urls: false; +} + +type ExternalTrendWindowStatus = + | "ok" + | "partial" + | "failed" + | "insufficient" + | "skipped"; +``` + +状态语义： + +- `ok`：7 日窗口中可用 aggregate 足够，至少形成一个趋势项或明确无相关趋势。 +- `partial`：部分日期或部分平台失败，但仍有可解释趋势项。 +- `failed`：窗口读取或解析失败，无法形成趋势窗口。 +- `insufficient`：可读数据太少，不足以做趋势判断。 +- `skipped`：外部趋势窗口被显式关闭，或上游 external discovery 未启用。 + +### 6.4 `ExternalTrendItem` + +```ts +interface ExternalTrendItem { + trend_id: string; + scope: "project" | "direction"; + target_key: string; + display_name: string; + target_url?: string; + binding_confidence: ExternalTrendBindingConfidence; + official_signal: boolean; + weekly_eligible: boolean; + weekly_gate_reasons: ExternalWeeklyGateReason[]; + weekly_gate_missing_reasons: ExternalWeeklyGateReason[]; + + daily_counts: ExternalTrendDailyCount[]; + + mention_count_total: number; + active_day_count: number; + platform_count: number; + cross_platform_days: number; + distinct_actor_count: number; + top_tier_actor_count: number; + named_registry_actors: ExternalNamedRegistryActor[]; + + components: ExternalTrendComponent[]; + momentum: ExternalTrendMomentum; + verdict: ExternalTrendVerdict; + + cannot_be_primary_conclusion: true; + evidence_ids: string[]; + source_aggregate_dates: string[]; + caveats: string[]; +} + +interface ExternalTrendDailyCount { + date: string; + mention_count: number; + source_count: number; + platform_count: number; + distinct_actor_count: number; + top_tier_actor_count: number; +} + +type ExternalTrendMomentum = + | "rising" + | "stable" + | "spike" + | "fading" + | "insufficient"; + +type ExternalTrendVerdict = + | "external_reinforcement" + | "watch_signal" + | "noise_spike" + | "insufficient"; + +type ExternalTrendBindingConfidence = + | "high" + | "medium" + | "low" + | "none"; + +type ExternalWeeklyGateReason = + | "cross_platform_confirmation" + | "multi_actor_confirmation" + | "multi_day_persistence" + | "registry_tier_participation"; +``` + +约束： + +- `cannot_be_primary_conclusion` 固定为 `true`。 +- `scope="direction"` 的趋势项不得含义漂移为项目结论。 +- `named_registry_actors` 只能来自 aggregate 中的 public-safe `ExternalEvidence.named_registry_actors`。 +- `evidence_ids` 必须能回到单日 `project_evidence` 或 `direction_evidence`。 +- `source_aggregate_dates` 记录参与当前趋势项的日期，便于审计 missing day。 +- `official_signal` 只能由 contributing evidence 的 `platforms` 包含 `official_web` 或 `official_blog` 得出，不能由 LLM 文案、title 或项目名推断。 +- `weekly_eligible` 表示该趋势项是否允许进入 weekly 正文的外部讨论趋势区；不等于主榜确认。 +- `weekly_gate_reasons` 与 `weekly_gate_missing_reasons` 只记录上方四个 frozen reason，不得写自由文本。 + +### 6.4.1 字段派生与来源映射 + +趋势窗口不得从 raw provider input、UI 状态或 LLM 自由文本中临时拼字段。以下字段来源固定： + +| 字段 | 生产者 | 来源优先级 | 缺失行为 | 禁止事项 | +| --- | --- | --- | --- | --- | +| `trend_id` | trend window builder | `${scope}:${target_key}` 的稳定 hash 或稳定拼接 | 缺失即 rejected，reason=`trend_item_unstable_target_key` | 不得使用数组下标或随机 ID | +| `display_name` | trend window builder | 1. public aggregate 未来若提供 display name，使用该字段；2. repo-like `target_key` 提取 `owner/repo`；3. topic key 转为可读 label；4. 最后回退 `target_key` | 不得为空 | 不得使用 raw title、raw text 或 LLM 生成名 | +| `target_url` | trend window builder | 1. public aggregate 未来若提供 target URL；2. canonical repo target key 可确定 repo URL 时派生 | 缺失则省略 | 不得读取 raw provider URL 补齐 | +| `binding_confidence` | trend window builder | `scope="project"` 且 target key 可确定 repo / paper / product 为 `medium`；public aggregate 未来提供明确绑定字段时可为 `high`；`scope="direction"` 为 `low`；target key 不稳定为 `none` | 缺失即按 `none` 处理并加 caveat | 不得由 LLM 或 title 猜测绑定 | +| `source_count` | trend window builder | V1 固定为 contributing `event_ids` 去重数量 | 可为 0，仅在 evidence 为空时 | UI 不得把它翻译为“独立来源 URL 数” | +| `mention_count` | trend window builder | contributing `ExternalEvidence.mention_count` 求和 | 无 evidence 时为 0 | 不得从 title 或摘要重新计数 | +| `platform_count` | trend window builder | contributing `ExternalEvidence.platforms` 去重数量 | 无 evidence 时为 0 | 不得把 partial 平台当作 0 信号 | +| `distinct_actor_count` | trend window builder | contributing `ExternalEvidence.distinct_actor_count` 求和后按 public-safe actor identity 去重；若无法跨 evidence 去重，必须保守使用单日最大值并加 caveat | 缺失时为 0 并加 caveat | 不得用 provider raw handle 去重 | +| `top_tier_actor_count` | trend window builder | registry 命中的 `named_registry_actors.entity_id` 去重数量 | 缺失时为 0 | `provider_tier_hint` 不得参与 | +| `official_signal` | trend window builder | contributing evidence 的 `platforms` 包含 `official_web` 或 `official_blog` | 默认为 `false` | 不得由项目名、标题、LLM 文案推断 | +| `caveats` | trend window builder | coverage、binding、gate、partial、public-safe 检查的结构化结果 | 无 caveat 时为空数组 | 不得把 caveat 只留给 UI 临时生成 | + +如果实现阶段发现 `DailyExternalAggregate` 无法提供上述 public-safe 来源，必须先扩展 aggregate contract 和测试，再生成趋势窗口；不得由 UI、LLM 或 raw input 补洞。 + +### 6.5 `ExternalTrendCoverage` + +```ts +interface ExternalTrendCoverage { + expected_dates: string[]; + loaded_dates: string[]; + missing_dates: string[]; + failed_dates: Array<{ + date: string; + reason_code: string; + reason_detail: string; + }>; + usable_day_count: number; + platform_counts: Partial>; + partial_platforms: ExternalPlatform[]; +} +``` + +Coverage 必须区分： + +- 日期缺失。 +- aggregate 文件存在但解析失败。 +- aggregate status 为 `partial`。 +- 平台 partial。 +- 可用天数不足。 + +### 6.6 `ExternalTrendAudit` + +```ts +interface ExternalTrendAudit { + rejected_items: Array<{ + scope?: "project" | "direction"; + target_key?: string; + reason_code: string; + reason_detail: string; + }>; + warnings: Array<{ + reason_code: string; + reason_detail: string; + }>; +} +``` + +典型 reason code： + +- `window_aggregate_missing` +- `window_aggregate_parse_failed` +- `window_usable_days_insufficient` +- `trend_item_single_day_single_platform` +- `trend_item_unstable_target_key` +- `direction_without_stable_topic_key` +- `project_trend_unbound` +- `named_actor_registry_empty` +- `named_actor_registry_miss` + +## 7. Project vs Direction Normalization + +趋势窗口必须先区分项目级和方向级： + +### 项目级趋势 + +满足以下条件之一可进入 `project_trends`： + +- 单日 evidence 的 `scope="project"`。 +- 目标有稳定 `target_key`，且可绑定 repo / paper / product。 +- `ObservationCandidate.candidate_kind` 为 `project | paper | product`，且不是 `qualification="direction_observation"`。 + +项目级趋势仍然只是外部补证或观察候选，不等于主榜项目成立。 + +项目级 `weekly_eligible` 规则： + +- `verdict="external_reinforcement"` 时可进入 weekly 正文外部补证区。 +- `verdict="watch_signal"` 时只进入 weekly 的观察 / 次级证据区，不得写入主趋势确认段。 +- `verdict="noise_spike"` 或 `insufficient` 只能进入 audit 或 UI 风险提示，不进入 weekly 正文正向材料。 + +### 方向级趋势 + +满足以下条件之一进入 `direction_trends`： + +- 单日 evidence 的 `scope="direction"`。 +- `target_type="topic"`。 +- `ObservationCandidate.candidate_kind="direction"`。 +- `qualification="direction_observation"`。 + +方向级趋势必须显示 caveat： + +```text +尚未绑定明确项目，不能作为项目级结论。 +``` + +英文 UI 可对应： + +```text +Not bound to a specific project yet; use as a direction-level watch signal only. +``` + +方向级 `weekly_eligible` gate 固定为 4 选 2。只有满足以下四个收敛条件中的至少两个，方向级趋势才允许进入 weekly 的方向观察区： + +1. `cross_platform_confirmation`：同一 `target_key` 在 7 日窗口内至少出现于 2 个 V1 外部平台。 +2. `multi_actor_confirmation`：同一 `target_key` 至少有 2 个 public-safe 可区分 actor 参与；若无法跨 evidence 去重，则必须用保守口径并加 caveat。 +3. `multi_day_persistence`：同一 `target_key` 在 7 日窗口内至少有 2 个 active day。 +4. `registry_tier_participation`：同一 `target_key` 至少有 1 个 registry 命中的 `core / proven / watch` actor 参与。 + +方向级趋势的消费边界： + +- 满足 2 个及以上 gate reason：`weekly_eligible=true`，可进入 weekly 方向观察区，但仍只能表达为方向级观察。 +- 只满足 1 个 gate reason：`weekly_eligible=false`，可在 `/agentreach` UI 中作为低置信观察或 audit 提示展示，不进入 weekly 正文。 +- 满足 0 个 gate reason：`weekly_eligible=false`，默认归为 `noise_spike` 或 `insufficient`，仅进入 audit。 +- `provider_tier_hint` 不得满足 `registry_tier_participation`。 +- 单个 HN 高热标题若无法形成多日、多主体、跨平台或 registry 命中，不得进入 weekly 方向观察区。 + +## 8. Component Design + +趋势窗口不使用主榜 `ScoreBreakdown`，但可以采用组件化解释方式。每个 `ExternalTrendItem.components` 包含： + +```ts +interface ExternalTrendComponent { + name: ExternalTrendComponentName; + level: "none" | "low" | "medium" | "high"; + evidence: string[]; +} + +type ExternalTrendComponentName = + | "discussion_volume" + | "persistence" + | "cross_platform_confirmation" + | "actor_authority" + | "binding_confidence" + | "noise_risk"; +``` + +### `discussion_volume` + +表达 7 日窗口总提及量。第一版只做桶化，不做复杂统计： + +- `none`：`mention_count_total = 0` +- `low`：`1-2` +- `medium`：`3-5` +- `high`：`>=6` + +### `persistence` + +表达持续天数： + +- `none`：`active_day_count = 0` +- `low`：`active_day_count = 1` +- `medium`：`active_day_count = 2-3` +- `high`：`active_day_count >= 4` + +### `cross_platform_confirmation` + +表达跨平台确认： + +- `none`：`platform_count <= 1` +- `medium`：`platform_count >= 2` +- `high`：`platform_count >= 2` 且 `cross_platform_days >= 2` + +### `actor_authority` + +表达“谁在讨论”的分层质量： + +- `none`：无可识别 actor。 +- `low`：只有普通或 unknown actor。 +- `medium`：有 distinct actor 多主体讨论。 +- `high`：存在 registry 命中 `core / proven / watch` 的具名讨论者。 + +`provider_tier_hint` 不得参与 `actor_authority=high`。 + +### `binding_confidence` + +表达对象绑定质量： + +- 项目级且可绑定 repo / paper / product：`medium` 或 `high`。 +- 方向级或 unbound：`low`。 +- 不稳定 target key：`none`，并进入 audit 或 caveat。 + +### `noise_risk` + +表达噪声风险。它是风险组件，不是正向分： + +- `high`：满足以下任一条件： + - `active_day_count = 1` 且 `platform_count = 1` 且 `top_tier_actor_count = 0` 且 `official_signal=false`。 + - `binding_confidence="none"` 或 target key 不稳定。 + - `mention_count_total >= 3` 但 `max_daily_share >= 0.80` 且无跨平台确认。 +- `medium`：不满足 high，且满足以下任一条件： + - `mention_count_total <= 2`。 + - `platform_count = 1`。 + - `binding_confidence="low"`。 + - 窗口状态为 `partial`。 +- `low`：不满足 high / medium，且满足以下至少一项： + - `active_day_count >= 2`。 + - `platform_count >= 2`。 + - `top_tier_actor_count >= 1`。 + - `official_signal=true`。 + +其中 `max_daily_share = max(daily_counts[].mention_count) / mention_count_total`；当 `mention_count_total=0` 时，`noise_risk` 必须为 `high` 或该项进入 `insufficient`。 + +## 9. Momentum / Verdict Rules + +### Momentum + +`momentum` 只基于 7 日结构化计数，不由 LLM 判断。以下列表是用户语义简述，实际实现必须以后方硬阈值为准。 + +- `rising`：后 3 日提及量高于前 4 日，且 `active_day_count >= 2`。 +- `stable`：至少 3 个活跃日，且没有明显单日尖峰。 +- `spike`：只有 1 个活跃日，或单日提及量占总量的 `>= 70%`。 +- `fading`：前 4 日明显高于后 3 日，且后 2 日无新增。 +- `insufficient`：可用天数不足，或有效计数不足。 + +上述自然语言必须按以下硬阈值实现： + +- 先计算： + - `early_loaded_days`：窗口前 4 天中成功 loaded 的天数。 + - `late_loaded_days`：窗口后 3 天中成功 loaded 的天数。 + - `early_avg = early_mentions / early_loaded_days`；若 `early_loaded_days=0`，该项为 `insufficient`。 + - `late_avg = late_mentions / late_loaded_days`；若 `late_loaded_days=0`，该项为 `insufficient`。 + - `max_daily_share = max(daily_counts[].mention_count) / mention_count_total`。 +- `insufficient` 优先级最高：`usable_day_count < 3` 或 `mention_count_total < 2`。 +- `spike`：不满足 insufficient，且 `active_day_count = 1` 或 `max_daily_share >= 0.70`。 +- `rising`：不满足 insufficient / spike，且 `active_day_count >= 2`，并且 `late_avg >= max(early_avg + 1, early_avg * 1.5)`。 +- `fading`：不满足 insufficient / spike，且 `active_day_count >= 2`，并且 `early_avg >= max(late_avg + 1, late_avg * 1.5)`，且窗口最后 2 个已 loaded 日期的 `mention_count` 均为 0。 +- `stable`：不满足以上条件，且 `active_day_count >= 3` 且 `max_daily_share < 0.60`。 +- 剩余情况归为 `insufficient`，不得勉强归为 stable。 + +阈值作为 V1 frozen constants；后续若需要配置化，必须另行设计，不得在本 exec-plan 中临时调整。 + +### Verdict + +`verdict` 是外部层内部的次级判断标签，不是主榜结论。 + +#### `external_reinforcement` + +适用于已有对象获得外部趋势补证： + +- `scope="project"`。 +- 目标可绑定 repo / paper / product。 +- `active_day_count >= 2`。 +- 满足以下至少一项： + - `platform_count >= 2` + - `top_tier_actor_count >= 1` + - 有 official signal。 +- `noise_risk` 不是 `high`。 +- `weekly_eligible=true`。项目级 `weekly_eligible` 可由本 verdict 自动成立，但仍不能进入主榜确认。 + +#### `watch_signal` + +适用于值得继续观察但不足以确认的信号： + +- 项目级或方向级均可。 +- 满足以下至少一项： + - `active_day_count >= 2` + - `platform_count >= 2` + - `top_tier_actor_count >= 1` +- 但绑定、持续性或证据数量不足以进入 `external_reinforcement`。 + +方向级趋势第一版最高只能到 `watch_signal`，不能成为 `external_reinforcement`。 + +方向级 `watch_signal` 若 `weekly_eligible=false`，只能在 `/agentreach` UI 中作为低置信观察或 audit 提示展示，不得进入 weekly 正文。 + +#### `noise_spike` + +适用于短促噪声： + +- `active_day_count = 1`。 +- `platform_count = 1`。 +- `top_tier_actor_count = 0`。 +- 无官方信号。 + +#### `insufficient` + +适用于： + +- 数据窗口不足。 +- 缺失日期过多。 +- target key 不稳定。 +- 单日 aggregate 本身 failed / partial 过多，无法形成趋势判断。 + +## 10. 7-Day Window Read Semantics + +### 可用天数 + +V1 固定口径： + +- `usable_day_count >= 5`：可正常判断。 +- `usable_day_count = 3-4`：窗口状态为 `partial`，趋势项可产出但必须带 caveat。 +- `usable_day_count < 3`：窗口状态为 `insufficient`，不输出强判断。 + +### 缺失日期 + +缺失日期不得被当作 0 信号。UI 和 weekly 应表达： + +```text +近 7 日外部窗口不完整，以下趋势仅基于可用日期。 +``` + +### Partial 平台 + +当某天 aggregate status 为 `partial`，趋势窗口必须继承 partial 信息。不能把 X / Reddit / HN 的失败写成“无讨论”。 + +## 11. Weekly Consumption + +Weekly 只消费 `ExternalDiscussionTrendWindow` 或 7 日 `DailyExternalAggregate[]` 派生出的同一结构，不直接读取 raw input。 + +Weekly 可使用的字段： + +- `project_trends[].verdict` +- `direction_trends[].verdict` +- `momentum` +- `active_day_count` +- `platform_count` +- `top_tier_actor_count` +- `named_registry_actors` +- `caveats` +- `evidence_ids` + +Weekly 不得使用外部趋势做以下事情： + +- 改变主榜 `total_score`。 +- 让方向级趋势升级为项目级高置信结论。 +- 把 `external_reinforcement` 写成主源确认。 +- 把 `noise_spike` 作为正向趋势材料。 + +Weekly 推荐表达： + +```text +外部讨论补证：该项目在近 7 日出现多日外部讨论，并有跨平台或具名讨论者参与，可作为次级证据继续观察。 +``` + +方向级推荐表达： + +```text +方向观察：该主题在近 7 日外部讨论中出现持续或跨平台信号，但尚未绑定明确项目，只作为方向级观察。 +``` + +## 12. UI Consumption + +`/agentreach` UI 只能读取趋势窗口 view-model，不得在 React 或 SSR 中重新计算趋势。 + +推荐 UI 展示层级： + +1. 顶部保留今日外部发现总览。 +2. 主工作区继续展示候选列表和候选详情。 +3. 在详情页增加轻量“近 7 日外部讨论”区块。 +4. 底部或侧栏弱化展示方向级趋势，不抢主候选区注意力。 + +V1 UI 展示方式固定为“详情页轻量补充”，不是新建宽屏趋势 dashboard： + +- 候选详情页显示一条紧凑的 `近 7 日外部讨论` 摘要行，最多展示 `momentum`、`active_day_count`、`platform_count`、`verdict`。 +- 展开详情时展示 components、gate reasons、caveats 和具名讨论者摘要。 +- 方向级趋势只进入弱化区或折叠区，不作为页面主排序入口。 +- `noise_spike` 不在 weekly 正文正向展示；UI 可在候选详情中作为风险提示展示。 +- 趋势区不存在时，保留今日候选和证据主结构，并显示“近 7 日趋势窗口尚不可用 / 数据不足”。 + +趋势文案建议： + +- `近 7 日外部讨论` +- `讨论势头` +- `持续天数` +- `跨平台` +- `谁在讨论` +- `趋势判断` +- `仍属次级证据` + +禁止 UI 文案： + +- 不显示 `score` 作为趋势主指标。 +- 不显示“主榜结论”“高置信推荐”。 +- 不把 `partial` 写成“无信号”。 +- 不把 `named_registry_actors=[]` 写成“没有人讨论”；应写成“没有可公开具名展示的讨论者”。 + +## 13. LLM Boundary + +LLM 可以用于： + +- 把 `ExternalTrendItem` 的结构化字段总结为短文案。 +- 解释“为什么值得看”。 +- 把多个 caveats 改写成更易读的中文/英文。 +- 生成 UI 的简短项目/方向解释。 + +LLM 不可以用于： + +- 决定 `momentum`。 +- 决定 `verdict`。 +- 直接生成 `active_day_count`、`platform_count`、`top_tier_actor_count`。 +- 编造项目用途、讨论者身份、机构名称或跨平台确认。 +- 在没有结构化证据时输出“趋势正在形成”。 + +如果 LLM 不可用，趋势窗口仍必须能 rules-only 生成结构化结果；UI 只降级为模板文案。 + +## 14. Public-Safe / Audit / Failure States + +趋势窗口是 public aggregate 的派生物，也必须满足 public-safe： + +- `public_safe=true` +- `contains_raw_text=false` +- `contains_profile_urls=false` +- 不包含 provider raw `text` +- 不包含 raw handle +- 不包含 profile URL +- 不包含 cookie / token / session / OAuth / password +- 不包含私有 provider diagnostics + +失败状态必须进入 artifact： + +- 无 aggregate：`insufficient` 或 `skipped` +- aggregate 解析失败：`partial` 或 `failed` +- 可用天数不足：`insufficient` +- 单平台 partial：`partial` +- public-safe 检查失败：`failed` + +Verify 阶段应检查： + +- 趋势窗口是否只读取 public aggregate。 +- 是否包含 forbidden raw 字段。 +- 是否把外部趋势写入主 score。 +- 是否把方向级趋势写成项目级结论。 +- 是否缺少 coverage / audit。 + +## 15. Test & Review Plan + +### Unit Tests + +- 7 日 aggregate merge：同一 `target_key` 跨天合并。 +- missing day：缺失日期进入 coverage，不当作 0。 +- partial aggregate：窗口状态变为 partial。 +- insufficient window：可用天数不足时不输出强判断。 +- canonical artifact path：只写入 `data/external-discovery/windows/*.discussion-trend-window.json`。 +- field derivation：`display_name / target_url / source_count / official_signal / binding_confidence` 不从 raw input、UI 或 LLM 生成。 +- project / direction separation：方向级不会进入项目级 trends。 +- single-platform spike：输出 `noise_spike`。 +- multi-day signal：输出 `watch_signal`。 +- multi-day + cross-platform project：输出 `external_reinforcement`。 +- direction weekly gate：方向级趋势必须满足 4 选 2 才能 `weekly_eligible=true`。 +- registry actor：`named_registry_actors` 只来自 public-safe registry 命中。 +- provider hint：不得产生 `actor_authority=high`。 +- redaction：趋势窗口不含 raw text / profile URL / token 类字段。 + +### Integration Tests + +- weekly 只读趋势窗口或 daily aggregate，不读 raw input。 +- weekly 输出 external secondary evidence，不改主 score。 +- `/agentreach` view-model 只消费趋势窗口字段，不在 UI 侧重算。 +- `dry-run` 只报告 planned writes，不写趋势窗口 artifact。 + +### Review Checks + +- 与需求文档一致：外部层仍为次级信号。 +- 与现有 AgentReach 设计一致：raw input local-only，public aggregate 可公开消费。 +- 与主榜评分 spec 一致：不新增 score component，不改 `discussion_score`。 +- 与 action-output spec 一致：weekly 输出趋势抽象，但不重新计算分数。 + +## 16. Risks / Non-blocking Future Questions + +### Risks + +- 7 日窗口对真实外部讨论仍可能较稀疏，早期经常得到 `insufficient`。 +- X / Reddit partial 会降低趋势可信度，用户可能误以为是无讨论。 +- 方向级 topic key 若不稳定，会导致趋势碎片化。 +- 单平台 HN 高热容易形成 spike，但不一定代表持续趋势。 +- 如果 UI 过度强调趋势卡片，用户可能误读为主榜结论。 + +### Non-blocking Future Questions + +以下问题不阻塞本设计进入 exec-plan，且不得在 V1 实现中临时改变本文已冻结契约： + +1. 方向级趋势未来是否需要单独维护 topic registry？V1 继续使用现有 topic key 归一规则；无稳定 topic key 的方向项不得进入 weekly。 +2. 趋势阈值未来是否需要配置化？V1 使用本文 frozen constants；配置化必须另行设计。 +3. UI 是否在后续版本中提供独立趋势页？V1 仅在 `/agentreach` 详情页和弱化区只读展示。 + +## 17. Rollout Plan + +本设计建议分三阶段实施： + +### Phase 1：后端趋势窗口 artifact + +- 新增 `ExternalDiscussionTrendWindow` 类型。 +- 新增 7 日 aggregate reader。 +- 新增 trend item builder。 +- 新增 component / momentum / verdict rules。 +- 新增 public-safe / coverage / audit 测试。 + +### Phase 2：Weekly secondary consumption + +- weekly 读取趋势窗口。 +- weekly 输出项目级补证和方向级观察趋势。 +- verify 检查外部趋势不污染主 score。 + +### Phase 3：UI read-only display + +- `/agentreach` 读取趋势窗口 view-model。 +- 候选详情展示“近 7 日外部讨论”。 +- 方向观察区展示低优先级趋势列表。 +- UI 只做展示，不重算趋势。 + +## 18. Acceptance Criteria + +- 系统能基于最近 7 天 `DailyExternalAggregate[]` 生成 `ExternalDiscussionTrendWindow`。 +- 趋势窗口只写入 `data/external-discovery/windows/YYYY-MM-DD.discussion-trend-window.json` 与 `latest.discussion-trend-window.json`。 +- 趋势窗口能表达持续天数、跨平台、讨论者层级、噪声风险和 caveats。 +- 项目级和方向级趋势分开输出。 +- 单日单平台爆发被标记为 `noise_spike`，不会成为趋势强化。 +- 多日或跨平台信号可进入 `watch_signal`。 +- 已绑定项目且满足多日 / 跨平台 / 头部讨论者条件时，可进入 `external_reinforcement`。 +- 方向级趋势必须满足 4 选 2 gate 才能进入 weekly 方向观察区。 +- `display_name / target_url / source_count / official_signal / binding_confidence` 的来源符合字段派生表，不由 raw input、UI 或 LLM 补齐。 +- 所有趋势项都带 `cannot_be_primary_conclusion=true`。 +- LLM 不参与趋势裁决；无 LLM key 时仍可 rules-only 产出。 +- weekly 只把趋势窗口作为次级证据，不改变主榜评分和排名。 +- public-safe / redaction 检查通过。 + +## 19. Explicit Non-Regression + +实现本设计后，下列行为必须保持不变： + +- `RawSignal.source` 不新增 external provider。 +- `ScoreBreakdown.total_score` 公式不变。 +- `discussion_score` 不消费 external evidence。 +- external raw input 默认 local-only，不被公开提交。 +- `DailyExternalAggregate` 仍可独立用于今日外部发现页面。 +- `/agentreach` UI 仍可在没有趋势窗口时展示今日候选和证据，并以空态或 partial 状态说明趋势窗口不可用。 diff --git a/docs/specs/exec-plans/agentreach-external-discussion-trend-window-v0.1.exec-plan.md b/docs/specs/exec-plans/agentreach-external-discussion-trend-window-v0.1.exec-plan.md new file mode 100644 index 0000000..93174f2 --- /dev/null +++ b/docs/specs/exec-plans/agentreach-external-discussion-trend-window-v0.1.exec-plan.md @@ -0,0 +1,604 @@ +# 执行计划：AgentReach 外部讨论趋势窗口 v0.1 + +## 文档状态 + +- 版本：`v0.1` +- 当前状态：`Approved for Implementation` +- 日期：`2026-07-03` +- 关联需求：`docs/specs/product-specs/外部发现与补证信号层需求分析.md` +- 关联设计：`docs/specs/design-docs/agentreach-external-discussion-trend-window-design.md` +- 设计状态：`Approved for ExecPlan` +- 说明：本计划只负责把已批准的外部讨论趋势窗口设计拆成可执行实现步骤，不修改代码、不调用外部平台 API、不生成真实运行产物。 + +## 任务信息 + +| 字段 | 内容 | +| --- | --- | +| 任务名称 | AgentReach External Discussion Trend Window | +| 负责人 | Codex | +| 风险等级 | `High` | +| 主要边界 | `External Discovery`、`Action / Weekly`、`Visual Console read model` | +| 主要影响范围 | `src/externalDiscovery/*`、`src/action/*weekly*`、`src/visualConsole/build.ts`、`app/client/*AgentReach*`、`src/__tests__/*`、`docs/specs/*`、`data/README.md` | +| 明确不触达 | 主评分公式、`RawSignal[]`、`discussion_score`、外部平台采集、AgentReach 上游配置、产品生态分支内容 | + +## 价值判断 + +当前 AgentReach 已能产出单日 `DailyExternalAggregate`，可以展示“今天外部层发现了什么”。但它还不能稳定回答“这波讨论是短促噪声，还是近 7 日持续发酵”。该缺口直接影响需求文档中的外部趋势判断、weekly 方向观察和用户对“外部发现是否有价值”的理解。 + +本计划的价值是新增一个可审计的趋势窗口 artifact，把外部层从“单日候选列表”推进为“可判断持续性、跨平台、讨论者层级和噪声风险的次级证据层”。 + +## 目标 + +1. 基于最近 7 天 `DailyExternalAggregate[]` 生成 public-safe `ExternalDiscussionTrendWindow`。 +2. 固定趋势窗口 canonical artifact： + - `data/external-discovery/windows/YYYY-MM-DD.discussion-trend-window.json` + - `data/external-discovery/windows/latest.discussion-trend-window.json` +3. 使用 rules-first 逻辑生成： + - `components` + - `momentum` + - `verdict` + - `weekly_eligible` + - `weekly_gate_reasons` + - `caveats` +4. 让 weekly 只把趋势窗口作为 external secondary evidence 消费，不改变主榜评分和排名。 +5. 让 `/agentreach` UI 只读展示“近 7 日外部讨论”轻量摘要，不在 UI 侧重算趋势。 + +## 非目标 + +- 不新增外部平台。 +- 不直接调用 AgentReach CLI、X / Reddit / HN / official web API。 +- 不读取 `data/raw/external-discovery/**`。 +- 不把 external trend 写入 `RawSignal[]`。 +- 不新增或修改 `ScoreBreakdown` score component。 +- 不让 LLM 生成 `momentum`、`verdict` 或 `weekly_eligible`。 +- 不提交 `data/external-discovery/` 运行产物。 +- 不把方向级趋势升级为项目级主结论。 + +## 不变量 + +实现完成后必须保持： + +- `ScoreBreakdown.total_score` 公式不变。 +- `discussion_score` 不消费 external evidence。 +- 主榜排序不因 external trend 改变。 +- `DailyExternalAggregate` 仍能独立服务今日外部发现页面。 +- external raw input 仍是 local-only。 +- `--dry-run` 不写 date-specific trend window 或 latest 指针。 +- UI 和 weekly 不回退读取 raw input。 + +## 架构落点 + +### 新增或扩展模块 + +- `src/externalDiscovery/types.ts` + - 追加 `ExternalDiscussionTrendWindow`、`ExternalTrendItem`、`ExternalTrendComponent`、`ExternalTrendCoverage`、`ExternalTrendAudit`、`ExternalTrendMomentum`、`ExternalTrendVerdict`、`ExternalTrendBindingConfidence`、`ExternalWeeklyGateReason`。 + - 不删除或重命名已有 external discovery 类型。 + +- `src/externalDiscovery/paths.ts` + - 新增 `externalTrendWindowPath(date: string): string`。 + - 新增 `externalTrendWindowLatestPath(): string`。 + - 路径固定到 `data/external-discovery/windows/`。 + +- `src/externalDiscovery/trendWindow.ts` + - 新增 7 日 aggregate reader、trend item builder、component builder、momentum / verdict / weekly gate rules、public-safe assertion。 + - 只消费 `DailyExternalAggregate[]`。 + - 不读取 provider raw input。 + +- `src/externalDiscovery/trendWindowIntegration.ts` + - 封装 run-daily 后的 trend window planned write / real write。 + - 保持 dry-run 只返回 planned writes。 + - 非 dry-run 下，若当日 aggregate 不存在、窗口不足或 usable day 不足，写入 public-safe `status="insufficient"` date-specific artifact。 + - dry-run 下只返回 planned `insufficient` write，不落盘、不更新 latest。 + +- `src/externalDiscovery/redaction.ts` + - 扩展 public-safe 检查以覆盖 `ExternalDiscussionTrendWindow`。 + - 禁止 raw text、raw handle、profile URL、cookie、token、session、OAuth、private diagnostics。 + +### 修改模块 + +- `src/action/runSummary.ts` + - 追加 external discussion trend window 读取状态、artifact 状态、路径、coverage、usable day count、project / direction trend counts。 + - 不标记为 primary freshness source。 + +- `src/action/weeklyEnhancement.ts` / `src/action/weeklyReport.ts` + - 读取 canonical trend window 或由 7 日 aggregate 派生的同一结构。 + - 只展示 `external_reinforcement` 与 `weekly_eligible=true` 的 `watch_signal`。 + - `noise_spike` 只进入风险提示或 audit，不作为正向趋势材料。 + +- `src/action/dailyVerification.ts` + - 增加 trend window public-safe、score contamination、raw input fallback、direction-to-project pollution 检查。 + +- `src/visualConsole/build.ts` + - 新增或扩展 AgentReach view-model，使 `/agentreach` 读取 trend window。 + - 只做展示适配，不重算 `momentum`、`verdict`、`weekly_eligible`。 + +- `app/client/*AgentReach*` 或当前 AgentReach 页面组件 + - 候选详情轻量显示 `近 7 日外部讨论`。 + - 展开区显示 components、gate reasons、caveats、具名讨论者摘要。 + - 不新增宽屏趋势 dashboard。 + +### 文档同步 + +- `data/README.md` + - 说明 `data/external-discovery/windows/*.discussion-trend-window.json` 是 public-safe 派生 artifact。 + - 说明 `data/raw/external-discovery/**` 仍 local-only。 + +- `docs/specs/services/action-output.md` + - 补充 weekly 可消费 external discussion trend window，但只作为 secondary evidence。 + +- `docs/specs/services/visual-console-testing-manual.md` + - 补充 `/agentreach` trend window read-only、not_found / insufficient / partial 状态检查。 + +- `docs/specs/services/scoring-engine.md` + - 如当前文档未说明 external trend 不进入 score，补充不变量。 + +## 数据契约 + +### Canonical artifact + +```text +data/external-discovery/windows/YYYY-MM-DD.discussion-trend-window.json +data/external-discovery/windows/latest.discussion-trend-window.json +``` + +### `ExternalDiscussionTrendWindow` + +必须实现设计文档冻结字段： + +- `schema_version="external-discussion-trend-window.v1"` +- `anchor_date` +- `window_start` +- `window_end` +- `window_days=7` +- `status` +- `project_trends` +- `direction_trends` +- `coverage` +- `audit` +- `public_safe=true` +- `contains_raw_text=false` +- `contains_profile_urls=false` + +### `ExternalTrendItem` + +必须实现： + +- `trend_id` +- `scope` +- `target_key` +- `display_name` +- `target_url?` +- `binding_confidence` +- `official_signal` +- `weekly_eligible` +- `weekly_gate_reasons` +- `weekly_gate_missing_reasons` +- `daily_counts` +- `mention_count_total` +- `active_day_count` +- `platform_count` +- `cross_platform_days` +- `distinct_actor_count` +- `top_tier_actor_count` +- `named_registry_actors` +- `components` +- `momentum` +- `verdict` +- `cannot_be_primary_conclusion=true` +- `evidence_ids` +- `source_aggregate_dates` +- `caveats` + +### `ExternalTrendWindowReadStatus` + +必须新增并冻结统一读取状态，供 UI / weekly / verify 共用： + +```ts +type ExternalTrendWindowReadStatus = + | "ok" + | "not_found" + | "parse_error" + | "partial" + | "insufficient" + | "failed" + | "skipped"; +``` + +读取状态映射必须固定： + +- 指定日期文件不存在 => `not_found`。 +- JSON 解析失败或 schema 不匹配 => `parse_error`。 +- artifact 内部 `status` 透传为 `ok` / `partial` / `insufficient` / `failed` / `skipped`。 +- date-specific 路由不得静默回退读取 `latest`。 +- latest 路由只读取 `latest.discussion-trend-window.json`。 +- UI / weekly / verify 都不得在 trend window 缺失时回退读取 raw input。 + +## 字段派生规则 + +实现必须遵循设计文档 `6.4.1 字段派生与来源映射`，重点验证： + +- `display_name` 不来自 raw title、raw text 或 LLM。 +- `target_url` 不读取 raw provider URL 补齐。 +- `source_count` 固定为 contributing `event_ids` 去重数量。 +- `official_signal` 只由 `official_web` / `official_blog` platform 得出。 +- `binding_confidence` 不由 LLM 或 title 猜测。 +- `distinct_actor_count` 无法跨 evidence 去重时必须用保守口径并加 caveat。 +- `top_tier_actor_count` 只来自 registry 命中的 `named_registry_actors.entity_id`。 + +## 阶段进度 + +| 阶段 | 状态 | 目标 | 完成标志 | +| --- | --- | --- | --- | +| Phase 0：执行前对齐 | `DONE` | 固定设计、当前代码基线、测试入口和不变量 | preflight receipt 已生成并通过 `--check` | +| Phase 1：类型、路径、public-safe 基座 | `DONE` | 增加趋势窗口类型、canonical path、redaction 检查 | 类型、路径、redaction 聚焦测试与 typecheck 通过 | +| Phase 2：趋势窗口 builder | `DONE` | 合并 7 日 aggregate，生成 trend items、coverage、audit | builder 单测覆盖窗口、字段派生、缺失日 | +| Phase 3：rules-first 判断 | `DONE` | 实现 components、momentum、verdict、weekly gate | 规则单测覆盖 spike/rising/fading/gate | +| Phase 4：artifact 写入与 dry-run | `DONE` | date-specific / latest 写入，dry-run planned write | 写入、latest、dry-run 单测通过 | +| Phase 5：weekly / run-summary / verify 消费 | `DONE` | weekly 只作为 secondary evidence 消费，verify 防污染 | action / verification 测试通过 | +| Phase 6：/agentreach UI read-only 展示 | `DONE` | 详情页显示近 7 日外部讨论，不重算趋势 | view-model / SSR 渲染测试通过 | +| Phase 7：文档与总体验收 | `DONE` | 同步 data README / specs，运行聚焦验证 | 验证记录补齐 | + +## 实施阶段 + +### Phase 0：执行前对齐 + +1. 读取并确认以下文档状态： + - `docs/specs/product-specs/外部发现与补证信号层需求分析.md` + - `docs/specs/design-docs/agentreach-external-discussion-trend-window-design.md` + - `docs/specs/design-docs/agent-reach-external-discovery-and-evidence-design.md` + - `docs/specs/exec-plans/agent-reach-external-discovery-and-evidence-v0.1.exec-plan.md` +2. 确认本计划不改变已存在 AgentReach provider / daily aggregate 语义。 +3. 检查当前 `src/externalDiscovery/` 中已有文件： + - `types.ts` + - `paths.ts` + - `aggregate.ts` + - `dailyIntegration.ts` + - `redaction.ts` +4. 确认 `src/externalDiscovery/weeklyWindow.ts` 若不存在，不复用旧名强行迁移；本计划新增 `trendWindow.ts` 作为趋势窗口 owner。 +5. 写或更新结构测试，先让以下约束可被测试捕获： + - canonical path 必须在 `data/external-discovery/windows/`。 + - UI / weekly 不得读取 `data/raw/external-discovery/**`。 + - trend window 不得引入 score component。 +6. 在进入 Phase 1 生产实现前，必须执行 CodeImplementation preflight 并生成 receipt： + - `corepack pnpm run code-implementation:preflight -- --exec-plan docs/specs/exec-plans/agentreach-external-discussion-trend-window-v0.1.exec-plan.md --write` + - `corepack pnpm run code-implementation:preflight -- --exec-plan docs/specs/exec-plans/agentreach-external-discussion-trend-window-v0.1.exec-plan.md --check` +7. 预期 receipt 位置： + - `docs/specs/agent-work/code-implementation-preflight.agentreach-external-discussion-trend-window-v0.1.json` + - 若项目脚本实际生成同名变体 receipt，以脚本输出的 matching receipt 为准，但必须记录在验证记录中。 +8. preflight `--check` 通过前，不得进入 Phase 1 生产代码实现。 +9. 本阶段不改生产逻辑。 + +### Phase 1：类型、路径、public-safe 基座 + +1. 在 `src/externalDiscovery/types.ts` 追加趋势窗口类型和统一读取状态： + - `ExternalDiscussionTrendWindow` + - `ExternalTrendItem` + - `ExternalTrendWindowReadStatus` +2. 在 `src/externalDiscovery/paths.ts` 追加： + - `externalTrendWindowPath(date: string)` + - `externalTrendWindowLatestPath()` +3. 在 `src/externalDiscovery/redaction.ts` 追加： + - `assertPublicSafeTrendWindow(value: unknown)` + - 或复用通用 public-safe scanner 并增加 trend window 测试。 +4. 新增测试： + - `src/__tests__/externalDiscussionTrendWindowTypeContract.test.ts` + - `src/__tests__/externalDiscussionTrendWindowPath.test.ts` + - `src/__tests__/externalDiscussionTrendWindowRedaction.test.ts` +5. 覆盖断言： + - `cannot_be_primary_conclusion` 固定为 `true`。 + - `weekly_gate_reasons` 只能取四个 frozen reason。 + - `ExternalTrendWindowReadStatus` 只能取 `ok` / `not_found` / `parse_error` / `partial` / `insufficient` / `failed` / `skipped`。 + - canonical path 不得平铺在 `data/external-discovery/`。 + - raw text / profile URL / token 类字段会 fail。 +6. 运行： + - `pnpm test -- externalDiscussionTrendWindowTypeContract.test.ts` + - `pnpm test -- externalDiscussionTrendWindowPath.test.ts` + - `pnpm test -- externalDiscussionTrendWindowRedaction.test.ts` + - `pnpm typecheck` + +### Phase 2：趋势窗口 builder + +1. 新增 `src/externalDiscovery/trendWindow.ts`。 +2. 实现 7 日窗口日期计算： + - `window_start = anchor_date - 6 days` + - `window_end = anchor_date` + - `window_days = 7` +3. 实现 aggregate loader： + - 只读取 `data/external-discovery/YYYY-MM-DD.aggregate.json` + - 记录 `expected_dates` + - 记录 `loaded_dates` + - 记录 `missing_dates` + - 记录 `failed_dates` + - 不读取 raw input +4. 实现趋势项归并： + - project evidence 按 `scope:target_key` 合并到 `project_trends` + - direction evidence 按 `scope:target_key` 合并到 `direction_trends` + - `source_aggregate_dates` 记录参与日期 + - `evidence_ids` 保留可追溯引用 +5. 实现字段派生： + - `display_name` + - `target_url` + - `source_count` + - `mention_count_total` + - `active_day_count` + - `platform_count` + - `cross_platform_days` + - `distinct_actor_count` + - `top_tier_actor_count` + - `official_signal` + - `binding_confidence` + - `caveats` +6. 若无法安全派生 `display_name`，使用 `target_key`，不得调用 LLM。 +7. 若无法跨 evidence 安全去重 actor，使用保守口径并增加 caveat。 +8. 新增测试： + - `src/__tests__/externalDiscussionTrendWindowBuilder.test.ts` +9. 覆盖断言： + - missing day 不当作 0。 + - failed aggregate 进入 coverage。 + - `display_name` 不使用 raw title。 + - `official_signal` 只来自 official platform。 + - direction 不进入 project trends。 +10. 运行： + - `pnpm test -- externalDiscussionTrendWindowBuilder.test.ts` + - `pnpm typecheck` + +### Phase 3：rules-first 判断 + +1. 在 `trendWindow.ts` 或独立 `trendRules.ts` 实现 frozen constants： + - `usable_day_count >= 5` 正常判断 + - `usable_day_count = 3-4` partial + - `usable_day_count < 3` insufficient + - `max_daily_share >= 0.70` spike + - `late_avg >= max(early_avg + 1, early_avg * 1.5)` rising + - `early_avg >= max(late_avg + 1, late_avg * 1.5)` 且后两 loaded day 为 0 时 fading + - stable 要求 `active_day_count >= 3` 且 `max_daily_share < 0.60` +2. 实现 components： + - `discussion_volume` + - `persistence` + - `cross_platform_confirmation` + - `actor_authority` + - `binding_confidence` + - `noise_risk` +3. 实现 verdict： + - `external_reinforcement` + - `watch_signal` + - `noise_spike` + - `insufficient` +4. 实现 direction weekly gate： + - 4 选 2 + - `provider_tier_hint` 不得满足 registry gate + - 未达 gate 的 direction 不进入 weekly 正文 +5. 新增测试： + - `src/__tests__/externalDiscussionTrendWindowRules.test.ts` + - `src/__tests__/externalDiscussionTrendWindowDirectionGate.test.ts` +6. 覆盖断言： + - 单日单平台无 official / registry => `noise_spike` + - 多日单平台 => `watch_signal` 但可能非 weekly eligible + - 多日跨平台项目 => `external_reinforcement` + - 方向满足 1 个 gate => `weekly_eligible=false` + - 方向满足 2 个 gate => `weekly_eligible=true` + - provider hint 不产生 `top_tier_actor_count` +7. 运行： + - `pnpm test -- externalDiscussionTrendWindowRules.test.ts` + - `pnpm test -- externalDiscussionTrendWindowDirectionGate.test.ts` + - `pnpm typecheck` + +### Phase 4：artifact 写入与 dry-run + +1. 新增 `src/externalDiscovery/trendWindowIntegration.ts`。 +2. 接入 run-daily 后置步骤： + - 当 daily aggregate 已真实写入时，构建并写入 trend window。 + - 当 `--dry-run` 时，只返回 planned write path。 + - 当 aggregate 缺失、窗口不足或 usable day 不足时，写入 `status="insufficient"` date-specific artifact，并保留 coverage / audit。 + - 当部分平台或部分日期可用但窗口仍可生成时，写入 `status="partial"` date-specific artifact，并保留 partial 平台与日期说明。 + - 当 JSON 解析失败但可以构造 public-safe 失败 artifact 时，写入 `status="failed"` date-specific artifact，并记录 `failed_dates`。 + - 当 public-safe 检查失败时，不写 date-specific artifact，也不更新 latest。 + - 当外部趋势窗口被显式禁用或 external discovery 未启用时，写入 public-safe `status="skipped"` date-specific artifact。 +3. 实现 writer： + - 写入 date-specific file + - public-safe 检查通过后更新 latest pointer。 + - `latest` 在任何成功写入的 public-safe date-specific artifact 后更新，包括 `partial` / `insufficient` / `failed`。 + - `skipped` 只有在 public-safe skipped artifact 成功写入后才更新 latest。 + - public-safe 检查失败时不写 date-specific，也不写 latest。 +4. 不改 raw input 读取逻辑。 +5. 新增测试： + - `src/__tests__/externalDiscussionTrendWindowWrite.test.ts` + - `src/__tests__/externalDiscussionTrendWindowDryRun.test.ts` +6. 覆盖断言： + - date-specific path 正确。 + - latest 只在 public-safe date-specific artifact 成功写入后更新。 + - `partial` / `insufficient` / `failed` 成功写入时也更新 latest。 + - aggregate missing / window insufficient / usable day insufficient 固定写 `insufficient` artifact，不回退为空列表。 + - parse failure 可构造 public-safe failure artifact 时固定写 `failed` artifact。 + - dry-run 不写任何 trend window 文件。 + - public-safe fail 不写 date-specific，也不写 latest。 +7. 运行： + - `pnpm test -- externalDiscussionTrendWindowWrite.test.ts` + - `pnpm test -- externalDiscussionTrendWindowDryRun.test.ts` + - `pnpm typecheck` + +### Phase 5：weekly / run-summary / verify 消费 + +1. `runSummary` 增加： + - trend window status + - trend window read status：`trend_window_read_status` + - trend window path + - usable day count + - project trend count + - direction trend count + - failed / missing date count +2. weekly 增加 external trend consumption： + - 读取 canonical trend window。 + - date-specific missing 时表达 `not_found`，不得静默读取 `latest`。 + - JSON 解析失败或 schema mismatch 时表达 `parse_error`，不得渲染成空列表。 + - artifact 内部 `status` 透传为 `ok` / `partial` / `insufficient` / `failed` / `skipped`。 + - 不回退读取 raw input。 + - `noise_spike` 不进入正向趋势材料。 + - `direction_trends` 只有 `weekly_eligible=true` 才进入 direction observation。 +3. verify 增加： + - trend window public-safe 检查。 + - `ExternalTrendWindowReadStatus` 映射检查。 + - date-specific route 不得 fallback 到 latest。 + - trend window 不得污染 score。 + - direction trend 不得写成 project conclusion。 + - weekly / UI 不得读取 raw input。 +4. 新增测试： + - `src/__tests__/externalDiscussionTrendWindowActionOutput.test.ts` + - `src/__tests__/externalDiscussionTrendWindowWeekly.test.ts` + - `src/__tests__/externalDiscussionTrendWindowVerification.test.ts` +5. 运行： + - `pnpm test -- externalDiscussionTrendWindowActionOutput.test.ts` + - `pnpm test -- externalDiscussionTrendWindowWeekly.test.ts` + - `pnpm test -- externalDiscussionTrendWindowVerification.test.ts` + - `pnpm typecheck` + +### Phase 6：/agentreach UI read-only 展示 + +1. 在 `src/visualConsole/build.ts` 或 AgentReach view-model adapter 中读取 trend window。 +2. 新增 view-model 字段： + - `trend_window_read_status` + - `trend_window_status` + - `trend_window_summary` + - `selected_candidate_trend` + - `direction_trend_items` +3. 候选详情展示： + - `近 7 日外部讨论` + - `momentum` + - `active_day_count` + - `platform_count` + - `verdict` + - caveats +4. 展开区展示： + - components + - gate reasons + - named registry actors + - insufficient / partial 状态说明 + - `not_found`：显示“指定日期暂无近 7 日外部讨论窗口”。 + - `parse_error`：显示“趋势窗口文件无法解析，请检查产物”。 + - `insufficient`：显示“近 7 日可用外部数据不足，暂不判断趋势”。 +5. UI 禁止： + - 不显示 `score` 作为主指标。 + - 不把 `partial` 写成无信号。 + - 不把 `not_found` / `parse_error` / `insufficient` 渲染成相同空态。 + - date-specific 页面不得 fallback 到 latest。 + - 不把空 `named_registry_actors` 写成没人讨论。 + - 不在 React / SSR 中重新计算 trend。 +6. 新增测试： + - `src/__tests__/agentReachTrendWindowViewModel.test.ts` + - `app/client/__tests__/AgentReachTrendWindowView.test.tsx` 或项目现有 React 测试路径 +7. 运行： + - `pnpm test -- agentReachTrendWindowViewModel.test.ts` + - `pnpm test -- AgentReachTrendWindowView.test.tsx` + - `pnpm typecheck` + +### Phase 7：文档与总体验收 + +1. 更新 `data/README.md`： + - 说明 trend window public-safe 派生 artifact。 + - 说明 raw input 仍 local-only。 +2. 更新 `docs/specs/services/action-output.md`： + - weekly 可展示 external discussion trend secondary evidence。 +3. 更新 `docs/specs/services/visual-console-testing-manual.md`： + - `/agentreach` trend window read-only、not_found / partial / insufficient 验收。 +4. 如 `docs/specs/services/scoring-engine.md` 尚未明确 external trend 不入 score，补充不变量。 +5. 运行总体验收： + - `pnpm test -- externalDiscussionTrendWindow` + - `pnpm typecheck` + - `pnpm test -- externalDiscovery` + - `pnpm run-daily -- --date 2026-07-03 --dry-run --no-external-discovery` + - `pnpm run-weekly -- --date 2026-07-03 --dry-run --no-external-discovery` +6. 检查 git diff： + - 不得出现 `data/external-discovery/` 运行产物被提交。 + - 不得出现 raw external input。 + - 不得出现主 score 公式改动。 + - 不得出现 `RawSignal.source` 新增 external provider。 + +## 验收标准 + +1. 能基于最近 7 天 `DailyExternalAggregate[]` 生成 `ExternalDiscussionTrendWindow`。 +2. 趋势窗口只写入 `data/external-discovery/windows/YYYY-MM-DD.discussion-trend-window.json` 与 `latest.discussion-trend-window.json`。 +3. `dry-run` 不写 date-specific trend window 或 latest 指针。 +4. 缺失日期进入 coverage，不当作 0。 +5. partial aggregate 会让窗口状态或 caveats 体现 partial。 +6. `display_name / target_url / source_count / official_signal / binding_confidence` 来源符合设计字段派生表。 +7. 单日单平台爆发输出 `noise_spike`，不进入 weekly 正向材料。 +8. 多日或跨平台项目可输出 `external_reinforcement`。 +9. 方向级趋势必须满足 4 选 2 gate 才能 `weekly_eligible=true`。 +10. `provider_tier_hint` 不得产生 registry participation。 +11. `named_registry_actors` 只来自 public-safe registry 命中。 +12. 所有趋势项都带 `cannot_be_primary_conclusion=true`。 +13. weekly 只消费 trend window 或 public aggregate，不读取 raw input。 +14. weekly 不改变主榜 score、ranking、discussion_score。 +15. UI 只读展示 trend window，不在 UI 侧重算。 +16. public-safe / redaction 检查通过。 +17. `partial` / `insufficient` / `failed` / `skipped` 的 artifact 写入语义固定，不能被实现阶段改成空列表或静默跳过。 +18. `latest` 只在 public-safe date-specific artifact 成功写入后更新，且覆盖 `partial` / `insufficient` / `failed` 成功写入场景。 +19. UI / weekly / verify 共用 `ExternalTrendWindowReadStatus`，并区分 `not_found` / `parse_error` / `insufficient`。 +20. date-specific 读取路径不得 fallback 到 latest，latest 路径不得读取 date-specific 或 raw input。 + +## 验证矩阵 + +| 文件位置或类型 | 验证内容 | 命令 | 通过标准 | +| --- | --- | --- | --- | +| `src/externalDiscovery/types.ts` | trend window 类型、枚举、`cannot_be_primary_conclusion` | `pnpm test -- externalDiscussionTrendWindowTypeContract.test.ts` | 字段和枚举与设计一致 | +| `src/externalDiscovery/types.ts`、consumer adapter | `ExternalTrendWindowReadStatus` 冻结与映射 | `pnpm test -- externalDiscussionTrendWindowTypeContract.test.ts`、`pnpm test -- externalDiscussionTrendWindowReadStatus.test.ts` | UI / weekly / verify 共用状态，`not_found` / `parse_error` 不被吞掉 | +| `src/externalDiscovery/paths.ts` | canonical windows path / latest path | `pnpm test -- externalDiscussionTrendWindowPath.test.ts` | 只允许 `data/external-discovery/windows/` | +| `src/externalDiscovery/redaction.ts` | public-safe trend window | `pnpm test -- externalDiscussionTrendWindowRedaction.test.ts` | raw text / handle / profile URL / token 类字段 fail | +| `src/externalDiscovery/trendWindow.ts` | 7 日合并、coverage、字段派生 | `pnpm test -- externalDiscussionTrendWindowBuilder.test.ts` | missing / failed / partial / field derivation 全覆盖 | +| `src/externalDiscovery/trendWindow.ts` 或 `trendRules.ts` | momentum / verdict / noise risk | `pnpm test -- externalDiscussionTrendWindowRules.test.ts` | spike/rising/fading/stable/insufficient 可复现 | +| `src/externalDiscovery/trendWindow.ts` | direction 4 选 2 gate | `pnpm test -- externalDiscussionTrendWindowDirectionGate.test.ts` | 未达 gate 不进 weekly | +| `src/externalDiscovery/trendWindowIntegration.ts` | write / latest / dry-run | `pnpm test -- externalDiscussionTrendWindowWrite.test.ts`、`pnpm test -- externalDiscussionTrendWindowDryRun.test.ts` | dry-run 不写，latest 只在 public-safe date-specific 成功写入后更新 | +| `src/externalDiscovery/trendWindowIntegration.ts` | `partial` / `insufficient` / `failed` / `skipped` artifact 写入语义 | `pnpm test -- externalDiscussionTrendWindowWrite.test.ts` | public-safe 状态产物写入规则和 latest 更新规则固定 | +| `src/action/*weekly*` | weekly secondary consumption | `pnpm test -- externalDiscussionTrendWindowWeekly.test.ts` | 不读 raw input，不污染主结论 | +| `src/action/*weekly*`、`src/action/dailyVerification.ts`、`src/visualConsole/build.ts` | date-specific read 不 fallback 到 latest | `pnpm test -- externalDiscussionTrendWindowReadStatus.test.ts` | date route 只读指定日期，latest route 只读 latest | +| `src/action/dailyVerification.ts` | verify 防污染和 public-safe | `pnpm test -- externalDiscussionTrendWindowVerification.test.ts` | score pollution / redaction fail 被拦截 | +| `src/visualConsole/build.ts`、AgentReach UI | read-only 展示 | `pnpm test -- agentReachTrendWindowViewModel.test.ts`、React 测试 | UI 不重算趋势，状态文案正确 | +| 全仓 | 类型和回归 | `pnpm typecheck`、`pnpm test -- externalDiscovery` | 聚焦测试通过 | + +## 回滚策略 + +1. 若 trend window builder 失败，禁用 trend window 写入，保留 daily aggregate 和现有 `/agentreach` 今日发现展示。 +2. 若 weekly 消费造成语义污染，回滚 weekly integration，仅保留 trend window artifact。 +3. 若 UI 展示过重或误导，隐藏 trend window UI 区块，保留后端 artifact。 +4. 任意回滚不得恢复 raw input 读取、不得改主 score、不得提交运行产物。 + +## 当前残余风险 + +- 真实外部讨论稀疏时，7 日窗口可能常见 `insufficient`；这是产品语义上的保守降级，不应通过放宽规则解决。 +- 当前 `DailyExternalAggregate` 若缺少 target display name / URL，V1 只能按设计派生表保守展示；不能用 raw title 或 LLM 补齐。 +- 如果上游 X / Reddit 长期 partial，趋势窗口可用性会下降，但不能把 partial 误写成无信号。 +- UI 具体组件路径取决于当前 AgentReach 页面实现位置，exec-plan 实现阶段需先定位文件，不能顺手改其他页面风格。 + +## 下一阶段入口 + +本 exec-plan 已通过审核并进入 `Approved for Implementation`。实现前必须先执行 Phase 0 的 CodeImplementation preflight，receipt 检查通过后，再按 Phase 1 -> Phase 7 顺序执行代码实现。 + +## 验证记录 + +| 日期 | 命令 | 结果 | 备注 | +| --- | --- | --- | --- | +| 2026-07-03 | 未运行 | `Not Started` | 本轮只生成 exec-plan，未修改实现代码 | +| 2026-07-03 | `corepack pnpm run code-implementation:preflight -- --exec-plan docs/specs/exec-plans/agentreach-external-discussion-trend-window-v0.1.exec-plan.md --write` | `PASS` | 生成 implementation preflight receipt | +| 2026-07-03 | `corepack pnpm run code-implementation:preflight -- --exec-plan docs/specs/exec-plans/agentreach-external-discussion-trend-window-v0.1.exec-plan.md --check` | `PASS` | receipt 校验通过，可以进入 Phase 1 | +| 2026-07-03 | `corepack pnpm exec vitest run src/__tests__/externalDiscussionTrendWindowTypeContract.test.ts` | `PASS` | Phase 1 类型与枚举契约 | +| 2026-07-03 | `corepack pnpm exec vitest run src/__tests__/externalDiscussionTrendWindowPath.test.ts` | `PASS` | Phase 1 canonical windows path | +| 2026-07-03 | `corepack pnpm exec vitest run src/__tests__/externalDiscussionTrendWindowRedaction.test.ts` | `PASS` | Phase 1 public-safe trend window | +| 2026-07-03 | `corepack pnpm typecheck` | `PASS` | Phase 1 类型检查 | +| 2026-07-03 | `corepack pnpm exec vitest run src/__tests__/externalDiscussionTrendWindowBuilder.test.ts` | `PASS` | Phase 2 builder、coverage、字段派生 | +| 2026-07-03 | `corepack pnpm exec vitest run src/__tests__/externalDiscussionTrendWindowRules.test.ts` | `PASS` | Phase 3 momentum / verdict / noise risk | +| 2026-07-03 | `corepack pnpm exec vitest run src/__tests__/externalDiscussionTrendWindowDirectionGate.test.ts` | `PASS` | Phase 3 direction 4 选 2 gate | +| 2026-07-03 | `corepack pnpm typecheck` | `PASS` | Phase 2/3 类型检查 | +| 2026-07-03 | `corepack pnpm exec vitest run src/__tests__/externalDiscussionTrendWindowWrite.test.ts` | `PASS` | Phase 4 write/latest/public-safe | +| 2026-07-03 | `corepack pnpm exec vitest run src/__tests__/externalDiscussionTrendWindowDryRun.test.ts` | `PASS` | Phase 4 dry-run 不落盘 | +| 2026-07-03 | `corepack pnpm exec vitest run src/__tests__/externalDiscoveryDailyIntegration.test.ts` | `PASS` | Phase 4 daily integration 写入趋势窗口 | +| 2026-07-03 | `corepack pnpm typecheck` | `PASS` | Phase 4 类型检查 | +| 2026-07-03 | `corepack pnpm exec vitest run src/__tests__/externalDiscussionTrendWindowReadStatus.test.ts src/__tests__/externalDiscussionTrendWindowActionOutput.test.ts src/__tests__/externalDiscussionTrendWindowVerification.test.ts src/__tests__/externalDiscussionTrendWindowWeekly.test.ts` | `PASS` | Phase 5 read-status / run-summary / verify / weekly secondary consumption | +| 2026-07-03 | `corepack pnpm exec vitest run src/__tests__/externalDiscussionTrendWindowTypeContract.test.ts src/__tests__/externalDiscussionTrendWindowPath.test.ts src/__tests__/externalDiscussionTrendWindowRedaction.test.ts src/__tests__/externalDiscussionTrendWindowBuilder.test.ts src/__tests__/externalDiscussionTrendWindowRules.test.ts src/__tests__/externalDiscussionTrendWindowDirectionGate.test.ts src/__tests__/externalDiscussionTrendWindowWrite.test.ts src/__tests__/externalDiscussionTrendWindowDryRun.test.ts src/__tests__/externalDiscoveryDailyIntegration.test.ts` | `PASS` | Phase 1-4 focused regression after Phase 5 | +| 2026-07-03 | `corepack pnpm typecheck` | `PASS` | Phase 5 typecheck | +| 2026-07-03 | `corepack pnpm exec vitest run src/__tests__/agentReachTrendWindowViewModel.test.ts` | `PASS` | Phase 6 /agentreach date-specific aggregate + trend window view-model and SSR rendering | +| 2026-07-03 | `corepack pnpm exec vitest run src/__tests__/externalDiscussionTrendWindowReadStatus.test.ts src/__tests__/externalDiscussionTrendWindowActionOutput.test.ts src/__tests__/externalDiscussionTrendWindowVerification.test.ts src/__tests__/externalDiscussionTrendWindowWeekly.test.ts src/__tests__/externalDiscussionTrendWindowTypeContract.test.ts src/__tests__/externalDiscussionTrendWindowPath.test.ts src/__tests__/externalDiscussionTrendWindowRedaction.test.ts src/__tests__/externalDiscussionTrendWindowBuilder.test.ts src/__tests__/externalDiscussionTrendWindowRules.test.ts src/__tests__/externalDiscussionTrendWindowDirectionGate.test.ts src/__tests__/externalDiscussionTrendWindowWrite.test.ts src/__tests__/externalDiscussionTrendWindowDryRun.test.ts src/__tests__/externalDiscoveryDailyIntegration.test.ts src/__tests__/agentReachTrendWindowViewModel.test.ts` | `PASS` | Phase 6 focused regression across trend window builder, consumers, and UI read model | +| 2026-07-03 | `corepack pnpm typecheck` | `PASS` | Phase 6 typecheck | +| 2026-07-04 | `corepack pnpm exec vitest run src/__tests__/agentReachTrendWindowViewModel.test.ts` | `PASS` | Phase 7 CLI text view coverage for `visual-console --view agentreach` | +| 2026-07-04 | `corepack pnpm exec vitest run src/__tests__/externalDiscussionTrendWindowReadStatus.test.ts src/__tests__/externalDiscussionTrendWindowActionOutput.test.ts src/__tests__/externalDiscussionTrendWindowVerification.test.ts src/__tests__/externalDiscussionTrendWindowWeekly.test.ts src/__tests__/externalDiscussionTrendWindowTypeContract.test.ts src/__tests__/externalDiscussionTrendWindowPath.test.ts src/__tests__/externalDiscussionTrendWindowRedaction.test.ts src/__tests__/externalDiscussionTrendWindowBuilder.test.ts src/__tests__/externalDiscussionTrendWindowRules.test.ts src/__tests__/externalDiscussionTrendWindowDirectionGate.test.ts src/__tests__/externalDiscussionTrendWindowWrite.test.ts src/__tests__/externalDiscussionTrendWindowDryRun.test.ts src/__tests__/externalDiscoveryDailyIntegration.test.ts src/__tests__/agentReachTrendWindowViewModel.test.ts` | `PASS` | Phase 7 focused regression after docs and CLI text view sync | +| 2026-07-04 | `corepack pnpm typecheck` | `PASS` | Phase 7 typecheck | +| 2026-07-04 | `corepack pnpm visual-console -- --view agentreach --date 2026-07-03` | `PASS` | Read-only CLI smoke; current workspace had no 2026-07-03 external aggregate/window, so output correctly reported `not_found` without writing artifacts | diff --git a/docs/specs/services/action-output.md b/docs/specs/services/action-output.md index 1b71ce4..013612d 100644 --- a/docs/specs/services/action-output.md +++ b/docs/specs/services/action-output.md @@ -69,6 +69,16 @@ Weekly report MUST 回答： 输出必须抽象为趋势，例如 `agent runtime + persistent memory + self-improving`，不能只列 repo。 +### AgentReach 外部讨论趋势 + +Weekly report MAY 消费 `data/external-discovery/windows/YYYY-MM-DD.discussion-trend-window.json`，但只能作为 external secondary evidence： + +- 只展示 `weekly_eligible=true` 且 `verdict=external_reinforcement` 的项目级趋势补证。 +- 只展示 `weekly_eligible=true` 且 `verdict=watch_signal` 的方向级观察。 +- `noise_spike` 只能进入风险、噪声或 audit 说明，不能作为正向趋势材料。 +- `not_found`、`parse_error`、`insufficient`、`partial`、`failed`、`skipped` 必须保留各自语义，不能渲染成同一种空列表。 +- Action 层不得回退读取 `data/raw/external-discovery/**`，不得把外部趋势写入 primary score、ranking 或 `discussion_score`。 + ## 失败模式 | 失败 | 语义 | 预期处理 | @@ -103,6 +113,7 @@ Weekly report MUST 回答： | --- | --- | --- | --- | | `src/action/dailyReport.ts` | 新项目、高分、异常增长、evidence 输出 | 单元测试 + dry-run | 本文件「Daily Report 契约」 | | `src/action/weeklyReport.ts` | 三个周报问题是否被回答 | 单元测试 + 报告审查 | 本文件「Weekly Report 契约」 | +| `src/action/*weekly*` | AgentReach 外部讨论趋势是否只作为 secondary evidence | `externalDiscussionTrendWindowWeekly.test.ts` | 本文件「AgentReach 外部讨论趋势」 | | `src/action/knowledgeCard.ts` | card 字段是否完整、机器区/人工区是否隔离 | 单元测试 | 本文件「Knowledge Card 契约」「KB 更新契约」 | | `data/reports/*.md` | 人类可读报告是否生成 | 冒烟检查 | 本文件「输出」 | | `data/kb/*.md` | KB card 是否生成且重入更新不覆盖人工区 | 冒烟检查 + 重入测试 | 本文件「KB 构建」「KB 重入更新」 | diff --git a/docs/specs/services/scoring-engine.md b/docs/specs/services/scoring-engine.md index fcb89a9..eb07b91 100644 --- a/docs/specs/services/scoring-engine.md +++ b/docs/specs/services/scoring-engine.md @@ -61,6 +61,7 @@ Score = w1 * star_velocity - rules-only 模式 MUST 在无 LLM key 时仍可运行。 - 如果启用 LLM，Scoring Engine MUST 只消费强类型 classification 字段，不得消费自由文本结论来直接判定分数。 - `autonomy_score`、`compounding_capability`、`architecture_shift` 的最终分数 MUST 由 TS 规则层根据 structured evidence 计算。 +- AgentReach 外部讨论趋势窗口只能作为展示层和 weekly secondary evidence；不得写入 `RawSignal[]`、不得改写 `discussion_score`、不得改变主榜排序或 `ScoreBreakdown.total_score`。 ## Fake Star 防护 diff --git a/src/__tests__/externalDiscoveryDailyIntegration.test.ts b/src/__tests__/externalDiscoveryDailyIntegration.test.ts index fac8df2..0d7dfba 100644 --- a/src/__tests__/externalDiscoveryDailyIntegration.test.ts +++ b/src/__tests__/externalDiscoveryDailyIntegration.test.ts @@ -91,8 +91,11 @@ describe("external discovery daily integration", () => { expect(result.aggregate.accepted_event_count).toBe(1); expect(result.input_build.eligible_count).toBe(1); expect(result.explanations.status).toBe("skipped"); + expect(result.trend_window.status).toBe("insufficient"); expect(fs.existsSync(path.join(root, "data", "external-discovery", "2026-06-30.aggregate.json"))).toBe(true); expect(fs.existsSync(path.join(root, "data", "external-discovery", "2026-06-30.candidate-explanations.json"))).toBe(true); + expect(fs.existsSync(path.join(root, "data", "external-discovery", "windows", "2026-06-30.discussion-trend-window.json"))).toBe(true); + expect(fs.existsSync(path.join(root, "data", "external-discovery", "windows", "latest.discussion-trend-window.json"))).toBe(true); expect(result.explanations.explanations[0]?.why_watch_cn).toContain("HN"); }); @@ -116,7 +119,9 @@ describe("external discovery daily integration", () => { expect(result.aggregate.accepted_event_count).toBe(1); expect(result.explanations.status).toBe("failed"); expect(result.explanations.status_reason).toBe("candidate_explanation_generation_failed"); + expect(result.trend_window.status).toBe("insufficient"); expect(fs.existsSync(path.join(root, "data", "external-discovery", "2026-06-30.aggregate.json"))).toBe(true); expect(fs.existsSync(path.join(root, "data", "external-discovery", "2026-06-30.candidate-explanations.json"))).toBe(true); + expect(fs.existsSync(path.join(root, "data", "external-discovery", "windows", "2026-06-30.discussion-trend-window.json"))).toBe(true); }); }); diff --git a/src/__tests__/externalDiscussionTrendWindowActionOutput.test.ts b/src/__tests__/externalDiscussionTrendWindowActionOutput.test.ts new file mode 100644 index 0000000..5ca2647 --- /dev/null +++ b/src/__tests__/externalDiscussionTrendWindowActionOutput.test.ts @@ -0,0 +1,88 @@ +import { describe, expect, it } from "vitest"; +import { renderDailyRunSummary } from "../action/runSummary.ts"; +import type { DailyRunSummary } from "../types.ts"; + +function makeSummary(): DailyRunSummary { + return { + date: "2026-07-03", + generated_at: "2026-07-03T00:00:00.000Z", + dry_run: true, + minimum_viable_run_completed: true, + completion_notes: [], + counts: { + raw_signals: 1, + normalized_projects: 1, + scored_projects: 1, + high_score_projects: 0, + anomaly_projects: 0, + new_projects: 1, + classifications: 0, + }, + source_status: [], + quality: { + missing_descriptions: 0, + watchlist_hits: 0, + low_confidence_projects: 0, + medium_confidence_projects: 0, + insufficient_metrics_projects: 0, + suspicious_growth_projects: 0, + single_source_projects: 0, + single_spike_projects: 0, + emerging_projects: 1, + persistent_projects: 0, + }, + diagnostics: { + anomaly_share: 0, + uniform_star_velocity_detected: false, + metrics_source_distribution: { embedded: 1, github_api: 0, github_html: 0, github_cache: 0, unavailable: 0 }, + star_delta_source_distribution: { github_live: 0, github_snapshot: 0, signal: 1, unavailable: 0 }, + github_star_delta: { + live_delta_attempts: 0, + live_delta_success: 0, + snapshot_delta_success: 0, + token_missing: 0, + rate_limit: 0, + network_blocked: 0, + }, + }, + top_projects: [], + external_discovery: { + aggregate_status: "ok", + accepted_event_count: 3, + rejected_event_count: 0, + observation_candidate_count: 1, + project_evidence_count: 1, + direction_evidence_count: 0, + explanation_status: "ok", + explanation_eligible_count: 1, + explanation_attempted_count: 1, + explanation_enhanced_count: 1, + explanation_fallback_count: 0, + explanation_rejected_count: 0, + trend_window_read_status: "ok", + trend_window_status: "ok", + trend_window_path: "data/external-discovery/windows/2026-07-03.discussion-trend-window.json", + trend_window_usable_day_count: 7, + trend_window_project_trend_count: 1, + trend_window_direction_trend_count: 0, + trend_window_failed_date_count: 0, + trend_window_missing_date_count: 0, + warning_count: 0, + warnings: [], + }, + observer_top_candidates: [], + watchouts: [], + next_focus: [], + recommended_actions: [], + }; +} + +describe("external discussion trend window action output", () => { + it("renders trend window read status and coverage in daily run summary", () => { + const rendered = renderDailyRunSummary(makeSummary()); + expect(rendered).toContain("trend_window_read_status: ok"); + expect(rendered).toContain("trend_window_status: ok"); + expect(rendered).toContain("trend_window_coverage: usable_days=7; failed_dates=0; missing_dates=0"); + expect(rendered).toContain("trend_window_items: project=1; direction=0"); + }); +}); diff --git a/src/__tests__/externalDiscussionTrendWindowBuilder.test.ts b/src/__tests__/externalDiscussionTrendWindowBuilder.test.ts new file mode 100644 index 0000000..9dab58a --- /dev/null +++ b/src/__tests__/externalDiscussionTrendWindowBuilder.test.ts @@ -0,0 +1,136 @@ +import { describe, expect, it } from "vitest"; +import { buildExternalDiscussionTrendWindow } from "../externalDiscovery/trendWindow.ts"; +import { aggregateForDate, loadedWindow, trendEvidence, TREND_TEST_DATES } from "./externalDiscussionTrendWindowFixtures.ts"; + +describe("external discussion trend window builder", () => { + it("builds a 7-day public-safe project trend from daily aggregates", () => { + const results = loadedWindow({ + "2026-07-01": aggregateForDate({ + date: "2026-07-01", + project_evidence: [ + trendEvidence({ + scope: "project", + target_key: "pydantic/pydantic-ai", + date: "2026-07-01", + mention_count: 1, + platforms: ["hacker_news"], + }), + ], + }), + "2026-07-02": aggregateForDate({ + date: "2026-07-02", + project_evidence: [ + trendEvidence({ + scope: "project", + target_key: "pydantic/pydantic-ai", + date: "2026-07-02", + mention_count: 2, + platforms: ["reddit", "official_blog"], + }), + ], + }), + }); + + const window = buildExternalDiscussionTrendWindow({ + anchorDate: "2026-07-03", + generatedAt: "2026-07-03T00:00:00.000Z", + aggregateResults: results, + }); + + expect(window.schema_version).toBe("external-discussion-trend-window.v1"); + expect(window.window_start).toBe("2026-06-27"); + expect(window.window_end).toBe("2026-07-03"); + expect(window.coverage.expected_dates).toEqual(TREND_TEST_DATES); + expect(window.coverage.usable_day_count).toBe(7); + expect(window.public_safe).toBe(true); + + const trend = window.project_trends[0]; + expect(trend).toMatchObject({ + scope: "project", + target_key: "pydantic/pydantic-ai", + display_name: "pydantic/pydantic-ai", + target_url: "https://github.com/pydantic/pydantic-ai", + official_signal: true, + mention_count_total: 3, + source_count: 3, + active_day_count: 2, + platform_count: 3, + cannot_be_primary_conclusion: true, + }); + expect(window.direction_trends).toEqual([]); + expect(trend?.daily_counts.find((count) => count.date === "2026-07-01")?.mention_count).toBe(1); + expect(trend?.daily_counts.find((count) => count.date === "2026-07-02")?.mention_count).toBe(2); + }); + + it("keeps missing days in coverage instead of treating them as zero signal", () => { + const results = loadedWindow({ + "2026-07-02": aggregateForDate({ + date: "2026-07-02", + project_evidence: [trendEvidence({ scope: "project", target_key: "openai/agents", date: "2026-07-02", mention_count: 2 })], + }), + }).map((result) => + result.date === "2026-07-01" + ? { status: "missing" as const, date: result.date, path: result.path } + : result, + ); + + const window = buildExternalDiscussionTrendWindow({ + anchorDate: "2026-07-03", + generatedAt: "2026-07-03T00:00:00.000Z", + aggregateResults: results, + }); + + expect(window.status).toBe("partial"); + expect(window.coverage.missing_dates).toEqual(["2026-07-01"]); + expect(window.project_trends[0]?.daily_counts.some((count) => count.date === "2026-07-01")).toBe(false); + expect(window.audit.warnings.some((warning) => warning.reason_code === "window_aggregate_missing")).toBe(true); + }); + + it("does not infer official signals from non-official platforms", () => { + const window = buildExternalDiscussionTrendWindow({ + anchorDate: "2026-07-03", + generatedAt: "2026-07-03T00:00:00.000Z", + aggregateResults: loadedWindow({ + "2026-07-01": aggregateForDate({ + date: "2026-07-01", + project_evidence: [ + trendEvidence({ + scope: "project", + target_key: "anthropics/claude-code", + date: "2026-07-01", + mention_count: 2, + platforms: ["hacker_news", "reddit"], + }), + ], + }), + }), + }); + + expect(window.project_trends[0]?.official_signal).toBe(false); + }); + + it("keeps direction evidence out of project trends", () => { + const window = buildExternalDiscussionTrendWindow({ + anchorDate: "2026-07-03", + generatedAt: "2026-07-03T00:00:00.000Z", + aggregateResults: loadedWindow({ + "2026-07-01": aggregateForDate({ + date: "2026-07-01", + direction_evidence: [ + trendEvidence({ + scope: "direction", + target_key: "agent-runtime", + date: "2026-07-01", + mention_count: 2, + platforms: ["hacker_news", "reddit"], + }), + ], + }), + }), + }); + + expect(window.project_trends).toEqual([]); + expect(window.direction_trends[0]?.scope).toBe("direction"); + expect(window.direction_trends[0]?.binding_confidence).toBe("low"); + }); +}); diff --git a/src/__tests__/externalDiscussionTrendWindowDirectionGate.test.ts b/src/__tests__/externalDiscussionTrendWindowDirectionGate.test.ts new file mode 100644 index 0000000..dc16950 --- /dev/null +++ b/src/__tests__/externalDiscussionTrendWindowDirectionGate.test.ts @@ -0,0 +1,120 @@ +import { describe, expect, it } from "vitest"; +import { buildExternalDiscussionTrendWindow } from "../externalDiscovery/trendWindow.ts"; +import { aggregateForDate, loadedWindow, trendEvidence } from "./externalDiscussionTrendWindowFixtures.ts"; + +describe("external discussion trend window direction weekly gate", () => { + it("does not make a one-gate direction weekly eligible", () => { + const window = buildExternalDiscussionTrendWindow({ + anchorDate: "2026-07-03", + generatedAt: "2026-07-03T00:00:00.000Z", + aggregateResults: loadedWindow({ + "2026-07-02": aggregateForDate({ + date: "2026-07-02", + direction_evidence: [ + trendEvidence({ + scope: "direction", + target_key: "agent-runtime", + date: "2026-07-02", + mention_count: 2, + platforms: ["hacker_news"], + distinct_actor_count: 1, + }), + ], + }), + }), + }); + + const trend = window.direction_trends[0]; + expect(trend?.weekly_gate_reasons).toEqual([]); + expect(trend?.weekly_eligible).toBe(false); + expect(trend?.verdict).not.toBe("external_reinforcement"); + }); + + it("allows direction weekly eligibility only after two frozen gates", () => { + const window = buildExternalDiscussionTrendWindow({ + anchorDate: "2026-07-03", + generatedAt: "2026-07-03T00:00:00.000Z", + aggregateResults: loadedWindow({ + "2026-07-01": aggregateForDate({ + date: "2026-07-01", + direction_evidence: [ + trendEvidence({ + scope: "direction", + target_key: "agent-runtime", + date: "2026-07-01", + mention_count: 1, + platforms: ["hacker_news"], + distinct_actor_count: 1, + }), + ], + }), + "2026-07-02": aggregateForDate({ + date: "2026-07-02", + direction_evidence: [ + trendEvidence({ + scope: "direction", + target_key: "agent-runtime", + date: "2026-07-02", + mention_count: 2, + platforms: ["reddit"], + distinct_actor_count: 2, + }), + ], + }), + }), + }); + + const trend = window.direction_trends[0]; + expect(trend?.verdict).toBe("watch_signal"); + expect(trend?.weekly_eligible).toBe(true); + expect(trend?.weekly_gate_reasons).toEqual([ + "cross_platform_confirmation", + "multi_actor_confirmation", + "multi_day_persistence", + ]); + }); + + it("does not treat provider tier hints as registry participation", () => { + const window = buildExternalDiscussionTrendWindow({ + anchorDate: "2026-07-03", + generatedAt: "2026-07-03T00:00:00.000Z", + aggregateResults: loadedWindow({ + "2026-07-01": aggregateForDate({ + date: "2026-07-01", + direction_evidence: [ + trendEvidence({ + scope: "direction", + target_key: "memory-agent", + date: "2026-07-01", + mention_count: 1, + platforms: ["hacker_news"], + distinct_actor_count: 1, + top_tier_actor_count: 0, + named_registry_actors: [], + }), + ], + }), + "2026-07-02": aggregateForDate({ + date: "2026-07-02", + direction_evidence: [ + trendEvidence({ + scope: "direction", + target_key: "memory-agent", + date: "2026-07-02", + mention_count: 1, + platforms: ["hacker_news"], + distinct_actor_count: 1, + top_tier_actor_count: 0, + named_registry_actors: [], + }), + ], + }), + }), + }); + + const trend = window.direction_trends[0]; + expect(trend?.weekly_gate_reasons).toEqual(["multi_day_persistence"]); + expect(trend?.weekly_gate_reasons).not.toContain("registry_tier_participation"); + expect(trend?.top_tier_actor_count).toBe(0); + }); +}); diff --git a/src/__tests__/externalDiscussionTrendWindowDryRun.test.ts b/src/__tests__/externalDiscussionTrendWindowDryRun.test.ts new file mode 100644 index 0000000..e968915 --- /dev/null +++ b/src/__tests__/externalDiscussionTrendWindowDryRun.test.ts @@ -0,0 +1,45 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { buildSkippedExternalDiscussionTrendWindow } from "../externalDiscovery/trendWindow.ts"; +import { writeExternalDiscussionTrendWindow } from "../externalDiscovery/trendWindowIntegration.ts"; + +const roots: string[] = []; +const originalCwd = process.cwd(); + +afterEach(() => { + process.chdir(originalCwd); + for (const root of roots.splice(0)) { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +function setupWorkspace(): string { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "external-trend-window-dry-run-")); + roots.push(root); + process.chdir(root); + return root; +} + +describe("external discussion trend window dry-run", () => { + it("returns planned paths without writing date-specific or latest files", () => { + const root = setupWorkspace(); + const trendWindow = buildSkippedExternalDiscussionTrendWindow({ + anchorDate: "2026-07-03", + generatedAt: "2026-07-03T00:00:00.000Z", + statusReason: "external_discovery_disabled", + }); + + const paths = writeExternalDiscussionTrendWindow({ + date: "2026-07-03", + trendWindow, + dryRun: true, + }); + + expect(paths.trend_window.replace(/\\/g, "/")).toBe("data/external-discovery/windows/2026-07-03.discussion-trend-window.json"); + expect(paths.trend_window_latest.replace(/\\/g, "/")).toBe("data/external-discovery/windows/latest.discussion-trend-window.json"); + expect(fs.existsSync(path.join(root, paths.trend_window))).toBe(false); + expect(fs.existsSync(path.join(root, paths.trend_window_latest))).toBe(false); + }); +}); diff --git a/src/__tests__/externalDiscussionTrendWindowFixtures.ts b/src/__tests__/externalDiscussionTrendWindowFixtures.ts new file mode 100644 index 0000000..b8f954f --- /dev/null +++ b/src/__tests__/externalDiscussionTrendWindowFixtures.ts @@ -0,0 +1,89 @@ +import type { + DailyExternalAggregate, + ExternalEvidence, + ExternalNamedRegistryActor, + ExternalPlatform, +} from "../externalDiscovery/types.ts"; +import type { ExternalAggregateWindowReadResult } from "../externalDiscovery/trendWindow.ts"; + +export const TREND_TEST_DATES = ["2026-06-27", "2026-06-28", "2026-06-29", "2026-06-30", "2026-07-01", "2026-07-02", "2026-07-03"]; + +export function trendEvidence(args: { + scope: "project" | "direction"; + target_key: string; + date: string; + mention_count?: number; + platforms?: ExternalPlatform[]; + event_ids?: string[]; + distinct_actor_count?: number; + top_tier_actor_count?: number; + named_registry_actors?: ExternalNamedRegistryActor[]; +}): ExternalEvidence { + const mentionCount = args.mention_count ?? args.event_ids?.length ?? 1; + const eventIds = args.event_ids ?? Array.from({ length: mentionCount }, (_, index) => `${args.scope}:${args.target_key}:${args.date}:${index}`); + return { + evidence_id: `${args.scope}:${args.target_key}:${args.date}`, + event_ids: eventIds, + scope: args.scope, + target_key: args.target_key, + derived_signal_kinds: ["evidence"], + platforms: args.platforms ?? ["hacker_news"], + named_registry_actors: args.named_registry_actors ?? [], + actor_tiers: {}, + actor_types: {}, + mention_count: mentionCount, + distinct_actor_count: args.distinct_actor_count ?? mentionCount, + top_tier_actor_count: args.top_tier_actor_count ?? (args.named_registry_actors?.length ?? 0), + first_seen_at: `${args.date}T00:00:00.000Z`, + last_seen_at: `${args.date}T00:00:00.000Z`, + }; +} + +export function aggregateForDate(args: { + date: string; + status?: DailyExternalAggregate["status"]; + project_evidence?: ExternalEvidence[]; + direction_evidence?: ExternalEvidence[]; +}): DailyExternalAggregate { + const projectEvidence = args.project_evidence ?? []; + const directionEvidence = args.direction_evidence ?? []; + const allEvidence = [...projectEvidence, ...directionEvidence]; + return { + schema_version: "external-discovery.aggregate.v1", + date: args.date, + generated_at: `${args.date}T00:00:00.000Z`, + provider: "agent-reach", + status: args.status ?? "ok", + source_input_hash: `hash-${args.date}`, + public_safe: true, + redaction_policy_version: "external-discovery-redaction.v1", + contains_raw_text: false, + contains_profile_urls: false, + event_count: allEvidence.reduce((total, evidence) => total + evidence.event_ids.length, 0), + accepted_event_count: allEvidence.reduce((total, evidence) => total + evidence.event_ids.length, 0), + rejected_event_count: 0, + platform_counts: countPlatforms(allEvidence.flatMap((evidence) => evidence.platforms)), + derived_signal_kind_counts: { evidence: allEvidence.length }, + project_evidence: projectEvidence, + direction_evidence: directionEvidence, + observation_candidates: [], + audit: { rejected_events: [], warnings: [] }, + }; +} + +export function loadedWindow(overrides: Record): ExternalAggregateWindowReadResult[] { + return TREND_TEST_DATES.map((date) => ({ + status: "loaded", + date, + path: `data/external-discovery/${date}.aggregate.json`, + aggregate: overrides[date] ?? aggregateForDate({ date }), + })); +} + +function countPlatforms(platforms: ExternalPlatform[]): Partial> { + const counts: Partial> = {}; + for (const platform of platforms) { + counts[platform] = (counts[platform] ?? 0) + 1; + } + return counts; +} diff --git a/src/__tests__/externalDiscussionTrendWindowPath.test.ts b/src/__tests__/externalDiscussionTrendWindowPath.test.ts new file mode 100644 index 0000000..0ee6668 --- /dev/null +++ b/src/__tests__/externalDiscussionTrendWindowPath.test.ts @@ -0,0 +1,14 @@ +import { describe, expect, it } from "vitest"; +import { externalTrendWindowLatestPath, externalTrendWindowPath } from "../externalDiscovery/paths.ts"; + +describe("external discussion trend window path contract", () => { + it("stores trend windows under the windows subdirectory", () => { + expect(slash(externalTrendWindowPath("2026-07-03"))).toBe("data/external-discovery/windows/2026-07-03.discussion-trend-window.json"); + expect(slash(externalTrendWindowLatestPath())).toBe("data/external-discovery/windows/latest.discussion-trend-window.json"); + expect(slash(externalTrendWindowPath("2026-07-03"))).not.toBe("data/external-discovery/2026-07-03.discussion-trend-window.json"); + }); +}); + +function slash(value: string): string { + return value.replace(/\\/g, "/"); +} diff --git a/src/__tests__/externalDiscussionTrendWindowReadStatus.test.ts b/src/__tests__/externalDiscussionTrendWindowReadStatus.test.ts new file mode 100644 index 0000000..9183434 --- /dev/null +++ b/src/__tests__/externalDiscussionTrendWindowReadStatus.test.ts @@ -0,0 +1,82 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { buildExternalDiscussionTrendWindow, buildSkippedExternalDiscussionTrendWindow } from "../externalDiscovery/trendWindow.ts"; +import { + readExternalDiscussionTrendWindowByDate, + readLatestExternalDiscussionTrendWindow, + writeExternalDiscussionTrendWindow, +} from "../externalDiscovery/trendWindowIntegration.ts"; +import { aggregateForDate, loadedWindow, trendEvidence } from "./externalDiscussionTrendWindowFixtures.ts"; + +const roots: string[] = []; +const originalCwd = process.cwd(); + +afterEach(() => { + process.chdir(originalCwd); + for (const root of roots.splice(0)) { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +function setupWorkspace(): string { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "external-trend-window-read-")); + roots.push(root); + process.chdir(root); + return root; +} + +describe("external discussion trend window read status", () => { + it("does not fallback from a date-specific read to latest", () => { + setupWorkspace(); + const skipped = buildSkippedExternalDiscussionTrendWindow({ + anchorDate: "2026-07-02", + generatedAt: "2026-07-02T00:00:00.000Z", + statusReason: "external_discovery_disabled", + }); + writeExternalDiscussionTrendWindow({ date: "2026-07-02", trendWindow: skipped }); + + expect(readLatestExternalDiscussionTrendWindow().read_status).toBe("skipped"); + const dateSpecific = readExternalDiscussionTrendWindowByDate("2026-07-03"); + expect(dateSpecific.read_status).toBe("not_found"); + expect(dateSpecific.trend_window).toBeUndefined(); + }); + + it("maps invalid JSON and schema mismatch to parse_error", () => { + const root = setupWorkspace(); + fs.mkdirSync(path.join(root, "data", "external-discovery", "windows"), { recursive: true }); + fs.writeFileSync( + path.join(root, "data", "external-discovery", "windows", "2026-07-03.discussion-trend-window.json"), + "{not-json", + "utf-8", + ); + expect(readExternalDiscussionTrendWindowByDate("2026-07-03").read_status).toBe("parse_error"); + + fs.writeFileSync( + path.join(root, "data", "external-discovery", "windows", "2026-07-03.discussion-trend-window.json"), + JSON.stringify({ schema_version: "wrong" }), + "utf-8", + ); + expect(readExternalDiscussionTrendWindowByDate("2026-07-03").read_status).toBe("parse_error"); + }); + + it("passes through the artifact status for valid windows", () => { + setupWorkspace(); + const trendWindow = buildExternalDiscussionTrendWindow({ + anchorDate: "2026-07-03", + generatedAt: "2026-07-03T00:00:00.000Z", + aggregateResults: loadedWindow({ + "2026-07-02": aggregateForDate({ + date: "2026-07-02", + project_evidence: [trendEvidence({ scope: "project", target_key: "openai/agents", date: "2026-07-02", mention_count: 2 })], + }), + }), + }); + writeExternalDiscussionTrendWindow({ date: "2026-07-03", trendWindow }); + + const result = readExternalDiscussionTrendWindowByDate("2026-07-03"); + expect(result.read_status).toBe(trendWindow.status); + expect(result.trend_window?.schema_version).toBe("external-discussion-trend-window.v1"); + }); +}); diff --git a/src/__tests__/externalDiscussionTrendWindowRedaction.test.ts b/src/__tests__/externalDiscussionTrendWindowRedaction.test.ts new file mode 100644 index 0000000..82c0413 --- /dev/null +++ b/src/__tests__/externalDiscussionTrendWindowRedaction.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, it } from "vitest"; +import { assertPublicSafeTrendWindow } from "../externalDiscovery/redaction.ts"; + +function safeTrendWindow(overrides: Record = {}): Record { + return { + schema_version: "external-discussion-trend-window.v1", + anchor_date: "2026-07-03", + window_start: "2026-06-27", + window_end: "2026-07-03", + window_days: 7, + generated_at: "2026-07-03T00:00:00.000Z", + status: "ok", + project_trends: [], + direction_trends: [], + coverage: { + expected_dates: [], + loaded_dates: [], + missing_dates: [], + failed_dates: [], + usable_day_count: 0, + platform_counts: {}, + partial_platforms: [], + }, + audit: { rejected_items: [], warnings: [] }, + public_safe: true, + redaction_policy_version: "external-discovery-redaction.v1", + contains_raw_text: false, + contains_profile_urls: false, + ...overrides, + }; +} + +describe("external discussion trend window redaction", () => { + it("accepts public-safe trend windows", () => { + expect(assertPublicSafeTrendWindow(safeTrendWindow()).ok).toBe(true); + }); + + it("rejects raw provider identity or private text leakage", () => { + const result = assertPublicSafeTrendWindow( + safeTrendWindow({ + project_trends: [ + { + trend_id: "project:test", + handle: "@raw", + platform_profile_url: "https://x.com/raw-profile", + status_reason: "oauth token leaked", + }, + ], + }), + ); + + expect(result.ok).toBe(false); + expect(result.reason_codes).toContain("forbidden_key:handle"); + expect(result.reason_codes).toContain("forbidden_key:platform_profile_url"); + expect(result.reason_codes).toContain("forbidden_profile_url_text"); + expect(result.reason_codes).toContain("forbidden_secret_text"); + }); + + it("requires trend-window schema and safety flags", () => { + const result = assertPublicSafeTrendWindow( + safeTrendWindow({ + schema_version: "external-discovery.aggregate.v1", + public_safe: false, + contains_raw_text: true, + contains_profile_urls: true, + }), + ); + + expect(result.ok).toBe(false); + expect(result.reason_codes).toContain("invalid_trend_window_schema_version"); + expect(result.reason_codes).toContain("public_safe_not_true"); + expect(result.reason_codes).toContain("contains_raw_text_not_false"); + expect(result.reason_codes).toContain("contains_profile_urls_not_false"); + }); +}); diff --git a/src/__tests__/externalDiscussionTrendWindowRules.test.ts b/src/__tests__/externalDiscussionTrendWindowRules.test.ts new file mode 100644 index 0000000..49850fc --- /dev/null +++ b/src/__tests__/externalDiscussionTrendWindowRules.test.ts @@ -0,0 +1,95 @@ +import { describe, expect, it } from "vitest"; +import { buildExternalDiscussionTrendWindow } from "../externalDiscovery/trendWindow.ts"; +import { aggregateForDate, loadedWindow, trendEvidence } from "./externalDiscussionTrendWindowFixtures.ts"; + +describe("external discussion trend window rules", () => { + it("marks single-day single-platform bursts as noise spikes", () => { + const window = buildExternalDiscussionTrendWindow({ + anchorDate: "2026-07-03", + generatedAt: "2026-07-03T00:00:00.000Z", + aggregateResults: loadedWindow({ + "2026-07-03": aggregateForDate({ + date: "2026-07-03", + project_evidence: [ + trendEvidence({ + scope: "project", + target_key: "single/spike", + date: "2026-07-03", + mention_count: 6, + platforms: ["hacker_news"], + distinct_actor_count: 1, + }), + ], + }), + }), + }); + + const trend = window.project_trends[0]; + expect(trend?.momentum).toBe("spike"); + expect(trend?.verdict).toBe("noise_spike"); + expect(trend?.weekly_eligible).toBe(false); + expect(trend?.components.find((component) => component.name === "noise_risk")?.level).toBe("high"); + }); + + it("marks multi-day cross-platform project trends as external reinforcement", () => { + const window = buildExternalDiscussionTrendWindow({ + anchorDate: "2026-07-03", + generatedAt: "2026-07-03T00:00:00.000Z", + aggregateResults: loadedWindow({ + "2026-06-30": aggregateForDate({ + date: "2026-06-30", + project_evidence: [trendEvidence({ scope: "project", target_key: "cross/platform", date: "2026-06-30", mention_count: 1, platforms: ["hacker_news"] })], + }), + "2026-07-01": aggregateForDate({ + date: "2026-07-01", + project_evidence: [trendEvidence({ scope: "project", target_key: "cross/platform", date: "2026-07-01", mention_count: 1, platforms: ["reddit"] })], + }), + "2026-07-02": aggregateForDate({ + date: "2026-07-02", + project_evidence: [trendEvidence({ scope: "project", target_key: "cross/platform", date: "2026-07-02", mention_count: 2, platforms: ["hacker_news", "reddit"] })], + }), + }), + }); + + const trend = window.project_trends[0]; + expect(trend?.verdict).toBe("external_reinforcement"); + expect(trend?.weekly_eligible).toBe(true); + expect(trend?.weekly_gate_reasons).toContain("cross_platform_confirmation"); + expect(trend?.weekly_gate_reasons).toContain("multi_day_persistence"); + }); + + it("detects fading when early activity dominates and the last two loaded days are zero", () => { + const window = buildExternalDiscussionTrendWindow({ + anchorDate: "2026-07-03", + generatedAt: "2026-07-03T00:00:00.000Z", + aggregateResults: loadedWindow({ + "2026-06-28": aggregateForDate({ + date: "2026-06-28", + project_evidence: [trendEvidence({ scope: "project", target_key: "fading/project", date: "2026-06-28", mention_count: 2, platforms: ["reddit", "hacker_news"] })], + }), + "2026-06-29": aggregateForDate({ + date: "2026-06-29", + project_evidence: [trendEvidence({ scope: "project", target_key: "fading/project", date: "2026-06-29", mention_count: 2, platforms: ["reddit", "hacker_news"] })], + }), + }), + }); + + expect(window.project_trends[0]?.momentum).toBe("fading"); + }); + + it("does not force stable when counts are too weak", () => { + const window = buildExternalDiscussionTrendWindow({ + anchorDate: "2026-07-03", + generatedAt: "2026-07-03T00:00:00.000Z", + aggregateResults: loadedWindow({ + "2026-07-02": aggregateForDate({ + date: "2026-07-02", + project_evidence: [trendEvidence({ scope: "project", target_key: "weak/project", date: "2026-07-02", mention_count: 1, platforms: ["reddit"] })], + }), + }), + }); + + expect(window.project_trends[0]?.momentum).toBe("insufficient"); + expect(window.project_trends[0]?.verdict).toBe("insufficient"); + }); +}); diff --git a/src/__tests__/externalDiscussionTrendWindowTypeContract.test.ts b/src/__tests__/externalDiscussionTrendWindowTypeContract.test.ts new file mode 100644 index 0000000..1567b57 --- /dev/null +++ b/src/__tests__/externalDiscussionTrendWindowTypeContract.test.ts @@ -0,0 +1,106 @@ +import { describe, expect, it } from "vitest"; +import { + EXTERNAL_TREND_COMPONENT_NAMES, + EXTERNAL_TREND_WINDOW_READ_STATUSES, + EXTERNAL_WEEKLY_GATE_REASONS, + type ExternalDiscussionTrendWindow, + type ExternalTrendItem, +} from "../externalDiscovery/types.ts"; + +describe("external discussion trend window type contract", () => { + it("freezes read statuses and weekly gate reasons", () => { + expect(EXTERNAL_TREND_WINDOW_READ_STATUSES).toEqual([ + "ok", + "not_found", + "parse_error", + "partial", + "insufficient", + "failed", + "skipped", + ]); + expect(EXTERNAL_WEEKLY_GATE_REASONS).toEqual([ + "cross_platform_confirmation", + "multi_actor_confirmation", + "multi_day_persistence", + "registry_tier_participation", + ]); + expect(EXTERNAL_TREND_COMPONENT_NAMES).toEqual([ + "discussion_volume", + "persistence", + "cross_platform_confirmation", + "actor_authority", + "binding_confidence", + "noise_risk", + ]); + }); + + it("keeps trend items secondary-only", () => { + const item: ExternalTrendItem = { + trend_id: "project:pydantic/pydantic-ai", + scope: "project", + target_key: "pydantic/pydantic-ai", + display_name: "pydantic/pydantic-ai", + target_url: "https://github.com/pydantic/pydantic-ai", + binding_confidence: "medium", + official_signal: false, + weekly_eligible: true, + weekly_gate_reasons: ["cross_platform_confirmation", "multi_day_persistence"], + weekly_gate_missing_reasons: ["multi_actor_confirmation", "registry_tier_participation"], + daily_counts: [{ date: "2026-07-03", mention_count: 2, source_count: 2, platform_count: 2 }], + mention_count_total: 2, + source_count: 2, + active_day_count: 1, + platform_count: 2, + cross_platform_days: 1, + distinct_actor_count: 2, + top_tier_actor_count: 0, + named_registry_actors: [], + components: [{ name: "noise_risk", level: "medium", evidence: ["single active day"] }], + momentum: "spike", + verdict: "external_reinforcement", + cannot_be_primary_conclusion: true, + evidence_ids: ["project:pydantic/pydantic-ai"], + source_aggregate_dates: ["2026-07-03"], + caveats: ["external trend is secondary evidence"], + }; + + expect(item.cannot_be_primary_conclusion).toBe(true); + expect(item.weekly_gate_reasons).not.toContain("provider_tier_hint"); + }); + + it("keeps the window public-safe by contract", () => { + const window: ExternalDiscussionTrendWindow = { + schema_version: "external-discussion-trend-window.v1", + anchor_date: "2026-07-03", + window_start: "2026-06-27", + window_end: "2026-07-03", + window_days: 7, + generated_at: "2026-07-03T00:00:00.000Z", + status: "insufficient", + status_reason: "window_usable_days_insufficient", + project_trends: [], + direction_trends: [], + coverage: { + expected_dates: ["2026-06-27", "2026-06-28", "2026-06-29", "2026-06-30", "2026-07-01", "2026-07-02", "2026-07-03"], + loaded_dates: [], + missing_dates: ["2026-06-27", "2026-06-28", "2026-06-29", "2026-06-30", "2026-07-01", "2026-07-02", "2026-07-03"], + failed_dates: [], + usable_day_count: 0, + platform_counts: {}, + partial_platforms: [], + }, + audit: { + rejected_items: [], + warnings: [{ reason_code: "window_usable_days_insufficient", reason_detail: "usable days 0/7" }], + }, + public_safe: true, + redaction_policy_version: "external-discovery-redaction.v1", + contains_raw_text: false, + contains_profile_urls: false, + }; + + expect(window.public_safe).toBe(true); + expect(window.contains_raw_text).toBe(false); + expect(window.contains_profile_urls).toBe(false); + }); +}); diff --git a/src/__tests__/externalDiscussionTrendWindowVerification.test.ts b/src/__tests__/externalDiscussionTrendWindowVerification.test.ts new file mode 100644 index 0000000..54571e8 --- /dev/null +++ b/src/__tests__/externalDiscussionTrendWindowVerification.test.ts @@ -0,0 +1,195 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { buildVerifyDailyResult } from "../action/dailyVerification.ts"; +import { buildExternalDiscussionTrendWindow } from "../externalDiscovery/trendWindow.ts"; +import { writeExternalDiscussionTrendWindow } from "../externalDiscovery/trendWindowIntegration.ts"; +import type { DailyReport, DailyRunSummary } from "../types.ts"; +import { aggregateForDate, loadedWindow, trendEvidence } from "./externalDiscussionTrendWindowFixtures.ts"; + +const roots: string[] = []; +const originalCwd = process.cwd(); + +afterEach(() => { + process.chdir(originalCwd); + for (const root of roots.splice(0)) { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +function setupWorkspace(): string { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "external-trend-window-verify-")); + roots.push(root); + fs.mkdirSync(path.join(root, "data", "reports"), { recursive: true }); + fs.mkdirSync(path.join(root, "data", "raw", "github"), { recursive: true }); + process.chdir(root); + return root; +} + +function writeJson(filepath: string, value: unknown): void { + fs.mkdirSync(path.dirname(filepath), { recursive: true }); + fs.writeFileSync(filepath, JSON.stringify(value, null, 2), "utf-8"); +} + +function makeReport(date: string): DailyReport { + return { + date, + generated_at: `${date}T00:00:00.000Z`, + enhancement_status: "rules-only", + enhancement_audit: { rejected_outputs: [] }, + personalized_relevance_applicable: false, + overall_daily_status: "鏁版嵁鏂伴矞锛屽彲鐩存帴闃呰", + freshness_sources: [], + today_fresh_candidate_count: 1, + context_candidate_count: 0, + pending_confirmation_count: 0, + main_board_mode: "fresh_today_only", + today_star_projects: [], + context_only_projects: [], + today_pulse_projects: [], + mission_match_projects: [], + explore_ribbon_projects: [], + coverage_atlas: [], + gap_ledger: [], + mission_discovery_status: "active", + mission_degraded_reason_codes: [], + global_hot_projects: [], + demand_relevant_projects: [], + searched_direction_statuses: [], + new_projects: [], + high_score_projects: [], + anomaly_projects: [], + all_projects: [], + } as unknown as DailyReport; +} + +function makeSummary(date: string, trendWindowStatus: "ok" | "partial" | "failed" | "insufficient" | "skipped"): DailyRunSummary { + return { + date, + generated_at: `${date}T00:00:00.000Z`, + dry_run: true, + minimum_viable_run_completed: true, + completion_notes: [], + counts: { + raw_signals: 1, + normalized_projects: 1, + scored_projects: 1, + high_score_projects: 0, + anomaly_projects: 0, + new_projects: 1, + classifications: 0, + }, + source_status: [ + { + source: "agents-radar", + enabled: true, + item_count: 1, + distinct_projects: 1, + status: "active", + notes: [], + }, + ], + quality: { + missing_descriptions: 0, + watchlist_hits: 0, + low_confidence_projects: 0, + medium_confidence_projects: 0, + insufficient_metrics_projects: 0, + suspicious_growth_projects: 0, + single_source_projects: 0, + single_spike_projects: 0, + emerging_projects: 1, + persistent_projects: 0, + }, + diagnostics: { + anomaly_share: 0, + uniform_star_velocity_detected: false, + metrics_source_distribution: { embedded: 1, github_api: 0, github_html: 0, github_cache: 0, unavailable: 0 }, + star_delta_source_distribution: { github_live: 0, github_snapshot: 0, signal: 1, unavailable: 0 }, + github_star_delta: { + live_delta_attempts: 0, + live_delta_success: 0, + snapshot_delta_success: 0, + token_missing: 0, + auth_invalid: 0, + rate_limit: 0, + network_blocked: 0, + }, + }, + top_projects: [], + external_discovery: { + aggregate_status: "ok", + accepted_event_count: 2, + rejected_event_count: 0, + observation_candidate_count: 1, + project_evidence_count: 1, + direction_evidence_count: 0, + explanation_status: "skipped", + explanation_eligible_count: 0, + explanation_attempted_count: 0, + explanation_enhanced_count: 0, + explanation_fallback_count: 0, + explanation_rejected_count: 0, + trend_window_read_status: trendWindowStatus, + trend_window_status: trendWindowStatus, + trend_window_path: `data/external-discovery/windows/${date}.discussion-trend-window.json`, + trend_window_usable_day_count: 7, + trend_window_project_trend_count: 1, + trend_window_direction_trend_count: 0, + trend_window_failed_date_count: 0, + trend_window_missing_date_count: 0, + warning_count: 0, + warnings: [], + }, + observer_top_candidates: [], + watchouts: [], + next_focus: [], + recommended_actions: [], + }; +} + +describe("external discussion trend window daily verification", () => { + it("passes the trend window contract for a public-safe date-specific artifact", () => { + const root = setupWorkspace(); + const date = "2026-07-03"; + const trendWindow = buildExternalDiscussionTrendWindow({ + anchorDate: date, + generatedAt: `${date}T00:00:00.000Z`, + aggregateResults: loadedWindow({ + "2026-07-02": aggregateForDate({ + date: "2026-07-02", + project_evidence: [trendEvidence({ scope: "project", target_key: "openai/agents", date: "2026-07-02", mention_count: 2 })], + }), + }), + }); + writeExternalDiscussionTrendWindow({ date, trendWindow }); + writeJson(path.join(root, "data", "reports", `${date}.run-summary.json`), makeSummary(date, trendWindow.status)); + writeJson(path.join(root, "data", "reports", `${date}.daily.json`), makeReport(date)); + writeJson(path.join(root, "data", "raw", "github", `${date}.enrichment.json`), []); + + const result = buildVerifyDailyResult(date); + const check = result.checks.find((item) => item.name === "external_discussion_trend_window_contract"); + expect(check?.status).toBe("pass"); + expect(check?.detail).toContain("project_trends=1"); + }); + + it("fails the trend window contract for parse errors", () => { + const root = setupWorkspace(); + const date = "2026-07-03"; + fs.mkdirSync(path.join(root, "data", "external-discovery", "windows"), { recursive: true }); + fs.writeFileSync( + path.join(root, "data", "external-discovery", "windows", `${date}.discussion-trend-window.json`), + "{not-json", + "utf-8", + ); + writeJson(path.join(root, "data", "reports", `${date}.run-summary.json`), makeSummary(date, "ok")); + writeJson(path.join(root, "data", "reports", `${date}.daily.json`), makeReport(date)); + writeJson(path.join(root, "data", "raw", "github", `${date}.enrichment.json`), []); + + const result = buildVerifyDailyResult(date); + const check = result.checks.find((item) => item.name === "external_discussion_trend_window_contract"); + expect(check?.status).toBe("fail"); + expect(check?.detail).toContain("trend window unreadable"); + }); +}); diff --git a/src/__tests__/externalDiscussionTrendWindowWeekly.test.ts b/src/__tests__/externalDiscussionTrendWindowWeekly.test.ts new file mode 100644 index 0000000..d815064 --- /dev/null +++ b/src/__tests__/externalDiscussionTrendWindowWeekly.test.ts @@ -0,0 +1,129 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import type { AppConfig } from "../config.ts"; +import { buildWeeklyArtifacts } from "../action/weeklyEnhancement.ts"; +import { renderWeeklyReport } from "../action/weeklyReport.ts"; +import { buildExternalDiscussionTrendWindow } from "../externalDiscovery/trendWindow.ts"; +import { writeExternalDiscussionTrendWindow } from "../externalDiscovery/trendWindowIntegration.ts"; +import type { DailyReport } from "../types.ts"; +import { aggregateForDate, loadedWindow, trendEvidence } from "./externalDiscussionTrendWindowFixtures.ts"; + +const roots: string[] = []; +const originalCwd = process.cwd(); + +afterEach(() => { + process.chdir(originalCwd); + for (const root of roots.splice(0)) { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +function setupWorkspace(): string { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "external-trend-window-weekly-")); + roots.push(root); + process.chdir(root); + return root; +} + +function config(): AppConfig { + return { + thresholds: { highScore: 70 }, + sources: { userInterestProfile: { enabled: false, topics: [] } }, + llm: { enabled: false, mode: "rules-only", provider: "none" }, + } as unknown as AppConfig; +} + +function daily(date: string): DailyReport { + return { + date, + generated_at: `${date}T00:00:00.000Z`, + enhancement_status: "rules-only", + enhancement_audit: { rejected_outputs: [] }, + personalized_relevance_applicable: false, + overall_daily_status: "鏁版嵁鏂伴矞锛屽彲鐩存帴闃呰", + freshness_sources: [], + today_fresh_candidate_count: 0, + context_candidate_count: 0, + pending_confirmation_count: 0, + main_board_mode: "fresh_today_only", + today_star_projects: [], + context_only_projects: [], + today_pulse_projects: [], + mission_match_projects: [], + explore_ribbon_projects: [], + coverage_atlas: [], + gap_ledger: [], + mission_discovery_status: "active", + mission_degraded_reason_codes: [], + global_hot_projects: [], + demand_relevant_projects: [], + searched_direction_statuses: [], + new_projects: [], + high_score_projects: [], + anomaly_projects: [], + all_projects: [], + } as unknown as DailyReport; +} + +describe("external discussion trend window weekly consumption", () => { + it("attaches only secondary external trend evidence to weekly artifacts", () => { + setupWorkspace(); + const trendWindow = buildExternalDiscussionTrendWindow({ + anchorDate: "2026-07-03", + generatedAt: "2026-07-03T00:00:00.000Z", + aggregateResults: loadedWindow({ + "2026-07-02": aggregateForDate({ + date: "2026-07-02", + project_evidence: [ + trendEvidence({ + scope: "project", + target_key: "openai/agents", + date: "2026-07-02", + mention_count: 2, + platforms: ["hacker_news", "reddit"], + }), + ], + }), + "2026-07-03": aggregateForDate({ + date: "2026-07-03", + project_evidence: [ + trendEvidence({ + scope: "project", + target_key: "openai/agents", + date: "2026-07-03", + mention_count: 2, + platforms: ["hacker_news", "reddit"], + }), + trendEvidence({ + scope: "project", + target_key: "noise/project", + date: "2026-07-03", + mention_count: 3, + platforms: ["hacker_news"], + }), + ], + }), + }), + }); + writeExternalDiscussionTrendWindow({ date: "2026-07-03", trendWindow }); + + const artifacts = buildWeeklyArtifacts([{ date: "2026-07-03", scored: [], daily: daily("2026-07-03") }], config()); + + expect(artifacts.report.external_discussion_trends?.read_status).toBe("ok"); + expect(artifacts.report.external_discussion_trends?.secondary_evidence.map((item) => item.target_key)).toContain( + "openai/agents", + ); + expect(artifacts.report.external_discussion_trends?.secondary_evidence.map((item) => item.target_key)).not.toContain( + "noise/project", + ); + expect(artifacts.report.external_discussion_trends?.noise_items.map((item) => item.target_key)).toContain( + "noise/project", + ); + + const rendered = renderWeeklyReport(artifacts.report, { judgment: artifacts.judgment }); + expect(rendered).toContain("## AgentReach 外部讨论趋势"); + expect(rendered).toContain("openai/agents"); + }); +}); diff --git a/src/__tests__/externalDiscussionTrendWindowWrite.test.ts b/src/__tests__/externalDiscussionTrendWindowWrite.test.ts new file mode 100644 index 0000000..0d0af25 --- /dev/null +++ b/src/__tests__/externalDiscussionTrendWindowWrite.test.ts @@ -0,0 +1,121 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { buildExternalDiscussionTrendWindow, buildSkippedExternalDiscussionTrendWindow } from "../externalDiscovery/trendWindow.ts"; +import { runDailyExternalTrendWindowIntegration, writeExternalDiscussionTrendWindow } from "../externalDiscovery/trendWindowIntegration.ts"; +import type { ExternalDiscussionTrendWindow } from "../externalDiscovery/types.ts"; +import { aggregateForDate, loadedWindow, trendEvidence, TREND_TEST_DATES } from "./externalDiscussionTrendWindowFixtures.ts"; + +const roots: string[] = []; +const originalCwd = process.cwd(); + +afterEach(() => { + process.chdir(originalCwd); + for (const root of roots.splice(0)) { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +function setupWorkspace(): string { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "external-trend-window-write-")); + roots.push(root); + process.chdir(root); + return root; +} + +describe("external discussion trend window writes", () => { + it("writes date-specific and latest files after public-safe checks", () => { + const root = setupWorkspace(); + const trendWindow = buildExternalDiscussionTrendWindow({ + anchorDate: "2026-07-03", + generatedAt: "2026-07-03T00:00:00.000Z", + aggregateResults: loadedWindow({ + "2026-07-02": aggregateForDate({ + date: "2026-07-02", + project_evidence: [trendEvidence({ scope: "project", target_key: "openai/agents", date: "2026-07-02", mention_count: 2 })], + }), + }), + }); + + const paths = writeExternalDiscussionTrendWindow({ date: "2026-07-03", trendWindow }); + + expect(paths.trend_window.replace(/\\/g, "/")).toBe("data/external-discovery/windows/2026-07-03.discussion-trend-window.json"); + expect(paths.trend_window_latest.replace(/\\/g, "/")).toBe("data/external-discovery/windows/latest.discussion-trend-window.json"); + expect(fs.existsSync(path.join(root, paths.trend_window))).toBe(true); + expect(fs.existsSync(path.join(root, paths.trend_window_latest))).toBe(true); + }); + + it("updates latest for insufficient, failed, and skipped public-safe artifacts", () => { + const root = setupWorkspace(); + const insufficient = buildExternalDiscussionTrendWindow({ + anchorDate: "2026-07-03", + generatedAt: "2026-07-03T00:00:00.000Z", + aggregateResults: TREND_TEST_DATES.map((date) => ({ + status: "missing", + date, + path: `data/external-discovery/${date}.aggregate.json`, + })), + }); + expect(insufficient.status).toBe("insufficient"); + writeExternalDiscussionTrendWindow({ date: "2026-07-03", trendWindow: insufficient }); + expect(readLatest(root).status).toBe("insufficient"); + + const failed = buildExternalDiscussionTrendWindow({ + anchorDate: "2026-07-03", + generatedAt: "2026-07-03T00:01:00.000Z", + aggregateResults: TREND_TEST_DATES.map((date) => ({ + status: "failed", + date, + path: `data/external-discovery/${date}.aggregate.json`, + reason_code: "window_aggregate_parse_failed", + reason_detail: "Unexpected token", + })), + }); + expect(failed.status).toBe("failed"); + writeExternalDiscussionTrendWindow({ date: "2026-07-03", trendWindow: failed }); + expect(readLatest(root).status).toBe("failed"); + + const skipped = buildSkippedExternalDiscussionTrendWindow({ + anchorDate: "2026-07-03", + generatedAt: "2026-07-03T00:02:00.000Z", + statusReason: "external_discovery_disabled", + }); + writeExternalDiscussionTrendWindow({ date: "2026-07-03", trendWindow: skipped }); + expect(readLatest(root).status).toBe("skipped"); + }); + + it("does not write date-specific or latest files when public-safe checks fail", () => { + const root = setupWorkspace(); + const invalid = { + ...buildSkippedExternalDiscussionTrendWindow({ + anchorDate: "2026-07-03", + generatedAt: "2026-07-03T00:00:00.000Z", + statusReason: "external_discovery_disabled", + }), + contains_raw_text: true, + } as unknown as ExternalDiscussionTrendWindow; + + expect(() => writeExternalDiscussionTrendWindow({ date: "2026-07-03", trendWindow: invalid })).toThrow(/not public-safe/); + expect(fs.existsSync(path.join(root, "data", "external-discovery", "windows", "2026-07-03.discussion-trend-window.json"))).toBe(false); + expect(fs.existsSync(path.join(root, "data", "external-discovery", "windows", "latest.discussion-trend-window.json"))).toBe(false); + }); + + it("writes skipped from daily integration when external discovery is skipped", () => { + const root = setupWorkspace(); + const result = runDailyExternalTrendWindowIntegration({ + date: "2026-07-03", + generatedAt: "2026-07-03T00:00:00.000Z", + currentAggregate: aggregateForDate({ date: "2026-07-03", status: "skipped" }), + }); + + expect(result.trend_window.status).toBe("skipped"); + expect(readLatest(root).status).toBe("skipped"); + }); +}); + +function readLatest(root: string): ExternalDiscussionTrendWindow { + return JSON.parse( + fs.readFileSync(path.join(root, "data", "external-discovery", "windows", "latest.discussion-trend-window.json"), "utf-8"), + ) as ExternalDiscussionTrendWindow; +} diff --git a/src/__tests__/runSummaryObserver.test.ts b/src/__tests__/runSummaryObserver.test.ts index e913444..04206fc 100644 --- a/src/__tests__/runSummaryObserver.test.ts +++ b/src/__tests__/runSummaryObserver.test.ts @@ -72,6 +72,14 @@ describe("renderDailyRunSummary observer section", () => { explanation_enhanced_count: 0, explanation_fallback_count: 1, explanation_rejected_count: 1, + trend_window_read_status: "insufficient", + trend_window_status: "insufficient", + trend_window_path: "data/external-discovery/windows/2026-06-12.discussion-trend-window.json", + trend_window_usable_day_count: 1, + trend_window_project_trend_count: 1, + trend_window_direction_trend_count: 0, + trend_window_failed_date_count: 0, + trend_window_missing_date_count: 6, warning_count: 1, warnings: ["summary_generation_failed:project:openai/agents-sdk"], }, @@ -135,6 +143,7 @@ describe("renderDailyRunSummary observer section", () => { expect(rendered).toContain("pressure_state_distribution"); expect(rendered).toContain("## AgentReach 外部发现"); expect(rendered).toContain("explanation_status: partial"); + expect(rendered).toContain("trend_window_read_status: insufficient"); expect(rendered).toContain("summary_generation_failed:project:openai/agents-sdk"); }); }); diff --git a/src/action/dailyVerification.ts b/src/action/dailyVerification.ts index 87bd6c9..825a813 100644 --- a/src/action/dailyVerification.ts +++ b/src/action/dailyVerification.ts @@ -1,9 +1,10 @@ import fs from "node:fs"; import path from "node:path"; import { externalAggregatePath, externalCandidateExplanationsPath } from "../externalDiscovery/paths.ts"; -import { assertPublicSafeAggregate } from "../externalDiscovery/redaction.ts"; +import { assertPublicSafeAggregate, assertPublicSafeTrendWindow } from "../externalDiscovery/redaction.ts"; import { assertPublicSafeCandidateExplanations } from "../externalDiscovery/explanationRedaction.ts"; import type { ExternalCandidateExplanationArtifact } from "../externalDiscovery/explanations.ts"; +import { readExternalDiscussionTrendWindowByDate, type ExternalDiscussionTrendWindowReadResult } from "../externalDiscovery/trendWindowIntegration.ts"; import { readJsonFile } from "../storage/files.ts"; import type { DailyReport, @@ -519,6 +520,83 @@ function externalCandidateExplanationContractChecks( ]; } +function externalTrendWindowContractChecks( + summary: DailyRunSummary, + trendWindowRead: ExternalDiscussionTrendWindowReadResult, +): VerificationCheck[] { + const externalSummary = summary.external_discovery; + if (trendWindowRead.read_status === "not_found") { + const status = externalSummary ? "warn" : "pass"; + return [ + buildCheck( + "external_discussion_trend_window_contract", + status, + externalSummary + ? `trend window missing at ${trendWindowRead.path}; external discovery summary expected a trend artifact` + : `trend window not present at ${trendWindowRead.path}; external layer not run for this date`, + ), + ]; + } + + if (trendWindowRead.read_status === "parse_error" || !trendWindowRead.trend_window) { + return [ + buildCheck( + "external_discussion_trend_window_contract", + "fail", + `trend window unreadable at ${trendWindowRead.path}: ${trendWindowRead.error ?? "parse_error"}`, + ), + ]; + } + + const trendWindow = trendWindowRead.trend_window; + const redaction = assertPublicSafeTrendWindow(trendWindow); + const itemIssues = [ + ...trendWindow.project_trends + .filter((item) => item.scope !== "project") + .map((item) => `project_trends contains non-project item ${item.trend_id}`), + ...trendWindow.direction_trends + .filter((item) => item.scope !== "direction") + .map((item) => `direction_trends contains non-direction item ${item.trend_id}`), + ...[...trendWindow.project_trends, ...trendWindow.direction_trends] + .filter((item) => item.cannot_be_primary_conclusion !== true) + .map((item) => `${item.trend_id} missing cannot_be_primary_conclusion=true`), + ...trendWindow.direction_trends + .filter((item) => item.weekly_eligible && item.weekly_gate_reasons.length < 2) + .map((item) => `${item.trend_id} direction weekly eligibility does not satisfy 4-choose-2 gate`), + ...[...trendWindow.project_trends, ...trendWindow.direction_trends] + .filter((item) => item.verdict === "noise_spike" && item.weekly_eligible) + .map((item) => `${item.trend_id} noise_spike must not be weekly eligible`), + ]; + const summaryIssues = externalSummary + ? [ + ...(externalSummary.trend_window_read_status !== trendWindowRead.read_status + ? [`summary read status ${externalSummary.trend_window_read_status} does not match artifact read status ${trendWindowRead.read_status}`] + : []), + ...(externalSummary.trend_window_status !== trendWindow.status + ? [`summary trend status ${externalSummary.trend_window_status ?? "missing"} does not match artifact status ${trendWindow.status}`] + : []), + ...(externalSummary.trend_window_path.replace(/\\/g, "/") !== trendWindowRead.path.replace(/\\/g, "/") + ? [`summary trend path ${externalSummary.trend_window_path} does not match ${trendWindowRead.path}`] + : []), + ] + : []; + const issues = [ + ...(!redaction.ok ? [`redaction=${redaction.reason_codes.join(",")}`] : []), + ...itemIssues, + ...summaryIssues, + ]; + + return [ + buildCheck( + "external_discussion_trend_window_contract", + issues.length === 0 ? "pass" : "fail", + issues.length === 0 + ? `trend window ${trendWindowRead.read_status}; usable_days=${trendWindow.coverage.usable_day_count}; project_trends=${trendWindow.project_trends.length}; direction_trends=${trendWindow.direction_trends.length}` + : issues.join("; "), + ), + ]; +} + function aggregateNeedsCandidateExplanations(aggregate: unknown): boolean { if (!isRecord(aggregate)) return false; const acceptedEventCount = typeof aggregate.accepted_event_count === "number" ? aggregate.accepted_event_count : 0; @@ -715,6 +793,7 @@ function buildChecks( externalAggregate: OptionalJsonRead, externalCandidateExplanationsFilepath: string, externalCandidateExplanations: OptionalJsonRead, + externalTrendWindow: ExternalDiscussionTrendWindowReadResult, ): VerificationCheck[] { const checks = [ ...completionChecks(summary), @@ -725,6 +804,7 @@ function buildChecks( ...projectSearchContractChecks(summary, report), ...externalAggregateContractChecks(externalAggregateFilepath, externalAggregate), ...externalCandidateExplanationContractChecks(externalCandidateExplanationsFilepath, externalAggregate, externalCandidateExplanations), + ...externalTrendWindowContractChecks(summary, externalTrendWindow), ]; const githubCheck = githubAuditCheck(summary, githubAudit); if (githubCheck) checks.push(githubCheck); @@ -758,6 +838,7 @@ export function buildVerifyDailyResult(date: string): VerifyDailyResult { const report = readJsonFile(reportPath, null); const externalAggregate = readOptionalJson(externalAggregateFilepath); const externalCandidateExplanations = readOptionalJson(externalCandidateExplanationsFilepath); + const externalTrendWindow = readExternalDiscussionTrendWindowByDate(date); if (!summary) { return missingSummaryResult(date, runSummaryPath, githubEnrichmentPath); @@ -772,6 +853,7 @@ export function buildVerifyDailyResult(date: string): VerifyDailyResult { externalAggregate, externalCandidateExplanationsFilepath, externalCandidateExplanations, + externalTrendWindow, ); return { date, diff --git a/src/action/runSummary.ts b/src/action/runSummary.ts index 8ea5d6d..fcfd87d 100644 --- a/src/action/runSummary.ts +++ b/src/action/runSummary.ts @@ -841,6 +841,11 @@ function renderExternalDiscoverySummary(summary: DailyRunSummary): string[] { `- evidence: project=${external.project_evidence_count}; direction=${external.direction_evidence_count}`, `- explanation_status: ${external.explanation_status}${external.explanation_status_reason ? ` (${external.explanation_status_reason})` : ""}`, `- explanations: eligible=${external.explanation_eligible_count}; attempted=${external.explanation_attempted_count}; enhanced=${external.explanation_enhanced_count}; fallback=${external.explanation_fallback_count}; rejected=${external.explanation_rejected_count}`, + `- trend_window_read_status: ${external.trend_window_read_status}`, + `- trend_window_status: ${external.trend_window_status ?? "unavailable"}`, + `- trend_window_path: ${external.trend_window_path}`, + `- trend_window_coverage: usable_days=${external.trend_window_usable_day_count}; failed_dates=${external.trend_window_failed_date_count}; missing_dates=${external.trend_window_missing_date_count}`, + `- trend_window_items: project=${external.trend_window_project_trend_count}; direction=${external.trend_window_direction_trend_count}`, ...(warnings.length > 0 ? warnings.slice(0, 8).map((warning) => `- warning: ${warning}`) : ["- warnings: none"]), ]; } diff --git a/src/action/weeklyEnhancement.ts b/src/action/weeklyEnhancement.ts index 50744fd..d61a6eb 100644 --- a/src/action/weeklyEnhancement.ts +++ b/src/action/weeklyEnhancement.ts @@ -16,6 +16,8 @@ import type { WeeklyEvidenceProject, WeeklyEvidenceAxis, WeeklyEvidenceMatrix, + WeeklyExternalDiscussionTrendItem, + WeeklyExternalDiscussionTrendSummary, WeeklyJudgmentReport, WeeklyReport, WeeklySemanticInputBundle, @@ -38,6 +40,8 @@ import { buildWeeklyTrendCandidatesV2, } from "./weeklyJudgmentRules.ts"; import { applyWeeklyTrendAgentReview, buildWeeklyTrendAgentPrompt } from "./weeklyTrendAgent.ts"; +import { readExternalDiscussionTrendWindowByDate } from "../externalDiscovery/trendWindowIntegration.ts"; +import type { ExternalTrendItem } from "../externalDiscovery/types.ts"; type WindowDay = { date: string; @@ -360,6 +364,65 @@ function buildWeeklyEvidenceMatrix( return buildTrendScopedEvidenceMatrix(focusedTrendKey, focusedTrendName, weeklyFocusProjects); } +function buildWeeklyExternalDiscussionTrendSummary(anchorDate: string): WeeklyExternalDiscussionTrendSummary { + const read = readExternalDiscussionTrendWindowByDate(anchorDate); + const trendWindow = read.trend_window; + if (!trendWindow) { + return { + read_status: read.read_status, + path: read.path, + usable_day_count: 0, + project_trend_count: 0, + direction_trend_count: 0, + failed_date_count: 0, + missing_date_count: 0, + secondary_evidence: [], + direction_observations: [], + noise_items: [], + }; + } + + const allItems = [...trendWindow.project_trends, ...trendWindow.direction_trends]; + return { + read_status: read.read_status, + status: trendWindow.status, + path: read.path, + usable_day_count: trendWindow.coverage.usable_day_count, + project_trend_count: trendWindow.project_trends.length, + direction_trend_count: trendWindow.direction_trends.length, + failed_date_count: trendWindow.coverage.failed_dates.length, + missing_date_count: trendWindow.coverage.missing_dates.length, + secondary_evidence: trendWindow.project_trends + .filter((item) => item.weekly_eligible && item.verdict === "external_reinforcement") + .map(weeklyExternalDiscussionTrendItem), + direction_observations: trendWindow.direction_trends + .filter((item) => item.weekly_eligible && item.verdict === "watch_signal") + .map(weeklyExternalDiscussionTrendItem), + noise_items: allItems.filter((item) => item.verdict === "noise_spike").map(weeklyExternalDiscussionTrendItem), + }; +} + +function weeklyExternalDiscussionTrendItem(item: ExternalTrendItem): WeeklyExternalDiscussionTrendItem { + return { + scope: item.scope, + target_key: item.target_key, + display_name: item.display_name, + target_url: item.target_url, + binding_confidence: item.binding_confidence, + official_signal: item.official_signal, + weekly_eligible: item.weekly_eligible, + momentum: item.momentum, + verdict: item.verdict, + mention_count_total: item.mention_count_total, + source_count: item.source_count, + active_day_count: item.active_day_count, + platform_count: item.platform_count, + named_registry_actor_count: item.named_registry_actors.length, + evidence_ids: [...item.evidence_ids], + caveats: [...item.caveats], + }; +} + function addBucket( buckets: Map, project: ScoredProject, @@ -1002,6 +1065,7 @@ function buildCompatibilityWeeklyReport( judgment.enhancement_status === "rules-only" ? overallSummary(coreCards, weakSignals) : judgment.executive_summary_cn, supporting_trend_keys: coreCards.map((card) => card.trend_key), core_trend_cards: attachCoreTrendEvidenceMatrices(coreCards, weeklyFocusProjects), + external_discussion_trends: judgment.external_discussion_trends, personalized_weekly_focus: personalized, weak_signal_cards: weakSignals, enhancement_audit: { rejected_outputs: [] }, @@ -1050,6 +1114,7 @@ function buildWeeklyJudgmentReport( observing_trends: review.observing_trends, audit_conclusion: review.audit_findings, evidence_matrix: undefined, + external_discussion_trends: buildWeeklyExternalDiscussionTrendSummary(days[days.length - 1]?.date ?? ""), enhancement_audit: { rejected_outputs: rejectedOutputs }, }; } diff --git a/src/action/weeklyReport.ts b/src/action/weeklyReport.ts index b6a513b..c2ef41b 100644 --- a/src/action/weeklyReport.ts +++ b/src/action/weeklyReport.ts @@ -1,4 +1,11 @@ -import type { ScoredProject, WeeklyEvidenceMatrix, WeeklyJudgmentReport, WeeklyReport } from "../types.ts"; +import type { + ScoredProject, + WeeklyEvidenceMatrix, + WeeklyExternalDiscussionTrendItem, + WeeklyExternalDiscussionTrendSummary, + WeeklyJudgmentReport, + WeeklyReport, +} from "../types.ts"; import { renderCoreTrendCard, renderWeakSignalCard } from "./weeklyEnhancement.ts"; function countByParadigm(items: ScoredProject[]): Array<[string, number]> { @@ -85,6 +92,34 @@ function renderWeeklyEvidenceMatrix(matrix?: WeeklyEvidenceMatrix): string[] { ]; } +function renderExternalDiscussionTrendItems(items: WeeklyExternalDiscussionTrendItem[], emptyText: string): string[] { + if (items.length === 0) return [`- ${emptyText}`]; + return items.slice(0, 8).map( + (item) => + `- ${item.display_name} [${item.scope}] verdict=${item.verdict}; momentum=${item.momentum}; mentions=${item.mention_count_total}; active_days=${item.active_day_count}; platforms=${item.platform_count}; named_actors=${item.named_registry_actor_count}`, + ); +} + +function renderExternalDiscussionTrends(summary?: WeeklyExternalDiscussionTrendSummary): string[] { + if (!summary) { + return ["- external_discussion_trends: unavailable"]; + } + + return [ + `- read_status: ${summary.read_status}`, + `- status: ${summary.status ?? "unavailable"}`, + `- path: ${summary.path}`, + `- coverage: usable_days=${summary.usable_day_count}; failed_dates=${summary.failed_date_count}; missing_dates=${summary.missing_date_count}`, + `- trend_counts: project=${summary.project_trend_count}; direction=${summary.direction_trend_count}`, + "- secondary_evidence:", + ...renderExternalDiscussionTrendItems(summary.secondary_evidence, "none"), + "- direction_observations:", + ...renderExternalDiscussionTrendItems(summary.direction_observations, "none"), + "- noise_items:", + ...renderExternalDiscussionTrendItems(summary.noise_items, "none"), + ]; +} + function renderAuditConclusion(judgment?: WeeklyJudgmentReport): string[] { if (!judgment) { return [ @@ -138,6 +173,10 @@ function renderEnhancedWeeklyReport(report: WeeklyReport, judgment?: WeeklyJudgm "", ...renderWeeklyEvidenceMatrix(report.evidence_matrix), "", + "## AgentReach 外部讨论趋势", + "", + ...renderExternalDiscussionTrends(report.external_discussion_trends), + "", "## 已成立趋势", "", ...(report.core_trend_cards.length > 0 diff --git a/src/cli.ts b/src/cli.ts index fea8a2c..21c2b43 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -73,7 +73,7 @@ interface CliOptions { includeAgentsRadar: boolean; includeTrendshift: boolean; configPath: string; - view?: "overview" | "projects" | "weekly" | "run-health" | "observer" | "knowledge-base" | "kb"; + view?: "overview" | "projects" | "weekly" | "run-health" | "observer" | "knowledge-base" | "kb"; project?: string; slug?: string; trendKey?: string; @@ -810,6 +810,14 @@ export async function runDaily(opts: CliOptions): Promise { explanation_enhanced_count: externalDiscovery.explanations.audit.enhanced_count, explanation_fallback_count: externalDiscovery.explanations.audit.fallback_count, explanation_rejected_count: externalDiscovery.explanations.audit.rejected_count, + trend_window_read_status: externalDiscovery.trend_window.status, + trend_window_status: externalDiscovery.trend_window.status, + trend_window_path: externalDiscovery.paths.trend_window, + trend_window_usable_day_count: externalDiscovery.trend_window.coverage.usable_day_count, + trend_window_project_trend_count: externalDiscovery.trend_window.project_trends.length, + trend_window_direction_trend_count: externalDiscovery.trend_window.direction_trends.length, + trend_window_failed_date_count: externalDiscovery.trend_window.coverage.failed_dates.length, + trend_window_missing_date_count: externalDiscovery.trend_window.coverage.missing_dates.length, warning_count: externalDiscovery.aggregate.audit.warnings.length + externalDiscovery.explanations.audit.warnings.length, warnings: [ ...externalDiscovery.aggregate.audit.warnings.map((warning) => `${warning.reason_code}:${warning.reason_detail}`), diff --git a/src/externalDiscovery/dailyIntegration.ts b/src/externalDiscovery/dailyIntegration.ts index 0d316a1..c99fda8 100644 --- a/src/externalDiscovery/dailyIntegration.ts +++ b/src/externalDiscovery/dailyIntegration.ts @@ -8,6 +8,7 @@ import { externalRawInputPath, } from "./paths.ts"; import { assertPublicSafeAggregate, stableSourceInputHash } from "./redaction.ts"; +import { runDailyExternalTrendWindowIntegration, type DailyExternalTrendWindowIntegrationResult } from "./trendWindowIntegration.ts"; import { buildCandidateExplanationInputs, generateExternalCandidateExplanations, @@ -24,11 +25,14 @@ export interface DailyExternalDiscoveryIntegrationResult { aggregate: DailyExternalAggregate; explanations: ExternalCandidateExplanationArtifact; input_build: CandidateExplanationBuildResult; + trend_window: DailyExternalTrendWindowIntegrationResult["trend_window"]; paths: { aggregate: string; aggregate_latest: string; explanations: string; explanations_latest: string; + trend_window: string; + trend_window_latest: string; }; } @@ -163,16 +167,25 @@ export async function runDailyExternalDiscoveryIntegration(args: { const explanationsLatest = externalCandidateExplanationsLatestPath(); writeJsonFile(explanationsPath, explanations, args.dryRun); writeJsonFile(explanationsLatest, explanations, args.dryRun); + const trendWindow = runDailyExternalTrendWindowIntegration({ + date: args.date, + generatedAt: args.generatedAt, + dryRun: args.dryRun, + currentAggregate: aggregate, + }); return { aggregate, explanations, input_build: inputBuild, + trend_window: trendWindow.trend_window, paths: { aggregate: aggregatePath, aggregate_latest: aggregateLatest, explanations: explanationsPath, explanations_latest: explanationsLatest, + trend_window: trendWindow.paths.trend_window, + trend_window_latest: trendWindow.paths.trend_window_latest, }, }; } diff --git a/src/externalDiscovery/paths.ts b/src/externalDiscovery/paths.ts index 97468e0..37722be 100644 --- a/src/externalDiscovery/paths.ts +++ b/src/externalDiscovery/paths.ts @@ -20,6 +20,14 @@ export function externalCandidateExplanationsLatestPath(): string { return path.join("data", "external-discovery", "latest.candidate-explanations.json"); } +export function externalTrendWindowPath(date: string): string { + return path.join("data", "external-discovery", "windows", `${date}.discussion-trend-window.json`); +} + +export function externalTrendWindowLatestPath(): string { + return path.join("data", "external-discovery", "windows", "latest.discussion-trend-window.json"); +} + export function externalEntityRegistryPath(): string { return path.join("data", "external-discovery", "entity-registry.json"); } diff --git a/src/externalDiscovery/redaction.ts b/src/externalDiscovery/redaction.ts index 934c102..b39517f 100644 --- a/src/externalDiscovery/redaction.ts +++ b/src/externalDiscovery/redaction.ts @@ -57,6 +57,28 @@ export function assertPublicSafeAggregate(value: unknown): RedactionCheckResult }; } +export function assertPublicSafeTrendWindow(value: unknown): RedactionCheckResult { + const reasonCodes = collectRedactionReasonCodes(value); + if (!isRecord(value)) { + reasonCodes.push("not_object"); + } else { + if (value.public_safe !== true) reasonCodes.push("public_safe_not_true"); + if (value.contains_raw_text !== false) reasonCodes.push("contains_raw_text_not_false"); + if (value.contains_profile_urls !== false) reasonCodes.push("contains_profile_urls_not_false"); + if (typeof value.redaction_policy_version !== "string" || value.redaction_policy_version.length === 0) { + reasonCodes.push("missing_redaction_policy_version"); + } + if (value.schema_version !== "external-discussion-trend-window.v1") { + reasonCodes.push("invalid_trend_window_schema_version"); + } + } + + return { + ok: reasonCodes.length === 0, + reason_codes: Array.from(new Set(reasonCodes)).sort(), + }; +} + function collectRedactionReasonCodes(value: unknown): string[] { const reasonCodes: string[] = []; visit(value, (currentValue, key) => { diff --git a/src/externalDiscovery/trendWindow.ts b/src/externalDiscovery/trendWindow.ts new file mode 100644 index 0000000..af6852e --- /dev/null +++ b/src/externalDiscovery/trendWindow.ts @@ -0,0 +1,695 @@ +import fs from "node:fs"; +import { externalAggregatePath } from "./paths.ts"; +import { REDACTION_POLICY_VERSION, assertPublicSafeTrendWindow } from "./redaction.ts"; +import type { + DailyExternalAggregate, + ExternalDiscussionTrendWindow, + ExternalEvidence, + ExternalEvidenceScope, + ExternalNamedRegistryActor, + ExternalPlatform, + ExternalTrendBindingConfidence, + ExternalTrendComponent, + ExternalTrendComponentLevel, + ExternalTrendCoverage, + ExternalTrendDailyCount, + ExternalTrendItem, + ExternalTrendMomentum, + ExternalTrendVerdict, + ExternalWeeklyGateReason, +} from "./types.ts"; + +export type ExternalAggregateWindowReadResult = + | { status: "loaded"; date: string; path: string; aggregate: DailyExternalAggregate } + | { status: "missing"; date: string; path: string } + | { status: "failed"; date: string; path: string; reason_code: string; reason_detail: string }; + +export interface BuildExternalDiscussionTrendWindowInput { + anchorDate: string; + generatedAt: string; + aggregateResults?: ExternalAggregateWindowReadResult[]; +} + +const WEEKLY_GATE_REASONS: ExternalWeeklyGateReason[] = [ + "cross_platform_confirmation", + "multi_actor_confirmation", + "multi_day_persistence", + "registry_tier_participation", +]; + +export function externalTrendWindowDates(anchorDate: string): string[] { + const anchor = parseDateUtc(anchorDate); + return Array.from({ length: 7 }, (_, index) => formatDateUtc(addDays(anchor, index - 6))); +} + +export function readExternalAggregateWindow(anchorDate: string): ExternalAggregateWindowReadResult[] { + return externalTrendWindowDates(anchorDate).map((date) => { + const filepath = externalAggregatePath(date); + if (!fs.existsSync(filepath)) { + return { status: "missing", date, path: filepath }; + } + try { + const aggregate = JSON.parse(fs.readFileSync(filepath, "utf-8")) as DailyExternalAggregate; + if (aggregate.schema_version !== "external-discovery.aggregate.v1") { + return { + status: "failed", + date, + path: filepath, + reason_code: "window_aggregate_schema_mismatch", + reason_detail: `unexpected schema_version ${String((aggregate as { schema_version?: unknown }).schema_version)}`, + }; + } + return { status: "loaded", date, path: filepath, aggregate }; + } catch (error) { + return { + status: "failed", + date, + path: filepath, + reason_code: "window_aggregate_parse_failed", + reason_detail: error instanceof Error ? error.message : String(error), + }; + } + }); +} + +export function buildExternalDiscussionTrendWindow(input: BuildExternalDiscussionTrendWindowInput): ExternalDiscussionTrendWindow { + const expectedDates = externalTrendWindowDates(input.anchorDate); + const results = input.aggregateResults ?? readExternalAggregateWindow(input.anchorDate); + const coverage = buildCoverage(expectedDates, results); + const loadedAggregates = results + .filter((result): result is Extract => result.status === "loaded") + .map((result) => result.aggregate); + + const audit: ExternalDiscussionTrendWindow["audit"] = { + rejected_items: [], + warnings: [], + }; + + if (coverage.usable_day_count < 3) { + audit.warnings.push({ + reason_code: "window_usable_days_insufficient", + reason_detail: `usable days ${coverage.usable_day_count}/7`, + }); + } + + for (const missingDate of coverage.missing_dates) { + audit.warnings.push({ + reason_code: "window_aggregate_missing", + reason_detail: `missing aggregate for ${missingDate}`, + }); + } + for (const failedDate of coverage.failed_dates) { + audit.warnings.push({ + reason_code: failedDate.reason_code, + reason_detail: `${failedDate.date}: ${failedDate.reason_detail}`, + }); + } + + const projectTrends = buildTrendItems("project", expectedDates, loadedAggregates, coverage, audit); + const directionTrends = buildTrendItems("direction", expectedDates, loadedAggregates, coverage, audit); + const status = windowStatus(coverage, loadedAggregates, projectTrends, directionTrends); + const statusReason = windowStatusReason(status, coverage); + + const trendWindow: ExternalDiscussionTrendWindow = { + schema_version: "external-discussion-trend-window.v1", + anchor_date: input.anchorDate, + window_start: expectedDates[0]!, + window_end: expectedDates[expectedDates.length - 1]!, + window_days: 7, + generated_at: input.generatedAt, + status, + status_reason: statusReason, + project_trends: projectTrends, + direction_trends: directionTrends, + coverage, + audit, + public_safe: true, + redaction_policy_version: REDACTION_POLICY_VERSION, + contains_raw_text: false, + contains_profile_urls: false, + }; + + const safety = assertPublicSafeTrendWindow(trendWindow); + if (!safety.ok) { + throw new Error(`external discussion trend window is not public-safe: ${safety.reason_codes.join(",")}`); + } + + return trendWindow; +} + +export function buildSkippedExternalDiscussionTrendWindow(args: { + anchorDate: string; + generatedAt: string; + statusReason: string; +}): ExternalDiscussionTrendWindow { + const expectedDates = externalTrendWindowDates(args.anchorDate); + return { + schema_version: "external-discussion-trend-window.v1", + anchor_date: args.anchorDate, + window_start: expectedDates[0]!, + window_end: expectedDates[expectedDates.length - 1]!, + window_days: 7, + generated_at: args.generatedAt, + status: "skipped", + status_reason: args.statusReason, + project_trends: [], + direction_trends: [], + coverage: { + expected_dates: expectedDates, + loaded_dates: [], + missing_dates: [], + failed_dates: [], + usable_day_count: 0, + platform_counts: {}, + partial_platforms: [], + }, + audit: { + rejected_items: [], + warnings: [{ reason_code: "external_trend_window_skipped", reason_detail: args.statusReason }], + }, + public_safe: true, + redaction_policy_version: REDACTION_POLICY_VERSION, + contains_raw_text: false, + contains_profile_urls: false, + }; +} + +function buildCoverage(expectedDates: string[], results: ExternalAggregateWindowReadResult[]): ExternalTrendCoverage { + const byDate = new Map(results.map((result) => [result.date, result] as const)); + const loadedDates: string[] = []; + const missingDates: string[] = []; + const failedDates: ExternalTrendCoverage["failed_dates"] = []; + const platformCounts: Partial> = {}; + const partialPlatforms = new Set(); + let usableDayCount = 0; + + for (const date of expectedDates) { + const result = byDate.get(date); + if (!result || result.status === "missing") { + missingDates.push(date); + continue; + } + if (result.status === "failed") { + failedDates.push({ + date, + reason_code: result.reason_code, + reason_detail: result.reason_detail, + }); + continue; + } + + loadedDates.push(date); + if (result.aggregate.status === "ok" || result.aggregate.status === "partial") { + usableDayCount += 1; + } + if (result.aggregate.status === "failed") { + failedDates.push({ + date, + reason_code: "window_aggregate_failed_status", + reason_detail: result.aggregate.status_reason ?? "aggregate status failed", + }); + } + if (result.aggregate.status === "partial") { + for (const platform of Object.keys(result.aggregate.platform_counts) as ExternalPlatform[]) { + partialPlatforms.add(platform); + } + } + for (const [platform, count] of Object.entries(result.aggregate.platform_counts) as Array<[ExternalPlatform, number]>) { + platformCounts[platform] = (platformCounts[platform] ?? 0) + count; + } + } + + return { + expected_dates: expectedDates, + loaded_dates: loadedDates, + missing_dates: missingDates, + failed_dates: failedDates, + usable_day_count: usableDayCount, + platform_counts: platformCounts, + partial_platforms: [...partialPlatforms].sort(), + }; +} + +function buildTrendItems( + scope: ExternalEvidenceScope, + expectedDates: string[], + aggregates: DailyExternalAggregate[], + coverage: ExternalTrendCoverage, + audit: ExternalDiscussionTrendWindow["audit"], +): ExternalTrendItem[] { + const evidenceByTarget = new Map>(); + for (const aggregate of aggregates) { + const evidenceList = scope === "project" ? aggregate.project_evidence : aggregate.direction_evidence; + for (const evidence of evidenceList) { + const key = evidence.target_key.trim(); + if (!key) { + audit.rejected_items.push({ + scope, + reason_code: "trend_item_unstable_target_key", + reason_detail: `empty target_key in aggregate ${aggregate.date}`, + }); + continue; + } + evidenceByTarget.set(key, [...(evidenceByTarget.get(key) ?? []), { date: aggregate.date, evidence }]); + } + } + + return [...evidenceByTarget.entries()] + .map(([targetKey, entries]) => trendItemFromEvidence(scope, targetKey, expectedDates, entries, coverage, audit)) + .filter((item): item is ExternalTrendItem => Boolean(item)) + .sort((left, right) => right.mention_count_total - left.mention_count_total || left.target_key.localeCompare(right.target_key)); +} + +function trendItemFromEvidence( + scope: ExternalEvidenceScope, + targetKey: string, + expectedDates: string[], + entries: Array<{ date: string; evidence: ExternalEvidence }>, + coverage: ExternalTrendCoverage, + audit: ExternalDiscussionTrendWindow["audit"], +): ExternalTrendItem | null { + const stableTarget = isStableTargetKey(targetKey); + if (!stableTarget) { + audit.rejected_items.push({ + scope, + target_key: targetKey, + reason_code: scope === "direction" ? "direction_without_stable_topic_key" : "trend_item_unstable_target_key", + reason_detail: "target_key is empty or contains unsupported whitespace/control characters", + }); + return null; + } + + const evidenceByDate = new Map(entries.map((entry) => [entry.date, entry.evidence] as const)); + const dailyCounts: ExternalTrendDailyCount[] = expectedDates + .filter((date) => coverage.loaded_dates.includes(date)) + .map((date) => { + const evidence = evidenceByDate.get(date); + return { + date, + mention_count: evidence?.mention_count ?? 0, + source_count: evidence ? unique(evidence.event_ids).length : 0, + platform_count: evidence ? evidence.platforms.length : 0, + }; + }); + const evidenceItems = entries.map((entry) => entry.evidence); + const evidenceIds = unique(evidenceItems.map((evidence) => evidence.evidence_id)).sort(); + const sourceAggregateDates = unique(entries.map((entry) => entry.date)).sort(); + const platforms = unique(evidenceItems.flatMap((evidence) => evidence.platforms)); + const namedRegistryActors = mergeNamedRegistryActors(evidenceItems); + const mentionCountTotal = evidenceItems.reduce((total, evidence) => total + evidence.mention_count, 0); + const sourceCount = unique(evidenceItems.flatMap((evidence) => evidence.event_ids)).length; + const activeDayCount = dailyCounts.filter((count) => count.mention_count > 0).length; + const crossPlatformDays = dailyCounts.filter((count) => count.platform_count >= 2).length; + const topTierActorCount = unique(namedRegistryActors.map((actor) => actor.entity_id)).length; + const distinctActorCount = Math.max(0, ...evidenceItems.map((evidence) => evidence.distinct_actor_count)); + const officialSignal = platforms.includes("official_web") || platforms.includes("official_blog"); + const bindingConfidence = bindingConfidenceForTarget(scope, targetKey); + const caveats: string[] = []; + + if (distinctActorCount > 0 && evidenceItems.length > 1) { + caveats.push("distinct_actor_count uses conservative daily maximum because cross-evidence public actor identity is not always available"); + } + if (bindingConfidence === "low" && scope === "direction") { + caveats.push("direction-level signal is not bound to a specific repo, paper, or product"); + } + if (bindingConfidence === "low" && scope === "project") { + caveats.push("project trend target is not confidently bound to a canonical repo URL"); + } + if (coverage.partial_platforms.length > 0) { + caveats.push(`partial external window: ${coverage.partial_platforms.join(", ")}`); + } + + const components = buildComponents({ + mentionCountTotal, + activeDayCount, + platformCount: platforms.length, + crossPlatformDays, + distinctActorCount, + topTierActorCount, + bindingConfidence, + officialSignal, + coverage, + dailyCounts, + }); + const momentum = momentumFor(dailyCounts, coverage.usable_day_count, mentionCountTotal, activeDayCount); + const weeklyGateReasons = weeklyGateReasonsFor({ + platformCount: platforms.length, + distinctActorCount, + activeDayCount, + topTierActorCount, + }); + const weeklyGateMissingReasons = WEEKLY_GATE_REASONS.filter((reason) => !weeklyGateReasons.includes(reason)); + const verdict = verdictFor({ + scope, + momentum, + components, + bindingConfidence, + activeDayCount, + platformCount: platforms.length, + topTierActorCount, + officialSignal, + weeklyGateReasons, + }); + const weeklyEligible = weeklyEligibleFor(scope, verdict, weeklyGateReasons); + + if (verdict === "noise_spike") { + audit.rejected_items.push({ + scope, + target_key: targetKey, + reason_code: "trend_item_single_day_single_platform", + reason_detail: `${targetKey} is treated as noise risk rather than weekly positive material`, + }); + } + if (namedRegistryActors.length === 0) { + audit.warnings.push({ + reason_code: "named_actor_registry_empty", + reason_detail: `${targetKey} has no public-safe named registry actor hit`, + }); + } + + return { + trend_id: `${scope}:${stableHashInput(targetKey)}`, + scope, + target_key: targetKey, + display_name: displayNameForTarget(targetKey), + target_url: targetUrlForTarget(targetKey), + binding_confidence: bindingConfidence, + official_signal: officialSignal, + weekly_eligible: weeklyEligible, + weekly_gate_reasons: weeklyGateReasons, + weekly_gate_missing_reasons: weeklyGateMissingReasons, + daily_counts: dailyCounts, + mention_count_total: mentionCountTotal, + source_count: sourceCount, + active_day_count: activeDayCount, + platform_count: platforms.length, + cross_platform_days: crossPlatformDays, + distinct_actor_count: distinctActorCount, + top_tier_actor_count: topTierActorCount, + named_registry_actors: namedRegistryActors, + components, + momentum, + verdict, + cannot_be_primary_conclusion: true, + evidence_ids: evidenceIds, + source_aggregate_dates: sourceAggregateDates, + caveats, + }; +} + +function buildComponents(args: { + mentionCountTotal: number; + activeDayCount: number; + platformCount: number; + crossPlatformDays: number; + distinctActorCount: number; + topTierActorCount: number; + bindingConfidence: ExternalTrendBindingConfidence; + officialSignal: boolean; + coverage: ExternalTrendCoverage; + dailyCounts: ExternalTrendDailyCount[]; +}): ExternalTrendComponent[] { + const maxDailyShare = maxDailyMentionShare(args.dailyCounts, args.mentionCountTotal); + const noiseRisk = noiseRiskLevel({ + activeDayCount: args.activeDayCount, + platformCount: args.platformCount, + topTierActorCount: args.topTierActorCount, + officialSignal: args.officialSignal, + bindingConfidence: args.bindingConfidence, + mentionCountTotal: args.mentionCountTotal, + maxDailyShare, + partialWindow: args.coverage.partial_platforms.length > 0 || args.coverage.failed_dates.length > 0, + }); + + return [ + { + name: "discussion_volume", + level: bucket(args.mentionCountTotal, [ + [0, "none"], + [2, "low"], + [5, "medium"], + ]), + evidence: [`mention_count_total=${args.mentionCountTotal}`], + }, + { + name: "persistence", + level: args.activeDayCount === 0 ? "none" : args.activeDayCount === 1 ? "low" : args.activeDayCount <= 3 ? "medium" : "high", + evidence: [`active_day_count=${args.activeDayCount}`], + }, + { + name: "cross_platform_confirmation", + level: args.platformCount <= 1 ? "none" : args.crossPlatformDays >= 2 ? "high" : "medium", + evidence: [`platform_count=${args.platformCount}`, `cross_platform_days=${args.crossPlatformDays}`], + }, + { + name: "actor_authority", + level: args.topTierActorCount > 0 ? "high" : args.distinctActorCount >= 2 ? "medium" : args.distinctActorCount > 0 ? "low" : "none", + evidence: [`distinct_actor_count=${args.distinctActorCount}`, `top_tier_actor_count=${args.topTierActorCount}`], + }, + { + name: "binding_confidence", + level: bindingComponentLevel(args.bindingConfidence), + evidence: [`binding_confidence=${args.bindingConfidence}`], + }, + { + name: "noise_risk", + level: noiseRisk, + evidence: [`max_daily_share=${maxDailyShare.toFixed(2)}`], + }, + ]; +} + +function momentumFor( + dailyCounts: ExternalTrendDailyCount[], + usableDayCount: number, + mentionCountTotal: number, + activeDayCount: number, +): ExternalTrendMomentum { + if (usableDayCount < 3 || mentionCountTotal < 2) return "insufficient"; + + const maxDailyShare = maxDailyMentionShare(dailyCounts, mentionCountTotal); + if (activeDayCount === 1 || maxDailyShare >= 0.7) return "spike"; + + const early = dailyCounts.slice(0, 4); + const late = dailyCounts.slice(4); + if (early.length === 0 || late.length === 0) return "insufficient"; + + const earlyAvg = sumMentions(early) / early.length; + const lateAvg = sumMentions(late) / late.length; + const lastTwoLoaded = dailyCounts.slice(-2); + + if (activeDayCount >= 2 && lateAvg >= Math.max(earlyAvg + 1, earlyAvg * 1.5)) return "rising"; + if ( + activeDayCount >= 2 && + earlyAvg >= Math.max(lateAvg + 1, lateAvg * 1.5) && + lastTwoLoaded.length === 2 && + lastTwoLoaded.every((count) => count.mention_count === 0) + ) { + return "fading"; + } + if (activeDayCount >= 3 && maxDailyShare < 0.6) return "stable"; + return "insufficient"; +} + +function verdictFor(args: { + scope: ExternalEvidenceScope; + momentum: ExternalTrendMomentum; + components: ExternalTrendComponent[]; + bindingConfidence: ExternalTrendBindingConfidence; + activeDayCount: number; + platformCount: number; + topTierActorCount: number; + officialSignal: boolean; + weeklyGateReasons: ExternalWeeklyGateReason[]; +}): ExternalTrendVerdict { + if (args.momentum === "insufficient") return "insufficient"; + const noiseRisk = componentLevel(args.components, "noise_risk"); + if (noiseRisk === "high" && args.momentum === "spike") return "noise_spike"; + if (args.scope === "direction") { + return args.weeklyGateReasons.length >= 2 && noiseRisk !== "high" ? "watch_signal" : "noise_spike"; + } + if ( + args.bindingConfidence !== "none" && + args.activeDayCount >= 2 && + (args.platformCount >= 2 || args.topTierActorCount >= 1 || args.officialSignal) + ) { + return "external_reinforcement"; + } + return noiseRisk === "high" ? "noise_spike" : "watch_signal"; +} + +function weeklyGateReasonsFor(args: { + platformCount: number; + distinctActorCount: number; + activeDayCount: number; + topTierActorCount: number; +}): ExternalWeeklyGateReason[] { + const reasons: ExternalWeeklyGateReason[] = []; + if (args.platformCount >= 2) reasons.push("cross_platform_confirmation"); + if (args.distinctActorCount >= 2) reasons.push("multi_actor_confirmation"); + if (args.activeDayCount >= 2) reasons.push("multi_day_persistence"); + if (args.topTierActorCount >= 1) reasons.push("registry_tier_participation"); + return reasons; +} + +function weeklyEligibleFor(scope: ExternalEvidenceScope, verdict: ExternalTrendVerdict, reasons: ExternalWeeklyGateReason[]): boolean { + if (scope === "project") return verdict === "external_reinforcement"; + return verdict === "watch_signal" && reasons.length >= 2; +} + +function windowStatus( + coverage: ExternalTrendCoverage, + aggregates: DailyExternalAggregate[], + projectTrends: ExternalTrendItem[], + directionTrends: ExternalTrendItem[], +): ExternalDiscussionTrendWindow["status"] { + if (coverage.loaded_dates.length === 0 && coverage.failed_dates.length > 0) return "failed"; + if (coverage.usable_day_count < 3) return "insufficient"; + if ( + coverage.failed_dates.length > 0 || + coverage.missing_dates.length > 0 || + coverage.partial_platforms.length > 0 || + aggregates.some((aggregate) => aggregate.status === "partial") + ) { + return "partial"; + } + void projectTrends; + void directionTrends; + return "ok"; +} + +function windowStatusReason(status: ExternalDiscussionTrendWindow["status"], coverage: ExternalTrendCoverage): string | undefined { + if (status === "failed") return "window_aggregate_parse_failed"; + if (status === "insufficient") return "window_usable_days_insufficient"; + if (status === "partial") { + if (coverage.failed_dates.length > 0) return "window_aggregate_partial_failure"; + if (coverage.missing_dates.length > 0) return "window_aggregate_missing"; + if (coverage.partial_platforms.length > 0) return "window_platform_partial"; + } + return undefined; +} + +function mergeNamedRegistryActors(evidenceItems: ExternalEvidence[]): ExternalNamedRegistryActor[] { + const byEntity = new Map(); + for (const actor of evidenceItems.flatMap((evidence) => evidence.named_registry_actors)) { + const existing = byEntity.get(actor.entity_id); + if (!existing) { + byEntity.set(actor.entity_id, { ...actor, platforms: [...actor.platforms], source_roles: [...actor.source_roles] }); + continue; + } + existing.event_count += actor.event_count; + existing.platforms = unique([...existing.platforms, ...actor.platforms]).sort() as ExternalPlatform[]; + existing.source_roles = unique([...existing.source_roles, ...actor.source_roles]); + existing.first_seen_at = existing.first_seen_at < actor.first_seen_at ? existing.first_seen_at : actor.first_seen_at; + existing.last_seen_at = existing.last_seen_at > actor.last_seen_at ? existing.last_seen_at : actor.last_seen_at; + } + return [...byEntity.values()].sort((left, right) => right.event_count - left.event_count || left.display_name.localeCompare(right.display_name)); +} + +function bindingConfidenceForTarget(scope: ExternalEvidenceScope, targetKey: string): ExternalTrendBindingConfidence { + if (!isStableTargetKey(targetKey)) return "none"; + if (scope === "direction") return "low"; + return isRepoLikeTarget(targetKey) ? "medium" : "low"; +} + +function bindingComponentLevel(confidence: ExternalTrendBindingConfidence): ExternalTrendComponentLevel { + if (confidence === "high") return "high"; + if (confidence === "medium") return "medium"; + if (confidence === "low") return "low"; + return "none"; +} + +function noiseRiskLevel(args: { + activeDayCount: number; + platformCount: number; + topTierActorCount: number; + officialSignal: boolean; + bindingConfidence: ExternalTrendBindingConfidence; + mentionCountTotal: number; + maxDailyShare: number; + partialWindow: boolean; +}): ExternalTrendComponentLevel { + if (args.mentionCountTotal === 0) return "high"; + if ( + (args.activeDayCount === 1 && args.platformCount === 1 && args.topTierActorCount === 0 && !args.officialSignal) || + args.bindingConfidence === "none" || + (args.mentionCountTotal >= 3 && args.maxDailyShare >= 0.8 && args.platformCount < 2) + ) { + return "high"; + } + if (args.mentionCountTotal <= 2 || args.platformCount === 1 || args.bindingConfidence === "low" || args.partialWindow) { + return "medium"; + } + return "low"; +} + +function bucket(value: number, ranges: Array<[number, ExternalTrendComponentLevel]>): ExternalTrendComponentLevel { + for (const [max, level] of ranges) { + if (value <= max) return level; + } + return "high"; +} + +function componentLevel(components: ExternalTrendComponent[], name: ExternalTrendComponent["name"]): ExternalTrendComponentLevel { + return components.find((component) => component.name === name)?.level ?? "none"; +} + +function maxDailyMentionShare(dailyCounts: ExternalTrendDailyCount[], total: number): number { + if (total <= 0) return 0; + return Math.max(0, ...dailyCounts.map((count) => count.mention_count)) / total; +} + +function sumMentions(counts: ExternalTrendDailyCount[]): number { + return counts.reduce((total, count) => total + count.mention_count, 0); +} + +function isStableTargetKey(targetKey: string): boolean { + return targetKey.trim().length > 0 && !/[\r\n\t]/.test(targetKey); +} + +function displayNameForTarget(targetKey: string): string { + const repo = repoFullNameFromTarget(targetKey); + if (repo) return repo; + return targetKey.replace(/[_-]+/g, " ").replace(/\s+/g, " ").trim() || targetKey; +} + +function targetUrlForTarget(targetKey: string): string | undefined { + const repo = repoFullNameFromTarget(targetKey); + return repo ? `https://github.com/${repo}` : undefined; +} + +function isRepoLikeTarget(targetKey: string): boolean { + return Boolean(repoFullNameFromTarget(targetKey)); +} + +function repoFullNameFromTarget(targetKey: string): string | undefined { + const trimmed = targetKey.trim().replace(/\.git$/i, ""); + const githubMatch = /^https?:\/\/github\.com\/([^/\s]+\/[^/\s#?]+)/i.exec(trimmed); + if (githubMatch) return githubMatch[1]!.toLowerCase(); + if (/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(trimmed)) return trimmed.toLowerCase(); + return undefined; +} + +function stableHashInput(targetKey: string): string { + return targetKey.trim().toLowerCase().replace(/[^a-z0-9._/-]+/g, "-").replace(/^-+|-+$/g, "") || "unknown"; +} + +function parseDateUtc(date: string): Date { + const parsed = Date.parse(`${date}T00:00:00.000Z`); + if (!Number.isFinite(parsed)) throw new Error(`invalid date: ${date}`); + return new Date(parsed); +} + +function addDays(date: Date, days: number): Date { + const copy = new Date(date.getTime()); + copy.setUTCDate(copy.getUTCDate() + days); + return copy; +} + +function formatDateUtc(date: Date): string { + return date.toISOString().slice(0, 10); +} + +function unique(values: T[]): T[] { + return [...new Set(values)]; +} diff --git a/src/externalDiscovery/trendWindowIntegration.ts b/src/externalDiscovery/trendWindowIntegration.ts new file mode 100644 index 0000000..3cdc9a0 --- /dev/null +++ b/src/externalDiscovery/trendWindowIntegration.ts @@ -0,0 +1,148 @@ +import fs from "node:fs"; +import { writeJsonFile } from "../storage/files.ts"; +import { externalTrendWindowLatestPath, externalTrendWindowPath } from "./paths.ts"; +import { assertPublicSafeTrendWindow } from "./redaction.ts"; +import { + buildExternalDiscussionTrendWindow, + buildSkippedExternalDiscussionTrendWindow, + readExternalAggregateWindow, + type ExternalAggregateWindowReadResult, +} from "./trendWindow.ts"; +import type { DailyExternalAggregate, ExternalDiscussionTrendWindow } from "./types.ts"; +import type { ExternalTrendWindowReadStatus, ExternalTrendWindowStatus } from "./types.ts"; + +export interface DailyExternalTrendWindowIntegrationResult { + trend_window: ExternalDiscussionTrendWindow; + paths: { + trend_window: string; + trend_window_latest: string; + }; +} + +export interface ExternalDiscussionTrendWindowReadResult { + read_status: ExternalTrendWindowReadStatus; + path: string; + trend_window?: ExternalDiscussionTrendWindow; + error?: string; +} + +const TREND_WINDOW_ARTIFACT_STATUSES: ExternalTrendWindowStatus[] = [ + "ok", + "partial", + "failed", + "insufficient", + "skipped", +]; + +export function readExternalDiscussionTrendWindowByDate(date: string): ExternalDiscussionTrendWindowReadResult { + return readExternalDiscussionTrendWindowFromPath(externalTrendWindowPath(date)); +} + +export function readLatestExternalDiscussionTrendWindow(): ExternalDiscussionTrendWindowReadResult { + return readExternalDiscussionTrendWindowFromPath(externalTrendWindowLatestPath()); +} + +function readExternalDiscussionTrendWindowFromPath(filepath: string): ExternalDiscussionTrendWindowReadResult { + if (!fs.existsSync(filepath)) { + return { read_status: "not_found", path: filepath }; + } + + try { + const trendWindow = JSON.parse(fs.readFileSync(filepath, "utf-8")) as ExternalDiscussionTrendWindow; + const validationError = validateTrendWindowArtifact(trendWindow); + if (validationError) { + return { read_status: "parse_error", path: filepath, error: validationError }; + } + return { + read_status: trendWindow.status, + path: filepath, + trend_window: trendWindow, + }; + } catch (error) { + return { + read_status: "parse_error", + path: filepath, + error: error instanceof Error ? error.message : String(error), + }; + } +} + +function validateTrendWindowArtifact(value: ExternalDiscussionTrendWindow): string | undefined { + if (!value || typeof value !== "object") return "trend window artifact must be an object"; + if (value.schema_version !== "external-discussion-trend-window.v1") { + return `unexpected schema_version ${String((value as { schema_version?: unknown }).schema_version)}`; + } + if (!TREND_WINDOW_ARTIFACT_STATUSES.includes(value.status)) { + return `unexpected status ${String((value as { status?: unknown }).status)}`; + } + const safety = assertPublicSafeTrendWindow(value); + if (!safety.ok) { + return `public-safe check failed: ${safety.reason_codes.join(",")}`; + } + return undefined; +} + +export function writeExternalDiscussionTrendWindow(args: { + date: string; + trendWindow: ExternalDiscussionTrendWindow; + dryRun?: boolean; +}): DailyExternalTrendWindowIntegrationResult["paths"] { + const safety = assertPublicSafeTrendWindow(args.trendWindow); + if (!safety.ok) { + throw new Error(`external discussion trend window is not public-safe: ${safety.reason_codes.join(",")}`); + } + + const trendWindowPath = externalTrendWindowPath(args.date); + const trendWindowLatestPath = externalTrendWindowLatestPath(); + writeJsonFile(trendWindowPath, args.trendWindow, args.dryRun); + writeJsonFile(trendWindowLatestPath, args.trendWindow, args.dryRun); + + return { + trend_window: trendWindowPath, + trend_window_latest: trendWindowLatestPath, + }; +} + +export function runDailyExternalTrendWindowIntegration(args: { + date: string; + generatedAt: string; + dryRun?: boolean; + currentAggregate: DailyExternalAggregate; +}): DailyExternalTrendWindowIntegrationResult { + const trendWindow = + args.currentAggregate.status === "skipped" + ? buildSkippedExternalDiscussionTrendWindow({ + anchorDate: args.date, + generatedAt: args.generatedAt, + statusReason: args.currentAggregate.status_reason ?? "external_discovery_skipped", + }) + : buildExternalDiscussionTrendWindow({ + anchorDate: args.date, + generatedAt: args.generatedAt, + aggregateResults: currentAggregateWindow(args.date, args.currentAggregate), + }); + + const paths = writeExternalDiscussionTrendWindow({ + date: args.date, + trendWindow, + dryRun: args.dryRun, + }); + + return { + trend_window: trendWindow, + paths, + }; +} + +function currentAggregateWindow(date: string, aggregate: DailyExternalAggregate): ExternalAggregateWindowReadResult[] { + return readExternalAggregateWindow(date).map((result) => + result.date === date + ? { + status: "loaded", + date, + path: result.path, + aggregate, + } + : result, + ); +} diff --git a/src/externalDiscovery/types.ts b/src/externalDiscovery/types.ts index c1a87d5..614727d 100644 --- a/src/externalDiscovery/types.ts +++ b/src/externalDiscovery/types.ts @@ -10,12 +10,52 @@ export type ExternalProviderTierHint = "core" | "proven" | "watch" | "ordinary" export type ExternalEvidenceScope = "project" | "direction"; export type ExternalTierBasis = "registry" | "provider_hint" | "none"; export type ExternalNamedActorSourceRole = "social_discussant" | "official_publisher" | "official_owner"; +export type ExternalTrendWindowStatus = "ok" | "partial" | "failed" | "insufficient" | "skipped"; +export type ExternalTrendWindowReadStatus = + | "ok" + | "not_found" + | "parse_error" + | "partial" + | "insufficient" + | "failed" + | "skipped"; +export type ExternalTrendBindingConfidence = "high" | "medium" | "low" | "none"; +export type ExternalTrendMomentum = "spike" | "rising" | "stable" | "fading" | "insufficient"; +export type ExternalTrendVerdict = "external_reinforcement" | "watch_signal" | "noise_spike" | "insufficient"; +export type ExternalWeeklyGateReason = + | "cross_platform_confirmation" + | "multi_actor_confirmation" + | "multi_day_persistence" + | "registry_tier_participation"; +export type ExternalTrendComponentName = + | "discussion_volume" + | "persistence" + | "cross_platform_confirmation" + | "actor_authority" + | "binding_confidence" + | "noise_risk"; +export type ExternalTrendComponentLevel = "none" | "low" | "medium" | "high"; export const EXTERNAL_PLATFORMS = ["x_twitter", "reddit", "hacker_news", "official_web", "official_blog"] as const; export const EXTERNAL_TARGET_TYPES = ["project", "paper", "product", "topic"] as const; export const EXTERNAL_ACTOR_TYPES = ["institution", "team", "person", "community", "unknown"] as const; export const EXTERNAL_REGISTRY_TIERS = ["core", "proven", "watch"] as const; export const EXTERNAL_NAMED_ACTOR_SOURCE_ROLES = ["social_discussant", "official_publisher", "official_owner"] as const; +export const EXTERNAL_TREND_WINDOW_READ_STATUSES = ["ok", "not_found", "parse_error", "partial", "insufficient", "failed", "skipped"] as const; +export const EXTERNAL_WEEKLY_GATE_REASONS = [ + "cross_platform_confirmation", + "multi_actor_confirmation", + "multi_day_persistence", + "registry_tier_participation", +] as const; +export const EXTERNAL_TREND_COMPONENT_NAMES = [ + "discussion_volume", + "persistence", + "cross_platform_confirmation", + "actor_authority", + "binding_confidence", + "noise_risk", +] as const; export interface ExternalSignalActor { actor_type: ExternalActorType; @@ -125,6 +165,94 @@ export interface DailyExternalAggregate { audit: ExternalDiscoveryAudit; } +export interface ExternalTrendDailyCount { + date: string; + mention_count: number; + source_count: number; + platform_count: number; +} + +export interface ExternalTrendComponent { + name: ExternalTrendComponentName; + level: ExternalTrendComponentLevel; + evidence: string[]; +} + +export interface ExternalTrendItem { + trend_id: string; + scope: ExternalEvidenceScope; + target_key: string; + display_name: string; + target_url?: string; + binding_confidence: ExternalTrendBindingConfidence; + official_signal: boolean; + weekly_eligible: boolean; + weekly_gate_reasons: ExternalWeeklyGateReason[]; + weekly_gate_missing_reasons: ExternalWeeklyGateReason[]; + daily_counts: ExternalTrendDailyCount[]; + mention_count_total: number; + source_count: number; + active_day_count: number; + platform_count: number; + cross_platform_days: number; + distinct_actor_count: number; + top_tier_actor_count: number; + named_registry_actors: ExternalNamedRegistryActor[]; + components: ExternalTrendComponent[]; + momentum: ExternalTrendMomentum; + verdict: ExternalTrendVerdict; + cannot_be_primary_conclusion: true; + evidence_ids: string[]; + source_aggregate_dates: string[]; + caveats: string[]; +} + +export interface ExternalTrendCoverage { + expected_dates: string[]; + loaded_dates: string[]; + missing_dates: string[]; + failed_dates: Array<{ + date: string; + reason_code: string; + reason_detail: string; + }>; + usable_day_count: number; + platform_counts: Partial>; + partial_platforms: ExternalPlatform[]; +} + +export interface ExternalTrendAudit { + rejected_items: Array<{ + scope?: ExternalEvidenceScope; + target_key?: string; + reason_code: string; + reason_detail: string; + }>; + warnings: Array<{ + reason_code: string; + reason_detail: string; + }>; +} + +export interface ExternalDiscussionTrendWindow { + schema_version: "external-discussion-trend-window.v1"; + anchor_date: string; + window_start: string; + window_end: string; + window_days: 7; + generated_at: string; + status: ExternalTrendWindowStatus; + status_reason?: string; + project_trends: ExternalTrendItem[]; + direction_trends: ExternalTrendItem[]; + coverage: ExternalTrendCoverage; + audit: ExternalTrendAudit; + public_safe: true; + redaction_policy_version: string; + contains_raw_text: false; + contains_profile_urls: false; +} + export interface ProviderRejectedEvent { event_id?: string; reason_code: string; diff --git a/src/types.ts b/src/types.ts index 1acdf61..5fc8537 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,5 +1,13 @@ -export type SignalSource = - | "agents-radar" +import type { + ExternalTrendBindingConfidence, + ExternalTrendMomentum, + ExternalTrendVerdict, + ExternalTrendWindowReadStatus, + ExternalTrendWindowStatus, +} from "./externalDiscovery/types.ts"; + +export type SignalSource = + | "agents-radar" | "trendshift" | "github_trending" | "github_live_star_delta" @@ -475,16 +483,49 @@ export interface WeeklyEvidenceAxis { summary_cn: string; } -export interface WeeklyEvidenceMatrix { - focused_trend_key: string | null; - focused_trend_name_cn: string | null; - summary_cn: string; - axes: WeeklyEvidenceAxis[]; -} - -export interface FinalWeeklyTrend { - trend_id: string; - trend_name_cn: string; +export interface WeeklyEvidenceMatrix { + focused_trend_key: string | null; + focused_trend_name_cn: string | null; + summary_cn: string; + axes: WeeklyEvidenceAxis[]; +} + +export interface WeeklyExternalDiscussionTrendItem { + scope: "project" | "direction"; + target_key: string; + display_name: string; + target_url?: string; + binding_confidence: ExternalTrendBindingConfidence; + official_signal: boolean; + weekly_eligible: boolean; + momentum: ExternalTrendMomentum; + verdict: ExternalTrendVerdict; + mention_count_total: number; + source_count: number; + active_day_count: number; + platform_count: number; + named_registry_actor_count: number; + evidence_ids: string[]; + caveats: string[]; +} + +export interface WeeklyExternalDiscussionTrendSummary { + read_status: ExternalTrendWindowReadStatus; + status?: ExternalTrendWindowStatus; + path: string; + usable_day_count: number; + project_trend_count: number; + direction_trend_count: number; + failed_date_count: number; + missing_date_count: number; + secondary_evidence: WeeklyExternalDiscussionTrendItem[]; + direction_observations: WeeklyExternalDiscussionTrendItem[]; + noise_items: WeeklyExternalDiscussionTrendItem[]; +} + +export interface FinalWeeklyTrend { + trend_id: string; + trend_name_cn: string; claim_cn: string; why_established_cn: string; supporting_candidate_ids: string[]; @@ -564,11 +605,12 @@ export interface WeeklyReport { personalized_weekly_focus_applicable: boolean; personalized_weekly_focus_note_cn?: string; overall_summary_cn: string; - supporting_trend_keys: string[]; - evidence_matrix?: WeeklyEvidenceMatrix; - core_trend_cards: CoreTrendCard[]; - personalized_weekly_focus: PersonalizedWeeklyFocus[]; - weak_signal_cards: WeakSignalCard[]; + supporting_trend_keys: string[]; + evidence_matrix?: WeeklyEvidenceMatrix; + external_discussion_trends?: WeeklyExternalDiscussionTrendSummary; + core_trend_cards: CoreTrendCard[]; + personalized_weekly_focus: PersonalizedWeeklyFocus[]; + weak_signal_cards: WeakSignalCard[]; enhancement_audit: EnhancementAudit; } @@ -595,11 +637,12 @@ export interface WeeklyJudgmentReport { executive_summary_cn: string; rule_materials: WeeklyJudgmentRuleMaterials; established_trends: FinalWeeklyTrend[]; - observing_trends: FinalWeeklyTrendObservation[]; - audit_conclusion: WeeklyAuditConclusion; - evidence_matrix?: WeeklyEvidenceMatrix; - enhancement_audit: EnhancementAudit; -} + observing_trends: FinalWeeklyTrendObservation[]; + audit_conclusion: WeeklyAuditConclusion; + evidence_matrix?: WeeklyEvidenceMatrix; + external_discussion_trends?: WeeklyExternalDiscussionTrendSummary; + enhancement_audit: EnhancementAudit; +} export interface DailyRunSummarySourceStatus { source: SignalSource | "github-enrichment"; @@ -806,6 +849,14 @@ export interface DailyRunSummaryExternalDiscovery { explanation_enhanced_count: number; explanation_fallback_count: number; explanation_rejected_count: number; + trend_window_read_status: ExternalTrendWindowReadStatus; + trend_window_status?: ExternalTrendWindowStatus; + trend_window_path: string; + trend_window_usable_day_count: number; + trend_window_project_trend_count: number; + trend_window_direction_trend_count: number; + trend_window_failed_date_count: number; + trend_window_missing_date_count: number; warning_count: number; warnings: string[]; } From a883ff3bfeceabb107b810e0f6fcdd84b773a411 Mon Sep 17 00:00:00 2001 From: Aspetta <892024285@qq.com> Date: Sun, 5 Jul 2026 22:19:16 +0800 Subject: [PATCH 6/8] feat(agentreach): persist public actor extraction backend --- .../externalDiscoveryAdapter.test.ts | 81 +++ .../externalDiscoveryAggregate.test.ts | 57 ++ ...overyCandidateExplanationDuplicate.test.ts | 112 ++++ ...DiscoveryCandidateExplanationInput.test.ts | 76 +++ .../externalDiscoveryPublicActors.test.ts | 264 +++++++++ ...lDiscoveryPublicActorsTypeContract.test.ts | 89 +++ .../externalDiscoveryRedaction.test.ts | 86 +++ .../externalDiscoveryVerification.test.ts | 51 ++ src/action/dailyVerification.ts | 130 +++++ src/externalDiscovery/agentReachProvider.ts | 95 ++- src/externalDiscovery/aggregate.ts | 4 + src/externalDiscovery/explanations.ts | 141 ++++- src/externalDiscovery/publicActors.ts | 547 ++++++++++++++++++ src/externalDiscovery/redaction.ts | 141 +++++ src/externalDiscovery/types.ts | 103 ++++ src/llm.ts | 54 +- src/providers/deepseek.ts | 9 +- src/providers/openai-compatible.ts | 40 +- 18 files changed, 2026 insertions(+), 54 deletions(-) create mode 100644 src/__tests__/externalDiscoveryCandidateExplanationDuplicate.test.ts create mode 100644 src/__tests__/externalDiscoveryPublicActors.test.ts create mode 100644 src/__tests__/externalDiscoveryPublicActorsTypeContract.test.ts create mode 100644 src/externalDiscovery/publicActors.ts diff --git a/src/__tests__/externalDiscoveryAdapter.test.ts b/src/__tests__/externalDiscoveryAdapter.test.ts index 96be54f..985ad32 100644 --- a/src/__tests__/externalDiscoveryAdapter.test.ts +++ b/src/__tests__/externalDiscoveryAdapter.test.ts @@ -433,6 +433,87 @@ describe("agent reach provider artifact adapter", () => { expect(result.rejected_events[0]?.reason_code).toBe("event_schema_invalid"); }); + it("preserves public actor aliases and canonical identity status", () => { + const filepath = tempFile("agent-reach-public-actor-aliases.json"); + fs.writeFileSync( + filepath, + JSON.stringify({ + provider: "agent-reach", + schema_version: "agent-reach.external-discovery.v1", + provider_run_id: "run-public-actor-aliases", + generated_at: "2026-07-05T00:00:00.000Z", + query: { keyword: "agents sdk" }, + platforms: ["reddit"], + status: "ok", + items: [ + { + event_id: "evt-reddit-aliases", + platform: "reddit", + raw_event_kind: "discussion", + derived_signal_kinds: ["evidence"], + scope: "project", + target_type: "project", + target_key: "openai/agents-sdk", + actor: { + actor_type: "community", + username: "agentbuilder", + subreddit: "LocalLLaMA", + }, + observed_at: "2026-07-05T00:00:00.000Z", + source_url: "https://www.reddit.com/r/LocalLLaMA/comments/abc/project_discussion/", + }, + ], + }), + "utf-8", + ); + + const result = readAgentReachProviderArtifact(filepath, { explicitInput: true }); + const event = result.events[0]!; + + expect(event.url).toBe("https://www.reddit.com/r/LocalLLaMA/comments/abc/project_discussion/"); + expect(event.source_url).toBe("https://www.reddit.com/r/LocalLLaMA/comments/abc/project_discussion/"); + expect(event.actor.username).toBe("agentbuilder"); + expect(event.actor.subreddit).toBe("LocalLLaMA"); + expect(event.actor_public_identity_status).toBe("available"); + expect(event.actor_public_identity_reason).toBe("actor_public_identity_available"); + }); + + it("marks reserved X status URLs as invalid public identity inputs", () => { + const filepath = tempFile("agent-reach-reserved-x-url.json"); + fs.writeFileSync( + filepath, + JSON.stringify({ + provider: "agent-reach", + schema_version: "agent-reach.external-discovery.v1", + provider_run_id: "run-reserved-x-url", + generated_at: "2026-07-05T00:00:00.000Z", + query: { keyword: "agents sdk" }, + platforms: ["x_twitter"], + status: "ok", + items: [ + { + event_id: "evt-reserved-x-url", + platform: "x_twitter", + raw_event_kind: "discussion", + derived_signal_kinds: ["evidence"], + scope: "project", + target_type: "project", + target_key: "openai/agents-sdk", + actor: { actor_type: "unknown" }, + observed_at: "2026-07-05T00:00:00.000Z", + url: "https://x.com/i/status/123", + }, + ], + }), + "utf-8", + ); + + const result = readAgentReachProviderArtifact(filepath, { explicitInput: true }); + + expect(result.events[0]?.actor_public_identity_status).toBe("invalid_reserved_path"); + expect(result.events[0]?.actor_public_identity_reason).toBe("x_reserved_or_indirect_url"); + }); + it("fails ok artifacts that miss required top-level audit fields", () => { const filepath = tempFile("agent-reach-missing-audit.json"); fs.writeFileSync( diff --git a/src/__tests__/externalDiscoveryAggregate.test.ts b/src/__tests__/externalDiscoveryAggregate.test.ts index 15b2075..74a7550 100644 --- a/src/__tests__/externalDiscoveryAggregate.test.ts +++ b/src/__tests__/externalDiscoveryAggregate.test.ts @@ -89,6 +89,63 @@ describe("external discovery aggregate", () => { expect(evidence.actor_types).toMatchObject({ person: 1 }); }); + it("publishes public actors only when they are explicit provider fields or URL-derived sources", () => { + const aggregate = buildDailyExternalAggregate({ + date: "2026-06-30", + generated_at: "2026-06-30T01:00:00.000Z", + events: [ + event({ + event_id: "evt-x", + platform: "x_twitter", + url: "https://x.com/openai/status/1", + actor: { + actor_type: "institution", + effective_tier: "proven", + tier_basis: "provider_hint", + handle: "openai", + }, + }), + event({ + event_id: "evt-reddit", + platform: "reddit", + url: "https://www.reddit.com/r/LocalLLaMA/comments/abc/project_discussion/", + actor: { + actor_type: "community", + effective_tier: "ordinary", + tier_basis: "none", + }, + observed_at: "2026-06-30T02:00:00.000Z", + }), + event({ + event_id: "evt-github", + platform: "official_web", + raw_event_kind: "official_release", + url: "https://github.com/anthropics/claude-code/releases/tag/v1", + target_repo_url: "https://github.com/anthropics/claude-code", + actor: { + actor_type: "team", + effective_tier: "watch", + tier_basis: "none", + }, + observed_at: "2026-06-30T03:00:00.000Z", + }), + ], + }); + + const evidence = aggregate.project_evidence[0]!; + expect((evidence.public_actors ?? []).map((actor) => ({ + id: actor.public_actor_id, + name: actor.display_name, + kind: actor.source_kind, + platforms: actor.platforms, + }))).toEqual(expect.arrayContaining([ + { id: "github:anthropics", name: "GitHub anthropics", kind: "github_owner", platforms: ["official_web"] }, + { id: "reddit:r:localllama", name: "r/LocalLLaMA", kind: "reddit_community", platforms: ["reddit"] }, + { id: "x:openai", name: "@openai", kind: "x_handle", platforms: ["x_twitter"] }, + ])); + expect(JSON.stringify(evidence.public_actors ?? [])).not.toContain("x.com/openai"); + }); + it("sorts named registry actors by tier, count, then name", () => { const aggregate = buildDailyExternalAggregate({ date: "2026-06-30", diff --git a/src/__tests__/externalDiscoveryCandidateExplanationDuplicate.test.ts b/src/__tests__/externalDiscoveryCandidateExplanationDuplicate.test.ts new file mode 100644 index 0000000..39702f2 --- /dev/null +++ b/src/__tests__/externalDiscoveryCandidateExplanationDuplicate.test.ts @@ -0,0 +1,112 @@ +import { describe, expect, it, vi } from "vitest"; +import type { AppConfig } from "../config.ts"; +import type { ExternalSignalEvent } from "../externalDiscovery/types.ts"; +import type { ScoredProject } from "../types.ts"; + +const structuredEnhancementMock = vi.hoisted(() => vi.fn()); + +vi.mock("../action/enhancementLlm.ts", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + callStructuredEnhancement: structuredEnhancementMock, + }; +}); + +const { buildDailyExternalAggregate } = await import("../externalDiscovery/aggregate.ts"); +const { buildCandidateExplanationInputs, generateExternalCandidateExplanations } = await import("../externalDiscovery/explanations.ts"); +const { assertPublicSafeCandidateExplanations } = await import("../externalDiscovery/explanationRedaction.ts"); + +function config(): AppConfig { + return { + llm: { + enabled: true, + mode: "semantic-classification", + provider: "deepseek", + }, + } as AppConfig; +} + +function event(overrides: Partial = {}): ExternalSignalEvent { + return { + event_id: "evt-1", + platform: "hacker_news", + raw_event_kind: "discussion", + derived_signal_kinds: ["discovery"], + scope: "project", + target_type: "project", + target_key: "owner/project-a", + actor: { actor_type: "community", effective_tier: "ordinary", tier_basis: "none" }, + observed_at: "2026-06-30T00:00:00.000Z", + raw_ref: "provider:event:1", + ...overrides, + }; +} + +describe("external candidate explanation duplicate handling", () => { + it("downgrades duplicate enhanced summaries instead of failing the whole artifact", async () => { + structuredEnhancementMock.mockResolvedValueOnce({ + explanations: [ + { + candidate_key: "project:owner/project-a", + what_it_is_cn: "这是一个用于构建 AI 工作流的项目候选。", + why_watch_cn: "外部来源出现讨论，适合作为次级证据继续观察。", + summary_confidence: "medium", + caveats: ["仍需主链路确认。"], + }, + { + candidate_key: "project:owner/project-b", + what_it_is_cn: "这是一个用于构建 AI 工作流的项目候选。", + why_watch_cn: "外部来源出现讨论，适合作为次级证据继续观察。", + summary_confidence: "medium", + caveats: ["仍需主链路确认。"], + }, + ], + }); + + const aggregate = buildDailyExternalAggregate({ + date: "2026-06-30", + generated_at: "2026-06-30T01:00:00.000Z", + events: [ + event(), + event({ + event_id: "evt-2", + target_key: "owner/project-b", + raw_ref: "provider:event:2", + }), + ], + }); + const scoredProjects = [ + { + project: { + repo_full_name: "owner/project-a", + repo_url: "https://github.com/owner/project-a", + project_name: "Project A", + description: "Build AI workflow automation.", + }, + }, + { + project: { + repo_full_name: "owner/project-b", + repo_url: "https://github.com/owner/project-b", + project_name: "Project B", + description: "Coordinate AI workflow agents.", + }, + }, + ] as ScoredProject[]; + const inputBuild = buildCandidateExplanationInputs({ aggregate, scoredProjects }); + + const artifact = await generateExternalCandidateExplanations({ + aggregate, + inputBuild, + config: config(), + generatedAt: "2026-06-30T02:00:00.000Z", + }); + + expect(artifact.status).toBe("partial"); + expect(artifact.audit.enhanced_count).toBe(1); + expect(artifact.audit.fallback_count).toBe(1); + expect(artifact.audit.warnings.map((warning) => warning.reason_code)).toContain("duplicate_summary_fallback"); + expect(assertPublicSafeCandidateExplanations(artifact).ok).toBe(true); + }); +}); diff --git a/src/__tests__/externalDiscoveryCandidateExplanationInput.test.ts b/src/__tests__/externalDiscoveryCandidateExplanationInput.test.ts index a5e6a1c..5024dc0 100644 --- a/src/__tests__/externalDiscoveryCandidateExplanationInput.test.ts +++ b/src/__tests__/externalDiscoveryCandidateExplanationInput.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; import { buildCandidateExplanationInputs } from "../externalDiscovery/explanations.ts"; +import { assertPublicSafeCandidateExplanationInputs } from "../externalDiscovery/explanationRedaction.ts"; import type { DailyExternalAggregate, ExternalSignalEvent } from "../externalDiscovery/types.ts"; import { buildDailyExternalAggregate } from "../externalDiscovery/aggregate.ts"; import type { ProjectLibraryEnhancementArtifact, ScoredProject } from "../types.ts"; @@ -101,4 +102,79 @@ describe("external candidate explanation input builder", () => { expect(result.warnings.map((item) => item.reason_code)).toContain("public_title_context_missing"); expect(result.warnings.map((item) => item.reason_code)).toContain("direction_candidate_low_confidence"); }); + + it("removes urls from natural-language explanation inputs before public-safe validation", () => { + const scored = [ + { + project: { + repo_full_name: "the-open-agent/openagent", + repo_url: "https://github.com/the-open-agent/openagent", + project_name: "OpenAgent", + description: "Personal AI assistant with browser-use and coding agent, demo: https://demo.openagentai.org", + }, + }, + ] as ScoredProject[]; + + const result = buildCandidateExplanationInputs({ + aggregate: aggregate([ + event({ + target_key: "the-open-agent/openagent", + }), + ]), + titleContext: [ + { + event_id: "evt-1", + target_key: "the-open-agent/openagent", + target_display_name: "OpenAgent", + public_evidence_title: "OpenAgent demo https://demo.openagentai.org", + public_source_title: "[Launch page](https://demo.openagentai.org)", + source_platform: "hacker_news", + }, + ], + scoredProjects: scored, + }); + + expect(result.inputs[0]?.repo_description).not.toContain("https://"); + expect(result.inputs[0]?.public_evidence_titles.join(" ")).not.toContain("https://"); + expect(result.inputs[0]?.public_source_titles.join(" ")).not.toContain("https://"); + expect(assertPublicSafeCandidateExplanationInputs(result.inputs).ok).toBe(true); + }); + + it("reserves explanation slots for direction and official candidates when project candidates exceed the limit", () => { + const projectEvents = Array.from({ length: 40 }, (_, index) => + event({ + event_id: `project-${index}`, + target_key: `owner/project-${index}`, + raw_ref: `provider:project:${index}`, + }), + ); + const directionEvents = Array.from({ length: 5 }, (_, index) => + event({ + event_id: `direction-${index}`, + scope: "direction", + target_type: "topic", + target_key: `agent workflow direction ${index}`, + derived_signal_kinds: ["discovery"], + raw_ref: `provider:direction:${index}`, + }), + ); + const officialEvents = Array.from({ length: 3 }, (_, index) => + event({ + event_id: `official-${index}`, + platform: "official_web", + raw_event_kind: "official_release", + target_key: `official/project-${index}`, + raw_ref: `provider:official:${index}`, + }), + ); + + const result = buildCandidateExplanationInputs({ + aggregate: aggregate([...projectEvents, ...directionEvents, ...officialEvents]), + topN: 30, + }); + + expect(result.inputs).toHaveLength(30); + expect(result.inputs.some((input) => input.explanation_scope === "direction_signal")).toBe(true); + expect(result.inputs.some((input) => input.platforms.includes("official_web"))).toBe(true); + }); }); diff --git a/src/__tests__/externalDiscoveryPublicActors.test.ts b/src/__tests__/externalDiscoveryPublicActors.test.ts new file mode 100644 index 0000000..ccfa972 --- /dev/null +++ b/src/__tests__/externalDiscoveryPublicActors.test.ts @@ -0,0 +1,264 @@ +import { describe, expect, it } from "vitest"; +import { buildPublicActorsForEvidence } from "../externalDiscovery/publicActors.ts"; +import type { ExternalSignalEvent } from "../externalDiscovery/types.ts"; + +function event(overrides: Partial): ExternalSignalEvent { + return { + event_id: "evt-1", + platform: "x_twitter", + raw_event_kind: "discussion", + derived_signal_kinds: ["evidence"], + scope: "project", + target_type: "project", + target_key: "openai/agents-sdk", + actor: { + actor_type: "unknown", + effective_tier: "unknown", + tier_basis: "none", + }, + observed_at: "2026-07-05T00:00:00.000Z", + raw_ref: "provider:event:1", + ...overrides, + }; +} + +describe("external discovery public actors", () => { + it("extracts X handles as discussion actors and audits available identity", () => { + const result = buildPublicActorsForEvidence([ + event({ + url: "https://x.com/openai/status/123", + actor: { + actor_type: "institution", + effective_tier: "ordinary", + tier_basis: "provider_hint", + handle: "OpenAI", + provider_tier_hint: "core", + }, + }), + ]); + + expect(result.public_actors).toEqual([ + expect.objectContaining({ + public_actor_id: "x:openai", + display_name: "@OpenAI", + actor_role: "discussion_actor", + source_kind: "x_handle", + source_basis: "explicit_actor_field", + tier_basis: "provider_hint", + is_head_actor: false, + }), + ]); + expect(result.public_actor_audit).toEqual([ + { platform: "x_twitter", status: "available", reason: "actor_public_identity_available", event_count: 1 }, + ]); + }); + + it("separates Reddit communities from Reddit users", () => { + const result = buildPublicActorsForEvidence([ + event({ + event_id: "evt-community", + platform: "reddit", + url: "https://www.reddit.com/r/LocalLLaMA/comments/abc/project_discussion/", + actor: { + actor_type: "community", + effective_tier: "ordinary", + tier_basis: "none", + subreddit: "LocalLLaMA", + }, + }), + event({ + event_id: "evt-user", + platform: "reddit", + url: "https://www.reddit.com/user/agentbuilder/comments/abc", + actor: { + actor_type: "person", + effective_tier: "ordinary", + tier_basis: "none", + username: "agentbuilder", + }, + observed_at: "2026-07-05T01:00:00.000Z", + }), + ]); + + expect(result.public_actors.map((actor) => ({ + id: actor.public_actor_id, + role: actor.actor_role, + kind: actor.source_kind, + }))).toEqual([ + { id: "reddit:r:localllama", role: "community_source", kind: "reddit_community" }, + { id: "reddit:u:agentbuilder", role: "discussion_actor", kind: "reddit_user" }, + ]); + }); + + it("extracts HN users from explicit fields and user URLs", () => { + const result = buildPublicActorsForEvidence([ + event({ + platform: "hacker_news", + url: "https://news.ycombinator.com/user?id=pg", + actor: { + actor_type: "person", + effective_tier: "ordinary", + tier_basis: "none", + hn_user: "pg", + }, + }), + ]); + + expect(result.public_actors[0]).toMatchObject({ + public_actor_id: "hn:pg", + display_name: "HN pg", + actor_role: "discussion_actor", + source_kind: "hn_user", + }); + }); + + it("keeps GitHub owners as project sources instead of discussion actors", () => { + const result = buildPublicActorsForEvidence([ + event({ + platform: "official_web", + raw_event_kind: "official_release", + url: "https://github.com/anthropics/claude-code/releases/tag/v1", + target_repo_url: "https://github.com/anthropics/claude-code", + actor: { + actor_type: "team", + effective_tier: "ordinary", + tier_basis: "none", + }, + }), + ]); + + expect(result.public_actors[0]).toMatchObject({ + public_actor_id: "github:anthropics", + display_name: "GitHub anthropics", + actor_role: "project_owner", + source_kind: "github_owner", + is_head_actor: false, + }); + }); + + it("keeps official domains out of discussion actors", () => { + const result = buildPublicActorsForEvidence([ + event({ + platform: "official_blog", + raw_event_kind: "blog_post", + source_url: "https://blog.langchain.com/agent-runtime-update/", + actor: { + actor_type: "team", + effective_tier: "ordinary", + tier_basis: "none", + }, + }), + ]); + + expect(result.public_actors[0]).toMatchObject({ + public_actor_id: "domain:blog.langchain.com", + actor_role: "official_publisher", + source_kind: "official_domain", + source_basis: "official_source_url", + is_head_actor: false, + }); + }); + + it("allows registry social actors to be head public discussion sources", () => { + const result = buildPublicActorsForEvidence([ + event({ + actor: { + actor_type: "institution", + effective_tier: "core", + tier_basis: "registry", + registry_entity_id: "entity-openai", + registry_display_name: "OpenAI", + registry_tier: "core", + source_roles: ["social_discussant"], + }, + }), + ]); + + expect(result.public_actors[0]).toMatchObject({ + public_actor_id: "registry:entity-openai", + display_name: "OpenAI", + actor_role: "registry_entity", + source_kind: "registry_entity", + source_basis: "registry_match", + tier_basis: "registry_match", + is_head_actor: true, + }); + }); + + it("does not turn official-only registry matches into discussion actors", () => { + const result = buildPublicActorsForEvidence([ + event({ + platform: "official_web", + raw_event_kind: "official_release", + actor: { + actor_type: "institution", + effective_tier: "core", + tier_basis: "registry", + registry_entity_id: "entity-openai", + registry_display_name: "OpenAI", + registry_tier: "core", + source_roles: ["official_owner"], + }, + }), + ]); + + expect(result.public_actors.some((actor) => actor.actor_role === "registry_entity")).toBe(false); + expect(result.public_actor_audit).toEqual([ + { platform: "official_web", status: "missing", reason: "official_source_url_missing", event_count: 1 }, + ]); + }); + + it("audits X reserved paths without inventing handles", () => { + const result = buildPublicActorsForEvidence([ + event({ + url: "https://x.com/i/status/123", + actor: { + actor_type: "unknown", + effective_tier: "unknown", + tier_basis: "none", + }, + }), + ]); + + expect(result.public_actors).toEqual([]); + expect(result.public_actor_audit).toEqual([ + { platform: "x_twitter", status: "invalid_reserved_path", reason: "x_reserved_or_indirect_url", event_count: 1 }, + ]); + }); + + it("does not publish opaque provider actor ids as social handles", () => { + const result = buildPublicActorsForEvidence([ + event({ + actor: { + actor_type: "person", + effective_tier: "ordinary", + tier_basis: "none", + provider_actor_id: "45f8279ab8e97386", + }, + }), + ]); + + expect(result.public_actors).toEqual([]); + expect(result.public_actor_audit).toEqual([ + { platform: "x_twitter", status: "missing", reason: "actor_public_identity_missing", event_count: 1 }, + ]); + }); + + it("does not extract discussion actors from target URLs", () => { + const result = buildPublicActorsForEvidence([ + event({ + target_url: "https://x.com/openai", + actor: { + actor_type: "unknown", + effective_tier: "unknown", + tier_basis: "none", + }, + }), + ]); + + expect(result.public_actors).toEqual([]); + expect(result.public_actor_audit).toEqual([ + { platform: "x_twitter", status: "missing", reason: "actor_public_identity_missing", event_count: 1 }, + ]); + }); +}); diff --git a/src/__tests__/externalDiscoveryPublicActorsTypeContract.test.ts b/src/__tests__/externalDiscoveryPublicActorsTypeContract.test.ts new file mode 100644 index 0000000..91180f9 --- /dev/null +++ b/src/__tests__/externalDiscoveryPublicActorsTypeContract.test.ts @@ -0,0 +1,89 @@ +import { describe, expect, it } from "vitest"; +import { + EXTERNAL_PUBLIC_ACTOR_IDENTITY_REASONS, + EXTERNAL_PUBLIC_ACTOR_IDENTITY_STATUSES, + EXTERNAL_PUBLIC_ACTOR_ROLES, + EXTERNAL_PUBLIC_ACTOR_SOURCE_BASES, + EXTERNAL_PUBLIC_ACTOR_SOURCE_KINDS, + EXTERNAL_PUBLIC_ACTOR_TIER_BASES, + type ExternalEvidence, + type ExternalSignalEvent, +} from "../externalDiscovery/types.ts"; + +describe("external discovery public actors type contract", () => { + it("keeps public actor enums closed over the reviewed contract", () => { + expect(EXTERNAL_PUBLIC_ACTOR_ROLES).toEqual([ + "discussion_actor", + "community_source", + "official_publisher", + "project_owner", + "registry_entity", + ]); + expect(EXTERNAL_PUBLIC_ACTOR_SOURCE_KINDS).toEqual([ + "registry_entity", + "x_handle", + "reddit_community", + "reddit_user", + "hn_user", + "github_owner", + "official_domain", + "provider_actor", + ]); + expect(EXTERNAL_PUBLIC_ACTOR_SOURCE_BASES).toContain("registry_match"); + expect(EXTERNAL_PUBLIC_ACTOR_SOURCE_BASES).toContain("explicit_actor_field"); + expect(EXTERNAL_PUBLIC_ACTOR_TIER_BASES).toEqual(["registry_match", "provider_hint", "none"]); + expect(EXTERNAL_PUBLIC_ACTOR_IDENTITY_STATUSES).toEqual(["available", "missing", "invalid_reserved_path", "redacted"]); + expect(EXTERNAL_PUBLIC_ACTOR_IDENTITY_REASONS).toContain("actor_public_identity_available"); + expect(EXTERNAL_PUBLIC_ACTOR_IDENTITY_REASONS).toContain("x_reserved_or_indirect_url"); + }); + + it("places actor public identity status on canonical ExternalSignalEvent", () => { + const event: ExternalSignalEvent = { + event_id: "evt-1", + platform: "x_twitter", + raw_event_kind: "discussion", + derived_signal_kinds: ["evidence"], + scope: "project", + target_type: "project", + target_key: "openai/agents-sdk", + actor: { + actor_type: "institution", + effective_tier: "core", + tier_basis: "registry", + registry_entity_id: "entity-openai", + registry_display_name: "OpenAI", + registry_tier: "core", + source_roles: ["social_discussant"], + }, + observed_at: "2026-07-05T00:00:00.000Z", + raw_ref: "provider:event:1", + actor_public_identity_status: "available", + actor_public_identity_reason: "actor_public_identity_available", + }; + + expect(event.actor_public_identity_status).toBe("available"); + expect(event.actor_public_identity_reason).toBe("actor_public_identity_available"); + }); + + it("keeps public actors optional for old ExternalEvidence artifacts", () => { + const evidence: ExternalEvidence = { + evidence_id: "project:openai/agents-sdk", + event_ids: ["evt-1"], + scope: "project", + target_key: "openai/agents-sdk", + derived_signal_kinds: ["evidence"], + platforms: ["x_twitter"], + named_registry_actors: [], + actor_tiers: { unknown: 1 }, + actor_types: { unknown: 1 }, + mention_count: 1, + distinct_actor_count: 1, + top_tier_actor_count: 0, + first_seen_at: "2026-07-05T00:00:00.000Z", + last_seen_at: "2026-07-05T00:00:00.000Z", + }; + + expect(evidence.public_actors).toBeUndefined(); + expect(evidence.public_actor_audit).toBeUndefined(); + }); +}); diff --git a/src/__tests__/externalDiscoveryRedaction.test.ts b/src/__tests__/externalDiscoveryRedaction.test.ts index 2f7cbef..326673d 100644 --- a/src/__tests__/externalDiscoveryRedaction.test.ts +++ b/src/__tests__/externalDiscoveryRedaction.test.ts @@ -84,4 +84,90 @@ describe("external discovery redaction", () => { expect(result.reason_codes).toContain("public_safe_not_true"); expect(result.reason_codes).toContain("contains_raw_text_not_false"); }); + + it("validates public actor safety and head actor semantics", () => { + const result = assertPublicSafeAggregate( + safeAggregate({ + project_evidence: [ + { + public_actors: [ + { + public_actor_id: "x:openai", + display_name: "@openai", + actor_type: "institution", + actor_role: "discussion_actor", + authority_tier: "core", + tier_basis: "provider_hint", + is_head_actor: true, + source_kind: "x_handle", + source_basis: "explicit_actor_field", + event_count: 1, + platforms: ["x_twitter"], + first_seen_at: "2026-07-05T00:00:00.000Z", + last_seen_at: "2026-07-05T00:00:00.000Z", + }, + ], + }, + ], + direction_evidence: [], + }), + ); + + expect(result.ok).toBe(false); + expect(result.reason_codes).toContain("public_actor_head_without_registry_match"); + }); + + it("rejects public actor ids that are URLs", () => { + const result = assertPublicSafeAggregate( + safeAggregate({ + project_evidence: [ + { + public_actors: [ + { + public_actor_id: "https://x.com/openai", + display_name: "@openai", + actor_type: "institution", + actor_role: "discussion_actor", + tier_basis: "none", + is_head_actor: false, + source_kind: "x_handle", + source_basis: "source_url_path", + event_count: 1, + platforms: ["x_twitter"], + first_seen_at: "2026-07-05T00:00:00.000Z", + last_seen_at: "2026-07-05T00:00:00.000Z", + }, + ], + }, + ], + direction_evidence: [], + }), + ); + + expect(result.ok).toBe(false); + expect(result.reason_codes).toContain("public_actor_id_url"); + }); + + it("rejects public actor audit status and reason mismatches", () => { + const result = assertPublicSafeAggregate( + safeAggregate({ + project_evidence: [ + { + public_actor_audit: [ + { + platform: "x_twitter", + status: "missing", + reason: "actor_public_identity_available", + event_count: 1, + }, + ], + }, + ], + direction_evidence: [], + }), + ); + + expect(result.ok).toBe(false); + expect(result.reason_codes).toContain("public_actor_audit_non_available_reason_mismatch"); + }); }); diff --git a/src/__tests__/externalDiscoveryVerification.test.ts b/src/__tests__/externalDiscoveryVerification.test.ts index d6f3021..5752912 100644 --- a/src/__tests__/externalDiscoveryVerification.test.ts +++ b/src/__tests__/externalDiscoveryVerification.test.ts @@ -282,4 +282,55 @@ describe("external discovery daily verification contract", () => { expect(check?.status).toBe("fail"); expect(check?.detail).toContain("aggregate_source_input_hash does not match"); }); + + it("fails persisted aggregates with invalid public actor audit reason", () => { + const root = setupWorkspace(); + const aggregate = makeExternalAggregate(); + const firstEvidence = (aggregate.project_evidence as Record[])[0]!; + firstEvidence.public_actor_audit = [ + { + platform: "x_twitter", + status: "missing", + reason: "actor_public_identity_available", + event_count: 1, + }, + ]; + writeDailyInputs(root, aggregate, makeCandidateExplanations()); + + const result = buildVerifyDailyResult(date); + const check = result.checks.find((item) => item.name === "external_discovery_aggregate_contract"); + + expect(check?.status).toBe("fail"); + expect(check?.detail).toContain("non-available status must not use actor_public_identity_available"); + }); + + it("fails persisted aggregates that mark project owners as head discussion actors", () => { + const root = setupWorkspace(); + const aggregate = makeExternalAggregate(); + const firstEvidence = (aggregate.project_evidence as Record[])[0]!; + firstEvidence.public_actors = [ + { + public_actor_id: "github:anthropics", + display_name: "GitHub anthropics", + actor_type: "team", + actor_role: "project_owner", + authority_tier: "core", + tier_basis: "provider_hint", + is_head_actor: true, + source_kind: "github_owner", + source_basis: "target_official_url", + event_count: 1, + platforms: ["official_web"], + first_seen_at: "2026-06-30T00:00:00.000Z", + last_seen_at: "2026-06-30T00:00:00.000Z", + }, + ]; + writeDailyInputs(root, aggregate, makeCandidateExplanations()); + + const result = buildVerifyDailyResult(date); + const check = result.checks.find((item) => item.name === "external_discovery_aggregate_contract"); + + expect(check?.status).toBe("fail"); + expect(check?.detail).toContain("official/project sources cannot be head discussion actors"); + }); }); diff --git a/src/action/dailyVerification.ts b/src/action/dailyVerification.ts index 825a813..c62c216 100644 --- a/src/action/dailyVerification.ts +++ b/src/action/dailyVerification.ts @@ -701,6 +701,7 @@ function inspectExternalAggregateContract(value: unknown): { issues.push(`${sectionName}[${evidenceIndex}].named_registry_actors[${actorIndex}].source_roles has invalid roles`); } }); + issues.push(...inspectPublicActorContract(evidence, `${sectionName}[${evidenceIndex}]`)); }); } @@ -722,6 +723,135 @@ function isNamedActorSourceRole(value: unknown): boolean { return value === "social_discussant" || value === "official_publisher" || value === "official_owner"; } +function inspectPublicActorContract(evidence: Record, prefix: string): string[] { + return [ + ...inspectPublicActors(evidence.public_actors, `${prefix}.public_actors`), + ...inspectPublicActorAudit(evidence.public_actor_audit, `${prefix}.public_actor_audit`), + ]; +} + +function inspectPublicActors(value: unknown, prefix: string): string[] { + if (value === undefined) return []; + if (!Array.isArray(value)) return [`${prefix} must be an array`]; + const issues: string[] = []; + value.forEach((actor, index) => { + const actorPrefix = `${prefix}[${index}]`; + if (!isRecord(actor)) { + issues.push(`${actorPrefix} must be an object`); + return; + } + if (typeof actor.public_actor_id !== "string" || actor.public_actor_id.length === 0) { + issues.push(`${actorPrefix}.public_actor_id missing`); + } else if (/^https?:\/\//i.test(actor.public_actor_id) || /[?#\s\u0000-\u001F\u007F]/.test(actor.public_actor_id)) { + issues.push(`${actorPrefix}.public_actor_id must be a safe non-URL id`); + } + if (typeof actor.display_name !== "string" || actor.display_name.length === 0) { + issues.push(`${actorPrefix}.display_name missing`); + } else if ( + actor.display_name.length > 80 || + /https?:\/\//i.test(actor.display_name) || + /[\u0000-\u001F\u007F]/.test(actor.display_name) || + /\b(cookie|session|oauth|bearer|token|api[_ -]?key|password)\b/i.test(actor.display_name) + ) { + issues.push(`${actorPrefix}.display_name must be public-safe`); + } + if (!isExternalActorType(actor.actor_type)) issues.push(`${actorPrefix}.actor_type invalid`); + if (!isPublicActorRole(actor.actor_role)) issues.push(`${actorPrefix}.actor_role invalid`); + if (!isPublicActorSourceKind(actor.source_kind)) issues.push(`${actorPrefix}.source_kind invalid`); + if (!isPublicActorSourceBasis(actor.source_basis)) issues.push(`${actorPrefix}.source_basis invalid`); + if (!isPublicActorTierBasis(actor.tier_basis)) issues.push(`${actorPrefix}.tier_basis invalid`); + if (actor.authority_tier !== undefined && !isPublicActorAuthorityTier(actor.authority_tier)) { + issues.push(`${actorPrefix}.authority_tier invalid`); + } + if (typeof actor.is_head_actor !== "boolean") { + issues.push(`${actorPrefix}.is_head_actor must be boolean`); + } + if (actor.is_head_actor === true && actor.tier_basis !== "registry_match") { + issues.push(`${actorPrefix}.is_head_actor requires registry_match tier_basis`); + } + if (actor.is_head_actor === true && actor.actor_role !== "registry_entity") { + issues.push(`${actorPrefix}.is_head_actor requires registry_entity role`); + } + if ((actor.actor_role === "official_publisher" || actor.actor_role === "project_owner") && actor.is_head_actor === true) { + issues.push(`${actorPrefix}.official/project sources cannot be head discussion actors`); + } + if (typeof actor.event_count !== "number" || actor.event_count <= 0) issues.push(`${actorPrefix}.event_count invalid`); + if (!Array.isArray(actor.platforms) || actor.platforms.some((platform) => !isExternalPlatform(platform))) { + issues.push(`${actorPrefix}.platforms invalid`); + } + }); + return issues; +} + +function inspectPublicActorAudit(value: unknown, prefix: string): string[] { + if (value === undefined) return []; + if (!Array.isArray(value)) return [`${prefix} must be an array`]; + const issues: string[] = []; + value.forEach((audit, index) => { + const auditPrefix = `${prefix}[${index}]`; + if (!isRecord(audit)) { + issues.push(`${auditPrefix} must be an object`); + return; + } + const extraKeys = Object.keys(audit).filter((key) => !["platform", "status", "reason", "event_count"].includes(key)); + if (extraKeys.length > 0) issues.push(`${auditPrefix} must not contain raw or extra fields`); + if (!isExternalPlatform(audit.platform)) issues.push(`${auditPrefix}.platform invalid`); + if (!isIdentityStatus(audit.status)) issues.push(`${auditPrefix}.status invalid`); + if (!isIdentityReason(audit.reason)) issues.push(`${auditPrefix}.reason invalid`); + if (audit.status === "available" && audit.reason !== "actor_public_identity_available") { + issues.push(`${auditPrefix}.available status must use actor_public_identity_available`); + } + if (audit.status !== "available" && audit.reason === "actor_public_identity_available") { + issues.push(`${auditPrefix}.non-available status must not use actor_public_identity_available`); + } + if (typeof audit.event_count !== "number" || audit.event_count <= 0) issues.push(`${auditPrefix}.event_count invalid`); + }); + return issues; +} + +function isExternalPlatform(value: unknown): boolean { + return value === "x_twitter" || value === "reddit" || value === "hacker_news" || value === "official_web" || value === "official_blog"; +} + +function isExternalActorType(value: unknown): boolean { + return value === "institution" || value === "team" || value === "person" || value === "community" || value === "unknown"; +} + +function isPublicActorRole(value: unknown): boolean { + return value === "discussion_actor" || value === "community_source" || value === "official_publisher" || value === "project_owner" || value === "registry_entity"; +} + +function isPublicActorSourceKind(value: unknown): boolean { + return value === "registry_entity" || value === "x_handle" || value === "reddit_community" || value === "reddit_user" || value === "hn_user" || value === "github_owner" || value === "official_domain" || value === "provider_actor"; +} + +function isPublicActorSourceBasis(value: unknown): boolean { + return value === "registry_match" || value === "explicit_actor_field" || value === "source_url_path" || value === "official_source_url" || value === "target_official_url"; +} + +function isPublicActorTierBasis(value: unknown): boolean { + return value === "registry_match" || value === "provider_hint" || value === "none"; +} + +function isPublicActorAuthorityTier(value: unknown): boolean { + return value === "core" || value === "proven" || value === "watch" || value === "ordinary" || value === "unknown"; +} + +function isIdentityStatus(value: unknown): boolean { + return value === "available" || value === "missing" || value === "invalid_reserved_path" || value === "redacted"; +} + +function isIdentityReason(value: unknown): boolean { + return ( + value === "actor_public_identity_available" || + value === "actor_public_identity_missing" || + value === "x_reserved_or_indirect_url" || + value === "official_source_url_missing" || + value === "registry_entity_not_matched" || + value === "redacted_for_public_safety" + ); +} + function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } diff --git a/src/externalDiscovery/agentReachProvider.ts b/src/externalDiscovery/agentReachProvider.ts index 5207dda..2beaf56 100644 --- a/src/externalDiscovery/agentReachProvider.ts +++ b/src/externalDiscovery/agentReachProvider.ts @@ -2,6 +2,7 @@ import fs from "node:fs"; import { applyEntityRegistry, readEntityRegistryWithWarnings, type ExternalEntityRegistryEntry } from "./entityRegistry.ts"; import { externalEntityRegistryPath } from "./paths.ts"; +import { publicActorIdentityForEvent } from "./publicActors.ts"; import { stableSourceInputHash } from "./redaction.ts"; import type { AgentReachProviderReadResult, @@ -142,10 +143,18 @@ function enrichEventActor( target_url: event.target_url, target_repo_url: event.target_repo_url, }); + const enrichedEvent = { + ...event, + actor: lookup.actor, + actor_public_identity_status: undefined, + actor_public_identity_reason: undefined, + }; + const identity = publicActorIdentityForEvent(enrichedEvent); return { event: { - ...event, - actor: lookup.actor, + ...enrichedEvent, + actor_public_identity_status: identity.status, + actor_public_identity_reason: identity.reason, }, warnings: lookup.warnings, }; @@ -167,7 +176,10 @@ function parseEvent(value: unknown): } const rawRef = typeof value.raw_ref === "string" ? value.raw_ref : undefined; - const url = typeof value.url === "string" ? value.url : undefined; + const sourceUrl = firstString([value.source_url]); + const permalink = firstString([value.permalink]); + const discussionUrl = firstString([value.discussion_url]); + const url = firstString([value.url, sourceUrl, permalink, discussionUrl]); const observedAt = typeof value.observed_at === "string" ? value.observed_at : undefined; const eventId = typeof value.event_id === "string" ? value.event_id : generatedEventId({ rawRef, url, observedAt }); const actor = isRecord(value.actor) ? value.actor : undefined; @@ -201,37 +213,54 @@ function parseEvent(value: unknown): }; } + const canonicalEvent: ExternalSignalEvent = { + event_id: eventId, + platform: value.platform, + raw_event_kind: value.raw_event_kind, + derived_signal_kinds: value.derived_signal_kinds, + scope, + target_type: targetType, + target_key: targetKey, + actor: { + actor_type: isActorType(actor.actor_type) ? actor.actor_type : "unknown", + effective_tier: isActorTier(actor.effective_tier) ? actor.effective_tier : "unknown", + tier_basis: actor.tier_basis === "registry" || actor.tier_basis === "provider_hint" ? actor.tier_basis : isProviderTierHint(actor.provider_tier_hint) || isProviderTierHint(actor.tier_hint) ? "provider_hint" : "none", + provider_actor_id: providerActorId(actor), + identity_hash: identityHash(actor), + display_name: firstString([actor.display_name]), + handle: firstString([actor.handle, actor.author, actor.username, actor.user, actor.hn_user]), + author: firstString([actor.author, value.author]), + username: firstString([actor.username, value.username]), + user: firstString([actor.user, value.user]), + subreddit: firstString([actor.subreddit, value.subreddit]), + community: firstString([actor.community, value.community]), + hn_user: firstString([actor.hn_user, value.hn_user]), + platform_profile_url: firstString([actor.platform_profile_url, actor.profile_url]), + provider_tier_hint: isProviderTierHint(actor.provider_tier_hint) ? actor.provider_tier_hint : isProviderTierHint(actor.tier_hint) ? actor.tier_hint : undefined, + registry_entity_id: typeof actor.registry_entity_id === "string" ? actor.registry_entity_id : undefined, + registry_display_name: typeof actor.registry_display_name === "string" ? actor.registry_display_name : undefined, + registry_tier: isRegistryTier(actor.registry_tier) ? actor.registry_tier : undefined, + }, + observed_at: observedAt, + source_published_at: typeof value.source_published_at === "string" ? value.source_published_at : undefined, + ingested_at: typeof value.ingested_at === "string" ? value.ingested_at : undefined, + url, + source_url: sourceUrl, + permalink, + discussion_url: discussionUrl, + target_url: typeof target?.url === "string" ? target.url : undefined, + target_repo_url: typeof target?.repo_url === "string" ? target.repo_url : undefined, + raw_ref: rawRef, + }; + + const identity = publicActorIdentityForEvent(canonicalEvent); + return { ok: true, value: { - event_id: eventId, - platform: value.platform, - raw_event_kind: value.raw_event_kind, - derived_signal_kinds: value.derived_signal_kinds, - scope, - target_type: targetType, - target_key: targetKey, - actor: { - actor_type: isActorType(actor.actor_type) ? actor.actor_type : "unknown", - effective_tier: isActorTier(actor.effective_tier) ? actor.effective_tier : "unknown", - tier_basis: actor.tier_basis === "registry" || actor.tier_basis === "provider_hint" ? actor.tier_basis : isProviderTierHint(actor.provider_tier_hint) || isProviderTierHint(actor.tier_hint) ? "provider_hint" : "none", - provider_actor_id: providerActorId(actor), - identity_hash: identityHash(actor), - display_name: typeof actor.display_name === "string" ? actor.display_name : undefined, - handle: typeof actor.handle === "string" ? actor.handle : undefined, - platform_profile_url: typeof actor.platform_profile_url === "string" ? actor.platform_profile_url : typeof actor.profile_url === "string" ? actor.profile_url : undefined, - provider_tier_hint: isProviderTierHint(actor.provider_tier_hint) ? actor.provider_tier_hint : isProviderTierHint(actor.tier_hint) ? actor.tier_hint : undefined, - registry_entity_id: typeof actor.registry_entity_id === "string" ? actor.registry_entity_id : undefined, - registry_display_name: typeof actor.registry_display_name === "string" ? actor.registry_display_name : undefined, - registry_tier: isRegistryTier(actor.registry_tier) ? actor.registry_tier : undefined, - }, - observed_at: observedAt, - source_published_at: typeof value.source_published_at === "string" ? value.source_published_at : undefined, - ingested_at: typeof value.ingested_at === "string" ? value.ingested_at : undefined, - url, - target_url: typeof target?.url === "string" ? target.url : undefined, - target_repo_url: typeof target?.repo_url === "string" ? target.repo_url : undefined, - raw_ref: rawRef, + ...canonicalEvent, + actor_public_identity_status: identity.status, + actor_public_identity_reason: identity.reason, }, }; } @@ -351,6 +380,10 @@ function emptyResult(status: ExternalProviderStatus, statusReason: string, sourc }; } +function firstString(values: unknown[]): string | undefined { + return values.find((candidate): candidate is string => typeof candidate === "string" && candidate.trim().length > 0)?.trim(); +} + function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } diff --git a/src/externalDiscovery/aggregate.ts b/src/externalDiscovery/aggregate.ts index 1707cb0..1ffd583 100644 --- a/src/externalDiscovery/aggregate.ts +++ b/src/externalDiscovery/aggregate.ts @@ -1,4 +1,5 @@ import { REDACTION_POLICY_VERSION, assertPublicSafeAggregate, stableSourceInputHash } from "./redaction.ts"; +import { buildPublicActorsForEvidence } from "./publicActors.ts"; import type { AgentReachProviderReadResult, DailyExternalAggregate, @@ -146,6 +147,7 @@ function evidenceFromEvents(events: ExternalSignalEvent[]): ExternalEvidence { const firstEvent = events[0]; if (!firstEvent) throw new Error("cannot build external evidence from empty events"); const observedTimes = events.map((event) => event.observed_at).sort(); + const publicActors = buildPublicActorsForEvidence(events); return { evidence_id: `${firstEvent.scope}:${firstEvent.target_key}`, @@ -155,6 +157,8 @@ function evidenceFromEvents(events: ExternalSignalEvent[]): ExternalEvidence { derived_signal_kinds: unique(events.flatMap((event) => event.derived_signal_kinds)).sort() as ExternalSignalKind[], platforms: unique(events.map((event) => event.platform)).sort() as ExternalPlatform[], named_registry_actors: buildNamedRegistryActors(events), + public_actors: publicActors.public_actors, + public_actor_audit: publicActors.public_actor_audit, actor_tiers: countBy(events.map((event) => event.actor.effective_tier)), actor_types: countBy(events.map((event) => event.actor.actor_type)), mention_count: events.length, diff --git a/src/externalDiscovery/explanations.ts b/src/externalDiscovery/explanations.ts index 7dfcda9..81cfb45 100644 --- a/src/externalDiscovery/explanations.ts +++ b/src/externalDiscovery/explanations.ts @@ -16,6 +16,7 @@ export type ExternalCandidateExplanationStatus = "ok" | "partial" | "skipped" | export type ExternalCandidateExplanationScope = "bound_object" | "direction_signal" | "external_evidence_boost"; export type ExternalCandidateExplanationConfidence = "high" | "medium" | "low"; export type ExternalCandidateExplanationSource = "llm" | "rules_fallback" | "existing_project_brief"; +type CandidateExplanationSelectionBucket = "project_evidence" | "new_discovery" | "direction_signal" | "official_signal"; export interface CandidateExplanationInput { candidate_key: string; @@ -117,8 +118,21 @@ type LlmExplanationDraft = { }; const POLICY_VERSION = "candidate-explanations.v1"; -const DEFAULT_TOP_N = 30; +const DEFAULT_TOP_N = 50; const ENHANCED_COVERAGE_THRESHOLD = 0.7; +const QUOTA_BASE_TOTAL = 50; +const DEFAULT_SELECTION_QUOTAS: Record = { + project_evidence: 16, + new_discovery: 16, + direction_signal: 12, + official_signal: 6, +}; +const SELECTION_BUCKET_ORDER: CandidateExplanationSelectionBucket[] = ["project_evidence", "new_discovery", "direction_signal", "official_signal"]; +const OFFICIAL_PLATFORMS = new Set(["official_web", "official_blog"]); +const textUrlPattern = /https?:\/\/\S+/gi; +const markdownLinkPattern = /\[([^\]]+)\]\([^)]+\)/g; +const rawHandlePattern = /(^|\s)@[\w.-]{2,}/g; +const sensitiveTermPattern = /\b(cookie|session|oauth|bearer|token|api[_ -]?key|password)\b/gi; const platformNames: Record = { x_twitter: "X", @@ -142,17 +156,17 @@ export function buildCandidateExplanationInputs(args: BuildInputsArgs): Candidat const evidenceIds = unique(evidenceRows.map((evidence) => evidence.evidence_id)).sort(); const platforms = unique(evidenceRows.flatMap((evidence) => evidence.platforms)).sort() as ExternalPlatform[]; const scored = findScoredProject(scoredProjects, candidate.target_key); - const projectBrief = findProjectBrief(projectBriefs, candidate.target_key, scored?.project.repo_full_name); + const projectBrief = cleanPublicText(findProjectBrief(projectBriefs, candidate.target_key, scored?.project.repo_full_name)); const repoUrl = scored?.project.repo_url ?? repoUrlFromTargetKey(candidate.target_key); - const repoDescription = scored?.project.description?.trim() || undefined; + const repoDescription = cleanPublicText(scored?.project.description); const targetUrl = candidate.target_key.startsWith("http") ? candidate.target_key : repoUrl; - const publicEvidenceTitles = unique(titleRows.map((item) => item.public_evidence_title).filter(isNonEmptyString)).slice(0, 5); - const publicSourceTitles = unique(titleRows.map((item) => item.public_source_title).filter(isNonEmptyString)).slice(0, 5); + const publicEvidenceTitles = unique(titleRows.map((item) => cleanPublicText(item.public_evidence_title)).filter(isNonEmptyString)).slice(0, 5); + const publicSourceTitles = unique(titleRows.map((item) => cleanPublicText(item.public_source_title)).filter(isNonEmptyString)).slice(0, 5); const targetDisplayName = firstNonEmpty([ ...titleRows.map((item) => item.target_display_name), scored?.project.project_name, displayNameFromTargetKey(candidate.target_key), - ]); + ].map((item) => cleanPublicText(item))); const explanationScope = explanationScopeFor(candidate, evidenceRows, Boolean(repoUrl || projectBrief)); const inputWarnings = inputWarningsFor({ projectBrief, @@ -173,7 +187,7 @@ export function buildCandidateExplanationInputs(args: BuildInputsArgs): Candidat candidate_key: `${candidate.candidate_kind}:${candidate.target_key}`, candidate_kind: candidate.candidate_kind, target_key: candidate.target_key, - display_name: targetDisplayName ?? candidate.target_key, + display_name: targetDisplayName ?? safeDisplayNameFromTargetKey(candidate.target_key), explanation_scope: explanationScope, target_url: targetUrl, repo_url: repoUrl, @@ -182,14 +196,14 @@ export function buildCandidateExplanationInputs(args: BuildInputsArgs): Candidat repo_description: repoDescription, public_evidence_titles: publicEvidenceTitles, public_source_titles: publicSourceTitles, - evidence_reason_facts: evidenceReasonFacts(candidate, evidenceRows), + evidence_reason_facts: evidenceReasonFacts(candidate, evidenceRows).map((item) => cleanPublicText(item)).filter(isNonEmptyString), evidence_ids: evidenceIds, platforms, mention_count: sum(evidenceRows.map((evidence) => evidence.mention_count)), distinct_actor_count: sum(evidenceRows.map((evidence) => evidence.distinct_actor_count)), top_tier_actor_count: sum(evidenceRows.map((evidence) => evidence.top_tier_actor_count)), named_registry_actor_names: unique( - evidenceRows.flatMap((evidence) => evidence.named_registry_actors.map((actor) => actor.display_name)), + evidenceRows.flatMap((evidence) => evidence.named_registry_actors.map((actor) => cleanPublicText(actor.display_name)).filter(isNonEmptyString)), ).slice(0, 8), can_enter_daily: candidate.can_enter_daily, can_enter_weekly: candidate.can_enter_weekly, @@ -207,7 +221,7 @@ export function buildCandidateExplanationInputs(args: BuildInputsArgs): Candidat }; }); - const sortedInputs = sortInputs(inputs).slice(0, Math.max(1, args.topN ?? DEFAULT_TOP_N)); + const sortedInputs = selectInputsWithQuotas(inputs, Math.max(1, args.topN ?? DEFAULT_TOP_N)); const redaction = assertPublicSafeCandidateExplanationInputs(sortedInputs); if (!redaction.ok) { warnings.push({ @@ -300,9 +314,9 @@ export async function generateExternalCandidateExplanations(args: GenerateArgs): } const fallback = [...fallbackInputs.values()].map((input) => rulesFallbackExplanation(input, args.generatedAt)); - const explanations = [...existing, ...llmExplanations, ...fallback].sort((left, right) => - left.candidate_key.localeCompare(right.candidate_key), - ); + const deduped = dedupeEnhancedSummaries([...existing, ...llmExplanations, ...fallback], inputs, args.generatedAt); + warnings.push(...deduped.warnings); + const explanations = deduped.explanations.sort((left, right) => left.candidate_key.localeCompare(right.candidate_key)); const enhancedCount = explanations.filter((item) => item.summary_source === "llm" || item.summary_source === "existing_project_brief").length; const status = explanationStatus({ enabled, @@ -509,6 +523,39 @@ function normalizeCaveats(caveats: unknown, input: CandidateExplanationInput): s return unique([...fromDraft, ...caveatsFor(input)]).slice(0, 5); } +function dedupeEnhancedSummaries( + explanations: ExternalCandidateExplanation[], + inputs: CandidateExplanationInput[], + generatedAt: string, +): { + explanations: ExternalCandidateExplanation[]; + warnings: Array<{ reason_code: string; reason_detail: string }>; +} { + const inputByKey = new Map(inputs.map((input) => [input.candidate_key, input] as const)); + const seen = new Set(); + const warnings: Array<{ reason_code: string; reason_detail: string }> = []; + + return { + explanations: explanations.map((explanation) => { + if (explanation.summary_source === "rules_fallback") return explanation; + const summaryKey = normalizeSummaryText(`${explanation.what_it_is_cn}${explanation.why_watch_cn}`); + if (!seen.has(summaryKey)) { + seen.add(summaryKey); + return explanation; + } + + const input = inputByKey.get(explanation.candidate_key); + if (!input) return explanation; + warnings.push({ + reason_code: "duplicate_summary_fallback", + reason_detail: explanation.candidate_key, + }); + return rulesFallbackExplanation(input, generatedAt); + }), + warnings, + }; +} + function buildDraftMap(raw: unknown): Map { if (!raw || typeof raw !== "object") return new Map(); const record = raw as Record; @@ -663,6 +710,53 @@ function evidenceReasonFacts(candidate: ObservationCandidate, evidenceRows: Exte return facts.filter(isNonEmptyString); } +function selectInputsWithQuotas(inputs: CandidateExplanationInput[], limit: number): CandidateExplanationInput[] { + const sorted = sortInputs(inputs); + const quotas = quotasForLimit(limit); + const selected = new Map(); + const selectedCounts = new Map(); + + for (const bucket of SELECTION_BUCKET_ORDER) { + const quota = quotas[bucket]; + if (quota <= 0) continue; + for (const input of sorted) { + if (selected.size >= limit) break; + if (selectionBucketFor(input) !== bucket || selected.has(input.candidate_key)) continue; + selected.set(input.candidate_key, input); + selectedCounts.set(bucket, (selectedCounts.get(bucket) ?? 0) + 1); + if ((selectedCounts.get(bucket) ?? 0) >= quota) break; + } + } + + for (const input of sorted) { + if (selected.size >= limit) break; + if (!selected.has(input.candidate_key)) selected.set(input.candidate_key, input); + } + + return sortInputs([...selected.values()]); +} + +function quotasForLimit(limit: number): Record { + return { + project_evidence: scaledQuota(DEFAULT_SELECTION_QUOTAS.project_evidence, limit), + new_discovery: scaledQuota(DEFAULT_SELECTION_QUOTAS.new_discovery, limit), + direction_signal: scaledQuota(DEFAULT_SELECTION_QUOTAS.direction_signal, limit), + official_signal: scaledQuota(DEFAULT_SELECTION_QUOTAS.official_signal, limit), + }; +} + +function scaledQuota(quota: number, limit: number): number { + if (limit >= QUOTA_BASE_TOTAL) return quota; + return Math.max(1, Math.floor((quota * limit) / QUOTA_BASE_TOTAL)); +} + +function selectionBucketFor(input: CandidateExplanationInput): CandidateExplanationSelectionBucket { + if (input.platforms.some((platform) => OFFICIAL_PLATFORMS.has(platform))) return "official_signal"; + if (input.explanation_scope === "direction_signal") return "direction_signal"; + if (input.explanation_scope === "external_evidence_boost") return "project_evidence"; + return "new_discovery"; +} + function sortInputs(inputs: CandidateExplanationInput[]): CandidateExplanationInput[] { const rank: Record = { external_evidence_boost: 0, @@ -685,6 +779,27 @@ function trimSentence(value: string): string { return normalized.length > 80 ? `${normalized.slice(0, 80)}...` : normalized; } +function cleanPublicText(value: string | undefined): string | undefined { + const cleaned = value + ?.replace(markdownLinkPattern, "$1") + .replace(textUrlPattern, "") + .replace(rawHandlePattern, "$1") + .replace(sensitiveTermPattern, "credential") + .replace(/\s+/g, " ") + .replace(/\s+([,.;:!?])/g, "$1") + .replace(/\b(?:demo|link|url|website)\s*:\s*$/i, "") + .trim(); + return cleaned || undefined; +} + +function safeDisplayNameFromTargetKey(value: string): string { + return cleanPublicText(displayNameFromTargetKey(value)) ?? cleanPublicText(value) ?? "external object"; +} + +function normalizeSummaryText(value: string): string { + return value.replace(/\s+/g, "").replace(/[锛屻€傘€佲€溾€?'锛涳細:,.!?锛侊紵()锛堬級\-\s]/g, "").toLowerCase(); +} + function firstNonEmpty(values: Array): string | undefined { return values.find(isNonEmptyString); } diff --git a/src/externalDiscovery/publicActors.ts b/src/externalDiscovery/publicActors.ts new file mode 100644 index 0000000..1ff4457 --- /dev/null +++ b/src/externalDiscovery/publicActors.ts @@ -0,0 +1,547 @@ +import type { + ExternalActorType, + ExternalPlatform, + ExternalPublicActor, + ExternalPublicActorAudit, + ExternalPublicActorIdentityReason, + ExternalPublicActorIdentityStatus, + ExternalPublicActorRole, + ExternalPublicActorSourceBasis, + ExternalPublicActorSourceKind, + ExternalPublicActorTierBasis, + ExternalSignalEvent, +} from "./types.ts"; + +export interface PublicActorsForEvidence { + public_actors: ExternalPublicActor[]; + public_actor_audit: ExternalPublicActorAudit[]; +} + +interface PublicActorCandidate { + public_actor_id: string; + display_name: string; + actor_type: ExternalActorType; + actor_role: ExternalPublicActorRole; + authority_tier?: ExternalPublicActor["authority_tier"]; + tier_basis: ExternalPublicActorTierBasis; + is_head_actor: boolean; + source_kind: ExternalPublicActorSourceKind; + source_basis: ExternalPublicActorSourceBasis; +} + +interface UrlCandidate { + url: URL; + source_basis: ExternalPublicActorSourceBasis; + source_field: "source" | "target"; +} + +const registryTierRank: Record, number> = { + core: 0, + proven: 1, + watch: 2, + ordinary: 3, + unknown: 4, +}; + +const discussionRoles = new Set([ + "discussion_actor", + "community_source", + "registry_entity", +]); + +const xReservedPathParts = new Set(["i", "intent", "share", "login", "redirect"]); +const githubReservedOwners = new Set(["features", "marketplace", "topics", "trending", "explore", "login"]); + +export function buildPublicActorsForEvidence(events: ExternalSignalEvent[]): PublicActorsForEvidence { + return { + public_actors: mergePublicActors(events.flatMap((event) => extractPublicActorsFromEvent(event))), + public_actor_audit: buildPublicActorAudit(events), + }; +} + +export function extractPublicActorsFromEvent(event: ExternalSignalEvent): ExternalPublicActor[] { + const candidates: PublicActorCandidate[] = [ + ...registryActorCandidates(event), + ...explicitActorCandidates(event), + ...urlActorCandidates(event), + ]; + const seen = new Set(); + const actors: ExternalPublicActor[] = []; + + for (const candidate of candidates) { + const key = publicActorKey(candidate); + if (seen.has(key)) continue; + seen.add(key); + actors.push(toPublicActor(candidate, event)); + } + + return actors; +} + +export function buildPublicActorAudit(events: ExternalSignalEvent[]): ExternalPublicActorAudit[] { + const byKey = new Map(); + for (const event of events) { + const identity = publicActorIdentityForEvent(event); + const key = `${event.platform}:${identity.status}:${identity.reason}`; + const existing = byKey.get(key); + if (existing) { + existing.event_count += 1; + } else { + byKey.set(key, { + platform: event.platform, + status: identity.status, + reason: identity.reason, + event_count: 1, + }); + } + } + + return [...byKey.values()].sort( + (a, b) => + a.platform.localeCompare(b.platform) || + a.status.localeCompare(b.status) || + a.reason.localeCompare(b.reason), + ); +} + +export function publicActorIdentityForEvent(event: ExternalSignalEvent): { + status: ExternalPublicActorIdentityStatus; + reason: ExternalPublicActorIdentityReason; +} { + if (event.actor_public_identity_status && event.actor_public_identity_reason) { + return { + status: event.actor_public_identity_status, + reason: event.actor_public_identity_reason, + }; + } + + if (extractPublicActorsFromEvent(event).length > 0) { + return { status: "available", reason: "actor_public_identity_available" }; + } + + if (hasXReservedOrIndirectUrl(event)) { + return { status: "invalid_reserved_path", reason: "x_reserved_or_indirect_url" }; + } + + if ((event.platform === "official_web" || event.platform === "official_blog") && !hasAnySourceUrl(event)) { + return { status: "missing", reason: "official_source_url_missing" }; + } + + if (hasRegistryHint(event)) { + return { status: "missing", reason: "registry_entity_not_matched" }; + } + + return { status: "missing", reason: "actor_public_identity_missing" }; +} + +function mergePublicActors(actors: ExternalPublicActor[]): ExternalPublicActor[] { + const byKey = new Map(); + + for (const actor of actors) { + const key = publicActorKey(actor); + const existing = byKey.get(key); + if (!existing) { + byKey.set(key, { ...actor, platforms: [...actor.platforms].sort() as ExternalPlatform[] }); + continue; + } + + existing.event_count += actor.event_count; + existing.platforms = unique([...existing.platforms, ...actor.platforms]).sort() as ExternalPlatform[]; + existing.first_seen_at = earliestIso(existing.first_seen_at, actor.first_seen_at); + existing.last_seen_at = latestIso(existing.last_seen_at, actor.last_seen_at); + } + + return [...byKey.values()].sort(publicActorSort); +} + +function registryActorCandidates(event: ExternalSignalEvent): PublicActorCandidate[] { + const actor = event.actor; + const isRegistryActor = + actor.tier_basis === "registry" && + actor.registry_entity_id && + actor.registry_display_name && + actor.registry_tier && + (actor.actor_type === "institution" || actor.actor_type === "team" || actor.actor_type === "person"); + if (!isRegistryActor) return []; + if (!actor.source_roles?.includes("social_discussant")) return []; + const registryEntityId = actor.registry_entity_id; + const registryDisplayName = actor.registry_display_name; + const registryTier = actor.registry_tier; + if (!registryEntityId || !registryDisplayName || !registryTier) return []; + + return [ + { + public_actor_id: `registry:${registryEntityId}`, + display_name: registryDisplayName, + actor_type: actor.actor_type, + actor_role: "registry_entity", + authority_tier: registryTier, + tier_basis: "registry_match", + is_head_actor: true, + source_kind: "registry_entity", + source_basis: "registry_match", + }, + ]; +} + +function explicitActorCandidates(event: ExternalSignalEvent): PublicActorCandidate[] { + const actor = event.actor; + const candidates: PublicActorCandidate[] = []; + + if (event.platform === "x_twitter") { + const handle = firstSafePublicToken([ + actor.handle, + actor.author, + actor.username, + actor.user, + handleLikeProviderActorId(actor.provider_actor_id), + xHandleFromProfileUrl(actor.platform_profile_url), + ]); + if (handle) { + candidates.push(actorCandidateFromToken(event, { + public_actor_id: `x:${handle.toLowerCase()}`, + display_name: `@${handle}`, + actor_type: actor.actor_type === "unknown" ? "person" : actor.actor_type, + actor_role: "discussion_actor", + source_kind: "x_handle", + source_basis: "explicit_actor_field", + })); + } + } + + if (event.platform === "reddit") { + const subreddit = firstSafePublicToken([actor.subreddit, actor.community]); + if (subreddit) { + candidates.push(actorCandidateFromToken(event, { + public_actor_id: `reddit:r:${subreddit.toLowerCase()}`, + display_name: `r/${subreddit}`, + actor_type: "community", + actor_role: "community_source", + source_kind: "reddit_community", + source_basis: "explicit_actor_field", + })); + } + + const user = firstSafePublicToken([ + actor.author, + actor.username, + actor.user, + actor.handle, + handleLikeProviderActorId(actor.provider_actor_id), + ]); + if (user) { + candidates.push(actorCandidateFromToken(event, { + public_actor_id: `reddit:u:${user.toLowerCase()}`, + display_name: `u/${user}`, + actor_type: "person", + actor_role: "discussion_actor", + source_kind: "reddit_user", + source_basis: "explicit_actor_field", + })); + } + } + + if (event.platform === "hacker_news") { + const user = firstSafePublicToken([ + actor.hn_user, + actor.author, + actor.username, + actor.user, + actor.handle, + handleLikeProviderActorId(actor.provider_actor_id), + ]); + if (user) { + candidates.push(actorCandidateFromToken(event, { + public_actor_id: `hn:${user.toLowerCase()}`, + display_name: `HN ${user}`, + actor_type: "person", + actor_role: "discussion_actor", + source_kind: "hn_user", + source_basis: "explicit_actor_field", + })); + } + } + + const providerDisplayName = safePublicDisplayName(actor.display_name); + if (providerDisplayName && candidates.length === 0 && isSocialPlatform(event.platform)) { + candidates.push(actorCandidateFromToken(event, { + public_actor_id: `provider:${event.platform}:${providerDisplayName.toLowerCase()}`, + display_name: providerDisplayName, + actor_type: actor.actor_type === "unknown" ? "person" : actor.actor_type, + actor_role: "discussion_actor", + source_kind: "provider_actor", + source_basis: "explicit_actor_field", + })); + } + + return candidates; +} + +function urlActorCandidates(event: ExternalSignalEvent): PublicActorCandidate[] { + const candidates: PublicActorCandidate[] = []; + const urls = urlCandidates(event); + + for (const candidate of urls) { + const hostname = candidate.url.hostname.toLowerCase().replace(/^www\./, ""); + const parts = candidate.url.pathname.split("/").map((part) => part.trim()).filter(Boolean); + + if (event.platform === "x_twitter" && (hostname === "x.com" || hostname === "twitter.com")) { + if (candidate.source_field !== "source") continue; + const handle = safePublicToken(parts[0]); + if (handle && !isXReservedPart(handle)) { + candidates.push(actorCandidateFromToken(event, { + public_actor_id: `x:${handle.toLowerCase()}`, + display_name: `@${handle}`, + actor_type: event.actor.actor_type === "unknown" ? "person" : event.actor.actor_type, + actor_role: "discussion_actor", + source_kind: "x_handle", + source_basis: "source_url_path", + })); + } + continue; + } + + if (event.platform === "reddit" && (hostname === "reddit.com" || hostname.endsWith(".reddit.com"))) { + if (candidate.source_field !== "source") continue; + const [kind, value] = parts; + if (kind?.toLowerCase() === "r") { + const subreddit = safePublicToken(value); + if (subreddit) { + candidates.push(actorCandidateFromToken(event, { + public_actor_id: `reddit:r:${subreddit.toLowerCase()}`, + display_name: `r/${subreddit}`, + actor_type: "community", + actor_role: "community_source", + source_kind: "reddit_community", + source_basis: "source_url_path", + })); + } + } + if (kind?.toLowerCase() === "user" || kind?.toLowerCase() === "u") { + const user = safePublicToken(value); + if (user) { + candidates.push(actorCandidateFromToken(event, { + public_actor_id: `reddit:u:${user.toLowerCase()}`, + display_name: `u/${user}`, + actor_type: "person", + actor_role: "discussion_actor", + source_kind: "reddit_user", + source_basis: "source_url_path", + })); + } + } + continue; + } + + if (event.platform === "hacker_news" && hostname === "news.ycombinator.com" && parts[0] === "user") { + if (candidate.source_field !== "source") continue; + const user = safePublicToken(candidate.url.searchParams.get("id") ?? undefined); + if (user) { + candidates.push(actorCandidateFromToken(event, { + public_actor_id: `hn:${user.toLowerCase()}`, + display_name: `HN ${user}`, + actor_type: "person", + actor_role: "discussion_actor", + source_kind: "hn_user", + source_basis: "source_url_path", + })); + } + continue; + } + + if (hostname === "github.com") { + const owner = safePublicToken(parts[0]); + if (owner && !githubReservedOwners.has(owner.toLowerCase())) { + candidates.push(actorCandidateFromToken(event, { + public_actor_id: `github:${owner.toLowerCase()}`, + display_name: `GitHub ${owner}`, + actor_type: event.actor.actor_type === "person" ? "person" : "team", + actor_role: "project_owner", + source_kind: "github_owner", + source_basis: candidate.source_basis, + })); + } + continue; + } + + if ((event.platform === "official_web" || event.platform === "official_blog") && candidate.source_field === "source") { + candidates.push(actorCandidateFromToken(event, { + public_actor_id: `domain:${hostname}`, + display_name: hostname, + actor_type: event.actor.actor_type === "institution" || event.actor.actor_type === "team" ? event.actor.actor_type : "team", + actor_role: "official_publisher", + source_kind: "official_domain", + source_basis: "official_source_url", + })); + } + } + + return candidates; +} + +function actorCandidateFromToken( + event: ExternalSignalEvent, + input: Omit, +): PublicActorCandidate { + const tierBasis = event.actor.provider_tier_hint ? "provider_hint" : "none"; + return { + ...input, + authority_tier: event.actor.provider_tier_hint, + tier_basis: tierBasis, + is_head_actor: false, + }; +} + +function toPublicActor(candidate: PublicActorCandidate, event: ExternalSignalEvent): ExternalPublicActor { + return { + public_actor_id: candidate.public_actor_id, + display_name: candidate.display_name, + actor_type: candidate.actor_type, + actor_role: candidate.actor_role, + authority_tier: candidate.authority_tier, + tier_basis: candidate.tier_basis, + is_head_actor: candidate.is_head_actor, + source_kind: candidate.source_kind, + source_basis: candidate.source_basis, + event_count: 1, + platforms: [event.platform], + first_seen_at: event.observed_at, + last_seen_at: event.observed_at, + }; +} + +function urlCandidates(event: ExternalSignalEvent): UrlCandidate[] { + const values: Array<{ value?: string; source_basis: ExternalPublicActorSourceBasis; source_field: UrlCandidate["source_field"] }> = [ + { value: event.url, source_basis: "source_url_path", source_field: "source" }, + { value: event.source_url, source_basis: "source_url_path", source_field: "source" }, + { value: event.permalink, source_basis: "source_url_path", source_field: "source" }, + { value: event.discussion_url, source_basis: "source_url_path", source_field: "source" }, + { value: event.target_url, source_basis: "target_official_url", source_field: "target" }, + { value: event.target_repo_url, source_basis: "target_official_url", source_field: "target" }, + ]; + const seen = new Set(); + const result: UrlCandidate[] = []; + + for (const item of values) { + const url = parseHttpUrl(item.value); + if (!url) continue; + const key = url.toString(); + if (seen.has(key)) continue; + seen.add(key); + result.push({ url, source_basis: item.source_basis, source_field: item.source_field }); + } + + return result; +} + +function hasXReservedOrIndirectUrl(event: ExternalSignalEvent): boolean { + return urlCandidates(event).some((candidate) => { + const hostname = candidate.url.hostname.toLowerCase().replace(/^www\./, ""); + if (hostname !== "x.com" && hostname !== "twitter.com") return false; + const parts = candidate.url.pathname.split("/").map((part) => part.trim()).filter(Boolean); + return parts.some(isXReservedPart); + }); +} + +function hasAnySourceUrl(event: ExternalSignalEvent): boolean { + return Boolean(event.url || event.source_url || event.permalink || event.discussion_url); +} + +function hasRegistryHint(event: ExternalSignalEvent): boolean { + return Boolean( + event.actor.registry_entity_id || + event.actor.registry_display_name || + event.actor.provider_tier_hint || + event.actor.display_name, + ); +} + +function parseHttpUrl(value: string | undefined): URL | null { + if (!value) return null; + try { + const url = new URL(value); + if (url.protocol !== "http:" && url.protocol !== "https:") return null; + return url; + } catch { + return null; + } +} + +function xHandleFromProfileUrl(value: string | undefined): string | undefined { + const url = parseHttpUrl(value); + if (!url) return undefined; + const hostname = url.hostname.toLowerCase().replace(/^www\./, ""); + if (hostname !== "x.com" && hostname !== "twitter.com") return undefined; + const handle = url.pathname.split("/").filter(Boolean)[0]; + return isXReservedPart(handle) ? undefined : handle; +} + +function handleLikeProviderActorId(value: string | undefined): string | undefined { + if (!value) return undefined; + const trimmed = value.trim(); + if (trimmed.startsWith("@")) return trimmed; + if (/^u\//i.test(trimmed)) return trimmed; + return undefined; +} + +function firstSafePublicToken(values: Array): string | undefined { + for (const value of values) { + const token = safePublicToken(value); + if (token) return token; + } + return undefined; +} + +function safePublicToken(value: string | undefined): string | undefined { + const normalized = value?.trim().replace(/^@/, "").replace(/^u\//i, "").replace(/^r\//i, ""); + return normalized && /^[a-zA-Z0-9_.-]{1,80}$/.test(normalized) ? normalized : undefined; +} + +function safePublicDisplayName(value: string | undefined): string | undefined { + const normalized = value?.replace(/\s+/g, " ").trim(); + if (!normalized || normalized.length > 80) return undefined; + if (/https?:\/\//i.test(normalized)) return undefined; + if (/[\u0000-\u001F\u007F]/.test(normalized)) return undefined; + if (/\b(cookie|session|oauth|bearer|token|api[_ -]?key|password)\b/i.test(normalized)) return undefined; + return normalized; +} + +function isXReservedPart(value: string | undefined): boolean { + return Boolean(value && xReservedPathParts.has(value.toLowerCase())); +} + +function isSocialPlatform(platform: ExternalPlatform): boolean { + return platform === "x_twitter" || platform === "reddit" || platform === "hacker_news"; +} + +function publicActorKey(actor: Pick): string { + return `${actor.public_actor_id}:${actor.actor_role}:${actor.source_kind}`; +} + +function publicActorSort(a: ExternalPublicActor, b: ExternalPublicActor): number { + const aRegistry = a.source_kind === "registry_entity" ? 0 : 1; + const bRegistry = b.source_kind === "registry_entity" ? 0 : 1; + return ( + aRegistry - bRegistry || + Number(b.is_head_actor) - Number(a.is_head_actor) || + registryTierRank[a.authority_tier ?? "unknown"] - registryTierRank[b.authority_tier ?? "unknown"] || + b.event_count - a.event_count || + a.display_name.localeCompare(b.display_name) + ); +} + +function earliestIso(a: string, b: string): string { + return a <= b ? a : b; +} + +function latestIso(a: string, b: string): string { + return a >= b ? a : b; +} + +function unique(values: T[]): T[] { + return Array.from(new Set(values)); +} + +export function actorRoleCanEnterDiscussion(actor: Pick): boolean { + return discussionRoles.has(actor.actor_role); +} diff --git a/src/externalDiscovery/redaction.ts b/src/externalDiscovery/redaction.ts index b39517f..3367ed1 100644 --- a/src/externalDiscovery/redaction.ts +++ b/src/externalDiscovery/redaction.ts @@ -49,6 +49,7 @@ export function assertPublicSafeAggregate(value: unknown): RedactionCheckResult if (typeof value.source_input_hash !== "string" || value.source_input_hash.length === 0) { reasonCodes.push("missing_source_input_hash"); } + reasonCodes.push(...inspectPublicActorContract(value)); } return { @@ -93,6 +94,146 @@ function collectRedactionReasonCodes(value: unknown): string[] { return reasonCodes; } +function inspectPublicActorContract(value: Record): string[] { + const reasonCodes: string[] = []; + const evidenceSections = [value.project_evidence, value.direction_evidence]; + for (const section of evidenceSections) { + if (!Array.isArray(section)) continue; + for (const evidence of section) { + if (!isRecord(evidence)) continue; + reasonCodes.push(...inspectPublicActors(evidence.public_actors)); + reasonCodes.push(...inspectPublicActorAudit(evidence.public_actor_audit)); + } + } + return reasonCodes; +} + +function inspectPublicActors(value: unknown): string[] { + if (value === undefined) return []; + if (!Array.isArray(value)) return ["public_actors_not_array"]; + const reasonCodes: string[] = []; + for (const actor of value) { + if (!isRecord(actor)) { + reasonCodes.push("public_actor_not_object"); + continue; + } + + const publicActorId = actor.public_actor_id; + const displayName = actor.display_name; + if (typeof publicActorId !== "string" || publicActorId.length === 0) { + reasonCodes.push("public_actor_id_missing"); + } else { + if (/^https?:\/\//i.test(publicActorId)) reasonCodes.push("public_actor_id_url"); + if (/[?#\s\u0000-\u001F\u007F]/.test(publicActorId)) reasonCodes.push("public_actor_id_unsafe"); + } + + if (typeof displayName !== "string" || displayName.length === 0) { + reasonCodes.push("public_actor_display_name_missing"); + } else { + if (displayName.length > 80) reasonCodes.push("public_actor_display_name_too_long"); + if (/https?:\/\//i.test(displayName)) reasonCodes.push("public_actor_display_name_url"); + if (/[\u0000-\u001F\u007F]/.test(displayName)) reasonCodes.push("public_actor_display_name_control_char"); + if (secretPattern.test(displayName) || tokenSecretPattern.test(displayName)) reasonCodes.push("public_actor_display_name_secret"); + } + + if (!isActorType(actor.actor_type)) reasonCodes.push("public_actor_type_invalid"); + if (!isPublicActorRole(actor.actor_role)) reasonCodes.push("public_actor_role_invalid"); + if (!isPublicActorSourceKind(actor.source_kind)) reasonCodes.push("public_actor_source_kind_invalid"); + if (!isPublicActorSourceBasis(actor.source_basis)) reasonCodes.push("public_actor_source_basis_invalid"); + if (!isPublicActorTierBasis(actor.tier_basis)) reasonCodes.push("public_actor_tier_basis_invalid"); + if (actor.authority_tier !== undefined && !isPublicActorAuthorityTier(actor.authority_tier)) { + reasonCodes.push("public_actor_authority_tier_invalid"); + } + if (typeof actor.is_head_actor !== "boolean") reasonCodes.push("public_actor_is_head_actor_invalid"); + if (actor.is_head_actor === true && actor.tier_basis !== "registry_match") { + reasonCodes.push("public_actor_head_without_registry_match"); + } + if (actor.is_head_actor === true && actor.actor_role !== "registry_entity") { + reasonCodes.push("public_actor_head_role_invalid"); + } + if ((actor.actor_role === "official_publisher" || actor.actor_role === "project_owner") && actor.is_head_actor === true) { + reasonCodes.push("public_actor_official_or_project_head_invalid"); + } + if (typeof actor.event_count !== "number" || actor.event_count <= 0) reasonCodes.push("public_actor_event_count_invalid"); + if (!Array.isArray(actor.platforms) || actor.platforms.some((platform) => !isPlatform(platform))) { + reasonCodes.push("public_actor_platforms_invalid"); + } + if (typeof actor.first_seen_at !== "string" || actor.first_seen_at.length === 0) reasonCodes.push("public_actor_first_seen_missing"); + if (typeof actor.last_seen_at !== "string" || actor.last_seen_at.length === 0) reasonCodes.push("public_actor_last_seen_missing"); + } + return reasonCodes; +} + +function inspectPublicActorAudit(value: unknown): string[] { + if (value === undefined) return []; + if (!Array.isArray(value)) return ["public_actor_audit_not_array"]; + const reasonCodes: string[] = []; + for (const audit of value) { + if (!isRecord(audit)) { + reasonCodes.push("public_actor_audit_not_object"); + continue; + } + const extraKeys = Object.keys(audit).filter((key) => !["platform", "status", "reason", "event_count"].includes(key)); + if (extraKeys.length > 0) reasonCodes.push("public_actor_audit_extra_keys"); + if (!isPlatform(audit.platform)) reasonCodes.push("public_actor_audit_platform_invalid"); + if (!isIdentityStatus(audit.status)) reasonCodes.push("public_actor_audit_status_invalid"); + if (!isIdentityReason(audit.reason)) reasonCodes.push("public_actor_audit_reason_invalid"); + if (audit.status === "available" && audit.reason !== "actor_public_identity_available") { + reasonCodes.push("public_actor_audit_available_reason_mismatch"); + } + if (audit.status !== "available" && audit.reason === "actor_public_identity_available") { + reasonCodes.push("public_actor_audit_non_available_reason_mismatch"); + } + if (typeof audit.event_count !== "number" || audit.event_count <= 0) { + reasonCodes.push("public_actor_audit_event_count_invalid"); + } + } + return reasonCodes; +} + +function isPlatform(value: unknown): boolean { + return value === "x_twitter" || value === "reddit" || value === "hacker_news" || value === "official_web" || value === "official_blog"; +} + +function isActorType(value: unknown): boolean { + return value === "institution" || value === "team" || value === "person" || value === "community" || value === "unknown"; +} + +function isPublicActorRole(value: unknown): boolean { + return value === "discussion_actor" || value === "community_source" || value === "official_publisher" || value === "project_owner" || value === "registry_entity"; +} + +function isPublicActorSourceKind(value: unknown): boolean { + return value === "registry_entity" || value === "x_handle" || value === "reddit_community" || value === "reddit_user" || value === "hn_user" || value === "github_owner" || value === "official_domain" || value === "provider_actor"; +} + +function isPublicActorSourceBasis(value: unknown): boolean { + return value === "registry_match" || value === "explicit_actor_field" || value === "source_url_path" || value === "official_source_url" || value === "target_official_url"; +} + +function isPublicActorTierBasis(value: unknown): boolean { + return value === "registry_match" || value === "provider_hint" || value === "none"; +} + +function isPublicActorAuthorityTier(value: unknown): boolean { + return value === "core" || value === "proven" || value === "watch" || value === "ordinary" || value === "unknown"; +} + +function isIdentityStatus(value: unknown): boolean { + return value === "available" || value === "missing" || value === "invalid_reserved_path" || value === "redacted"; +} + +function isIdentityReason(value: unknown): boolean { + return ( + value === "actor_public_identity_available" || + value === "actor_public_identity_missing" || + value === "x_reserved_or_indirect_url" || + value === "official_source_url_missing" || + value === "registry_entity_not_matched" || + value === "redacted_for_public_safety" + ); +} + function visit(value: unknown, visitor: (value: unknown, key?: string) => void, key?: string): void { visitor(value, key); if (Array.isArray(value)) { diff --git a/src/externalDiscovery/types.ts b/src/externalDiscovery/types.ts index 614727d..f9b4886 100644 --- a/src/externalDiscovery/types.ts +++ b/src/externalDiscovery/types.ts @@ -10,6 +10,28 @@ export type ExternalProviderTierHint = "core" | "proven" | "watch" | "ordinary" export type ExternalEvidenceScope = "project" | "direction"; export type ExternalTierBasis = "registry" | "provider_hint" | "none"; export type ExternalNamedActorSourceRole = "social_discussant" | "official_publisher" | "official_owner"; +export type ExternalPublicActorRole = + | "discussion_actor" + | "community_source" + | "official_publisher" + | "project_owner" + | "registry_entity"; +export type ExternalPublicActorSourceBasis = + | "registry_match" + | "explicit_actor_field" + | "source_url_path" + | "official_source_url" + | "target_official_url"; +export type ExternalPublicActorAuthorityTier = "core" | "proven" | "watch" | "ordinary" | "unknown"; +export type ExternalPublicActorTierBasis = "registry_match" | "provider_hint" | "none"; +export type ExternalPublicActorIdentityStatus = "available" | "missing" | "invalid_reserved_path" | "redacted"; +export type ExternalPublicActorIdentityReason = + | "actor_public_identity_available" + | "actor_public_identity_missing" + | "x_reserved_or_indirect_url" + | "official_source_url_missing" + | "registry_entity_not_matched" + | "redacted_for_public_safety"; export type ExternalTrendWindowStatus = "ok" | "partial" | "failed" | "insufficient" | "skipped"; export type ExternalTrendWindowReadStatus = | "ok" @@ -41,6 +63,41 @@ export const EXTERNAL_TARGET_TYPES = ["project", "paper", "product", "topic"] as export const EXTERNAL_ACTOR_TYPES = ["institution", "team", "person", "community", "unknown"] as const; export const EXTERNAL_REGISTRY_TIERS = ["core", "proven", "watch"] as const; export const EXTERNAL_NAMED_ACTOR_SOURCE_ROLES = ["social_discussant", "official_publisher", "official_owner"] as const; +export const EXTERNAL_PUBLIC_ACTOR_ROLES = [ + "discussion_actor", + "community_source", + "official_publisher", + "project_owner", + "registry_entity", +] as const; +export const EXTERNAL_PUBLIC_ACTOR_SOURCE_KINDS = [ + "registry_entity", + "x_handle", + "reddit_community", + "reddit_user", + "hn_user", + "github_owner", + "official_domain", + "provider_actor", +] as const; +export const EXTERNAL_PUBLIC_ACTOR_SOURCE_BASES = [ + "registry_match", + "explicit_actor_field", + "source_url_path", + "official_source_url", + "target_official_url", +] as const; +export const EXTERNAL_PUBLIC_ACTOR_AUTHORITY_TIERS = ["core", "proven", "watch", "ordinary", "unknown"] as const; +export const EXTERNAL_PUBLIC_ACTOR_TIER_BASES = ["registry_match", "provider_hint", "none"] as const; +export const EXTERNAL_PUBLIC_ACTOR_IDENTITY_STATUSES = ["available", "missing", "invalid_reserved_path", "redacted"] as const; +export const EXTERNAL_PUBLIC_ACTOR_IDENTITY_REASONS = [ + "actor_public_identity_available", + "actor_public_identity_missing", + "x_reserved_or_indirect_url", + "official_source_url_missing", + "registry_entity_not_matched", + "redacted_for_public_safety", +] as const; export const EXTERNAL_TREND_WINDOW_READ_STATUSES = ["ok", "not_found", "parse_error", "partial", "insufficient", "failed", "skipped"] as const; export const EXTERNAL_WEEKLY_GATE_REASONS = [ "cross_platform_confirmation", @@ -65,6 +122,12 @@ export interface ExternalSignalActor { identity_hash?: string; display_name?: string; handle?: string; + author?: string; + username?: string; + user?: string; + subreddit?: string; + community?: string; + hn_user?: string; platform_profile_url?: string; provider_tier_hint?: ExternalProviderTierHint; registry_entity_id?: string; @@ -86,9 +149,14 @@ export interface ExternalSignalEvent { source_published_at?: string; ingested_at?: string; url?: string; + source_url?: string; + permalink?: string; + discussion_url?: string; target_url?: string; target_repo_url?: string; raw_ref?: string; + actor_public_identity_status?: ExternalPublicActorIdentityStatus; + actor_public_identity_reason?: ExternalPublicActorIdentityReason; } export interface ExternalNamedRegistryActor { @@ -103,6 +171,39 @@ export interface ExternalNamedRegistryActor { last_seen_at: string; } +export type ExternalPublicActorSourceKind = + | "registry_entity" + | "x_handle" + | "reddit_community" + | "reddit_user" + | "hn_user" + | "github_owner" + | "official_domain" + | "provider_actor"; + +export interface ExternalPublicActor { + public_actor_id: string; + display_name: string; + actor_type: ExternalActorType; + actor_role: ExternalPublicActorRole; + authority_tier?: ExternalPublicActorAuthorityTier; + tier_basis: ExternalPublicActorTierBasis; + is_head_actor: boolean; + source_kind: ExternalPublicActorSourceKind; + source_basis: ExternalPublicActorSourceBasis; + event_count: number; + platforms: ExternalPlatform[]; + first_seen_at: string; + last_seen_at: string; +} + +export interface ExternalPublicActorAudit { + platform: ExternalPlatform; + status: ExternalPublicActorIdentityStatus; + reason: ExternalPublicActorIdentityReason; + event_count: number; +} + export interface ExternalEvidence { evidence_id: string; event_ids: string[]; @@ -111,6 +212,8 @@ export interface ExternalEvidence { derived_signal_kinds: ExternalSignalKind[]; platforms: ExternalPlatform[]; named_registry_actors: ExternalNamedRegistryActor[]; + public_actors?: ExternalPublicActor[]; + public_actor_audit?: ExternalPublicActorAudit[]; actor_tiers: Partial>; actor_types: Partial>; mention_count: number; diff --git a/src/llm.ts b/src/llm.ts index c0bea79..701d222 100644 --- a/src/llm.ts +++ b/src/llm.ts @@ -3,12 +3,14 @@ import { createProvider, formatProviderError, isRetryableProviderError, + ProviderCallError, type LlmProvider, } from "./providers/index.ts"; export const LLM_TOKENS_CLASSIFICATION = 1024; const DEFAULT_LLM_CONCURRENCY = 2; +const DEFAULT_LLM_CALL_TIMEOUT_MS = 120_000; const DEFAULT_LLM_MAX_RETRIES = 1; const DEFAULT_LLM_RETRY_BASE_MS = 1000; const REDACTED_SECRET = "[REDACTED_SECRET]"; @@ -132,6 +134,30 @@ function releaseSlot(): void { } } +function withLlmTimeout(operation: Promise, providerName: string, timeoutMs: number): Promise { + let timeout: ReturnType | undefined; + const timeoutPromise = new Promise((_resolve, reject) => { + timeout = setTimeout(() => { + reject( + new ProviderCallError(`Provider timeout for ${providerName}`, { + providerName, + kind: "timeout", + retryable: true, + code: "LLM_CALL_TIMEOUT", + }), + ); + }, timeoutMs); + }); + + return Promise.race([operation, timeoutPromise]).finally(() => { + if (timeout) clearTimeout(timeout); + }); +} + +function shouldLogLlmProgress(): boolean { + return process.env["LLM_DEBUG_PROGRESS"] === "1"; +} + /** * LLM 调用层只消费 provider 归一化后的错误语义。 * 这样重试策略不再依赖 SDK 原始字符串细节，后续接更多 provider 也能复用。 @@ -141,16 +167,40 @@ export async function callLlm( opts: { providerName?: string; maxTokens?: number; maxRetries?: number; retryBaseMs?: number } = {}, ): Promise { const provider = resolveProvider(opts.providerName); - const sanitizedPrompt = sanitizePromptForLlm(prompt).prompt; + if (shouldLogLlmProgress()) { + console.error(`[llm] preparing provider=${provider.name} prompt_chars=${prompt.length}`); + } + const sanitizeStartedAt = Date.now(); + const sanitized = sanitizePromptForLlm(prompt); + const sanitizedPrompt = sanitized.prompt; const maxTokens = opts.maxTokens ?? LLM_TOKENS_CLASSIFICATION; + const callTimeoutMs = readPositiveInt( + process.env["LLM_CALL_TIMEOUT_MS"] ?? process.env["LLM_PROVIDER_TIMEOUT_MS"] ?? process.env["LLM_TIMEOUT_MS"], + DEFAULT_LLM_CALL_TIMEOUT_MS, + ); const maxRetries = opts.maxRetries ?? readPositiveInt(process.env["LLM_MAX_RETRIES"], DEFAULT_LLM_MAX_RETRIES); const retryBaseMs = opts.retryBaseMs ?? readPositiveInt(process.env["LLM_RETRY_BASE_MS"], DEFAULT_LLM_RETRY_BASE_MS); + if (shouldLogLlmProgress()) { + console.error( + `[llm] sanitized provider=${provider.name} prompt_chars=${sanitizedPrompt.length} redactions=${sanitized.redactionCount} elapsed_ms=${Date.now() - sanitizeStartedAt}`, + ); + } for (let attempt = 0; ; attempt++) { await acquireSlot(); let released = false; try { - return await provider.call(sanitizedPrompt, maxTokens); + if (shouldLogLlmProgress()) { + console.error( + `[llm] call start provider=${provider.name} attempt=${attempt + 1} max_tokens=${maxTokens} timeout_ms=${callTimeoutMs}`, + ); + } + const callStartedAt = Date.now(); + const response = await withLlmTimeout(provider.call(sanitizedPrompt, maxTokens), provider.name, callTimeoutMs); + if (shouldLogLlmProgress()) { + console.error(`[llm] call done provider=${provider.name} elapsed_ms=${Date.now() - callStartedAt}`); + } + return response; } catch (err) { if (attempt < maxRetries && isRetryableProviderError(err)) { releaseSlot(); diff --git a/src/providers/deepseek.ts b/src/providers/deepseek.ts index 0aa67e9..db84a78 100644 --- a/src/providers/deepseek.ts +++ b/src/providers/deepseek.ts @@ -100,9 +100,12 @@ export class DeepSeekProvider extends OpenAICompatibleProvider { thinking: { type: "disabled" }, ...(wantsJsonOutput(prompt) ? { response_format: { type: "json_object" } } : {}), }; - const response = await this.client.chat.completions.create( - request as OpenAI.Chat.ChatCompletionCreateParamsNonStreaming, - ); + const response = await this.runWithTimeout((signal) => + this.client.chat.completions.create( + request as OpenAI.Chat.ChatCompletionCreateParamsNonStreaming, + { signal }, + ), + ); const text = response.choices[0]?.message?.content; if (!text) { throw buildProviderResponseError(this.name, "empty_response", `Unexpected empty response from ${this.name}`); diff --git a/src/providers/openai-compatible.ts b/src/providers/openai-compatible.ts index 61e5da4..60be32f 100644 --- a/src/providers/openai-compatible.ts +++ b/src/providers/openai-compatible.ts @@ -14,6 +14,20 @@ import OpenAI from "openai"; import { buildProviderResponseError, classifyProviderError } from "./providerErrors.ts"; import type { LlmProvider } from "./types.ts"; +const DEFAULT_PROVIDER_TIMEOUT_MS = 60_000; + +function readPositiveInt(value: string | undefined, fallback: number): number { + const parsed = Number.parseInt(value ?? "", 10); + return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; +} + +function resolveProviderTimeoutMs(): number { + return readPositiveInt( + process.env["LLM_PROVIDER_TIMEOUT_MS"] ?? process.env["LLM_TIMEOUT_MS"] ?? process.env["OPENAI_TIMEOUT_MS"], + DEFAULT_PROVIDER_TIMEOUT_MS, + ); +} + export abstract class OpenAICompatibleProvider implements LlmProvider { abstract readonly name: string; protected readonly client: OpenAI; @@ -25,17 +39,33 @@ export abstract class OpenAICompatibleProvider implements LlmProvider { apiKey: opts.apiKey, baseURL: opts.baseURL, maxRetries: 0, + timeout: resolveProviderTimeoutMs(), }); } + protected async runWithTimeout(operation: (signal: AbortSignal) => Promise): Promise { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), resolveProviderTimeoutMs()); + try { + return await operation(controller.signal); + } finally { + clearTimeout(timeout); + } + } + async call(prompt: string, maxTokens: number): Promise { try { // 这里统一走 chat.completions，保证兼容 provider 的最小公共能力一致。 - const response = await this.client.chat.completions.create({ - model: this.model, - max_completion_tokens: maxTokens, - messages: [{ role: "user", content: prompt }], - }); + const response = await this.runWithTimeout((signal) => + this.client.chat.completions.create( + { + model: this.model, + max_completion_tokens: maxTokens, + messages: [{ role: "user", content: prompt }], + }, + { signal }, + ), + ); const text = response.choices[0]?.message?.content; if (!text) { throw buildProviderResponseError(this.name, "empty_response", `Unexpected empty response from ${this.name}`); From 00bc6ac0bcd9b6bc6d679a8726e8cb12adca1fcd Mon Sep 17 00:00:00 2001 From: Aspetta <892024285@qq.com> Date: Mon, 6 Jul 2026 00:42:42 +0800 Subject: [PATCH 7/8] fix(agentreach): clean backend verification formatting --- src/action/dailyVerification.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/action/dailyVerification.ts b/src/action/dailyVerification.ts index 97095f4..c6ccdd5 100644 --- a/src/action/dailyVerification.ts +++ b/src/action/dailyVerification.ts @@ -6,9 +6,9 @@ import { assertPublicSafeCandidateExplanations } from "../externalDiscovery/expl import type { ExternalCandidateExplanationArtifact } from "../externalDiscovery/explanations.ts"; import { readExternalDiscussionTrendWindowByDate, type ExternalDiscussionTrendWindowReadResult } from "../externalDiscovery/trendWindowIntegration.ts"; import { readJsonFile } from "../storage/files.ts"; -import type { - DailyReport, - DailyRunSummary, +import type { + DailyReport, + DailyRunSummary, DailyFreshnessSource, GitHubEnrichmentAuditEntry, IndustryRuntimeSummaryArtifact, From cad4f18687d8e7872be1b53c9ff96eeeb955751a Mon Sep 17 00:00:00 2001 From: Aspetta <892024285@qq.com> Date: Mon, 6 Jul 2026 00:54:22 +0800 Subject: [PATCH 8/8] refactor(agentreach): reuse external platform validator --- src/action/dailyVerification.ts | 5 +---- src/externalDiscovery/agentReachProvider.ts | 5 +---- src/externalDiscovery/redaction.ts | 9 +++------ src/externalDiscovery/types.ts | 4 ++++ 4 files changed, 9 insertions(+), 14 deletions(-) diff --git a/src/action/dailyVerification.ts b/src/action/dailyVerification.ts index c6ccdd5..a83f419 100644 --- a/src/action/dailyVerification.ts +++ b/src/action/dailyVerification.ts @@ -5,6 +5,7 @@ import { assertPublicSafeAggregate, assertPublicSafeTrendWindow } from "../exter import { assertPublicSafeCandidateExplanations } from "../externalDiscovery/explanationRedaction.ts"; import type { ExternalCandidateExplanationArtifact } from "../externalDiscovery/explanations.ts"; import { readExternalDiscussionTrendWindowByDate, type ExternalDiscussionTrendWindowReadResult } from "../externalDiscovery/trendWindowIntegration.ts"; +import { isExternalPlatform } from "../externalDiscovery/types.ts"; import { readJsonFile } from "../storage/files.ts"; import type { DailyReport, @@ -853,10 +854,6 @@ function inspectPublicActorAudit(value: unknown, prefix: string): string[] { return issues; } -function isExternalPlatform(value: unknown): boolean { - return value === "x_twitter" || value === "reddit" || value === "hacker_news" || value === "official_web" || value === "official_blog"; -} - function isExternalActorType(value: unknown): boolean { return value === "institution" || value === "team" || value === "person" || value === "community" || value === "unknown"; } diff --git a/src/externalDiscovery/agentReachProvider.ts b/src/externalDiscovery/agentReachProvider.ts index 2beaf56..dedb225 100644 --- a/src/externalDiscovery/agentReachProvider.ts +++ b/src/externalDiscovery/agentReachProvider.ts @@ -4,6 +4,7 @@ import { applyEntityRegistry, readEntityRegistryWithWarnings, type ExternalEntit import { externalEntityRegistryPath } from "./paths.ts"; import { publicActorIdentityForEvent } from "./publicActors.ts"; import { stableSourceInputHash } from "./redaction.ts"; +import { isExternalPlatform } from "./types.ts"; import type { AgentReachProviderReadResult, ExternalCandidateExplanationTitleContext, @@ -417,10 +418,6 @@ function validateTopLevelContract(value: Record): return { ok: true }; } -function isExternalPlatform(value: unknown): value is ExternalPlatform { - return value === "x_twitter" || value === "reddit" || value === "hacker_news" || value === "official_web" || value === "official_blog"; -} - function isProviderStatus(value: unknown): value is ExternalProviderStatus { return value === "ok" || value === "skipped" || value === "partial" || value === "failed"; } diff --git a/src/externalDiscovery/redaction.ts b/src/externalDiscovery/redaction.ts index 3367ed1..ffd5a7b 100644 --- a/src/externalDiscovery/redaction.ts +++ b/src/externalDiscovery/redaction.ts @@ -1,4 +1,5 @@ import crypto from "node:crypto"; +import { isExternalPlatform } from "./types.ts"; export const REDACTION_POLICY_VERSION = "external-discovery-redaction.v1"; @@ -155,7 +156,7 @@ function inspectPublicActors(value: unknown): string[] { reasonCodes.push("public_actor_official_or_project_head_invalid"); } if (typeof actor.event_count !== "number" || actor.event_count <= 0) reasonCodes.push("public_actor_event_count_invalid"); - if (!Array.isArray(actor.platforms) || actor.platforms.some((platform) => !isPlatform(platform))) { + if (!Array.isArray(actor.platforms) || actor.platforms.some((platform) => !isExternalPlatform(platform))) { reasonCodes.push("public_actor_platforms_invalid"); } if (typeof actor.first_seen_at !== "string" || actor.first_seen_at.length === 0) reasonCodes.push("public_actor_first_seen_missing"); @@ -175,7 +176,7 @@ function inspectPublicActorAudit(value: unknown): string[] { } const extraKeys = Object.keys(audit).filter((key) => !["platform", "status", "reason", "event_count"].includes(key)); if (extraKeys.length > 0) reasonCodes.push("public_actor_audit_extra_keys"); - if (!isPlatform(audit.platform)) reasonCodes.push("public_actor_audit_platform_invalid"); + if (!isExternalPlatform(audit.platform)) reasonCodes.push("public_actor_audit_platform_invalid"); if (!isIdentityStatus(audit.status)) reasonCodes.push("public_actor_audit_status_invalid"); if (!isIdentityReason(audit.reason)) reasonCodes.push("public_actor_audit_reason_invalid"); if (audit.status === "available" && audit.reason !== "actor_public_identity_available") { @@ -191,10 +192,6 @@ function inspectPublicActorAudit(value: unknown): string[] { return reasonCodes; } -function isPlatform(value: unknown): boolean { - return value === "x_twitter" || value === "reddit" || value === "hacker_news" || value === "official_web" || value === "official_blog"; -} - function isActorType(value: unknown): boolean { return value === "institution" || value === "team" || value === "person" || value === "community" || value === "unknown"; } diff --git a/src/externalDiscovery/types.ts b/src/externalDiscovery/types.ts index f9b4886..d1911d7 100644 --- a/src/externalDiscovery/types.ts +++ b/src/externalDiscovery/types.ts @@ -114,6 +114,10 @@ export const EXTERNAL_TREND_COMPONENT_NAMES = [ "noise_risk", ] as const; +export function isExternalPlatform(value: unknown): value is ExternalPlatform { + return EXTERNAL_PLATFORMS.includes(value as ExternalPlatform); +} + export interface ExternalSignalActor { actor_type: ExternalActorType; effective_tier: ExternalActorTier;