From 1561073c78a4c0e693a91cad2501e522320d1855 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 7 Aug 2026 15:38:18 +0000 Subject: [PATCH 01/19] chore: release release --- .github/.release-please-manifest.json | 2 +- CHANGELOG.md | 7 +++++++ package.json | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/.release-please-manifest.json b/.github/.release-please-manifest.json index cddbaefb9..c95106184 100644 --- a/.github/.release-please-manifest.json +++ b/.github/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "1.1.0" + ".": "1.2.0" } diff --git a/CHANGELOG.md b/CHANGELOG.md index 0fa2ed919..e2dea4a35 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [1.2.0](https://github.com/trycompai/crm/compare/v1.1.0...v1.2.0) (2026-08-07) + + +### Features + +* **api:** add microsoft sign-in and outlook mailbox sync ([#73](https://github.com/trycompai/crm/issues/73)) ([2a0062f](https://github.com/trycompai/crm/commit/2a0062fb76ffdaa5bbbb3848a5573b8b53cd0036)) + ## [1.1.0](https://github.com/trycompai/crm/compare/v1.0.0...v1.1.0) (2026-08-06) diff --git a/package.json b/package.json index 58f50e8b1..bd61fd43b 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "crm", "private": true, "license": "MIT", - "version": "1.1.0", + "version": "1.2.0", "scripts": { "prepare": "git rev-parse --git-dir >/dev/null 2>&1 && git config core.hooksPath .githooks || true", "build": "turbo run build", From cb257d1851ca1e80bb8c8cf070b4297c0aefdbd6 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 2 Sep 2026 15:46:36 +0000 Subject: [PATCH 02/19] fix(dev): stop the tRPC watcher self-triggering on Linux nestjs-trpc watch reacts to inotify open events, so one read of a router file starts an endless regeneration loop. dev:trpc now runs a small fs.watch script that regenerates only on writes. Turbo passes the proxy and CA variables through so eve can fetch the AI Gateway catalog behind a proxy, engines moves to Node 24 as eve requires, and a SessionStart hook script prepares a remote session. Claude-Session: https://claude.ai/code/session_011fuk1HeytBt9RU12xKY8WB --- .claude/hooks/session-start.sh | 43 +++++++++++++++++++++++++++++++++ apps/api/package.json | 2 +- apps/api/scripts/dev-config.ts | 6 +++++ apps/api/scripts/trpc-watch.ts | 44 ++++++++++++++++++++++++++++++++++ package.json | 2 +- turbo.json | 7 +++++- 6 files changed, 101 insertions(+), 3 deletions(-) create mode 100644 .claude/hooks/session-start.sh create mode 100644 apps/api/scripts/dev-config.ts create mode 100644 apps/api/scripts/trpc-watch.ts diff --git a/.claude/hooks/session-start.sh b/.claude/hooks/session-start.sh new file mode 100644 index 000000000..cfc531451 --- /dev/null +++ b/.claude/hooks/session-start.sh @@ -0,0 +1,43 @@ +#!/bin/bash +set -euo pipefail + +if [ "${CLAUDE_CODE_REMOTE:-}" != "true" ]; then + exit 0 +fi + +cd "$CLAUDE_PROJECT_DIR" + +NODE_DIR=/opt/node24 +if [ ! -x "$NODE_DIR/bin/node" ]; then + VERSION=$(curl -sSL https://nodejs.org/dist/index.json | python3 -c 'import sys,json; print(next(v["version"] for v in json.load(sys.stdin) if v["version"].startswith("v24.")))') + mkdir -p "$NODE_DIR" + curl -sSL "https://nodejs.org/dist/$VERSION/node-$VERSION-linux-x64.tar.xz" | tar -xJ -C "$NODE_DIR" --strip-components=1 +fi +export PATH="$NODE_DIR/bin:$PATH" +echo "export PATH=\"$NODE_DIR/bin:\$PATH\"" >> "$CLAUDE_ENV_FILE" + +if [ ! -f .env ]; then + cp .env.example .env + sed -i "s|^BETTER_AUTH_SECRET=\"\"|BETTER_AUTH_SECRET=\"$(openssl rand -base64 32)\"|" .env + sed -i "s|^# AGENT_URL=\"http://127.0.0.1:2000\"|AGENT_URL=\"http://127.0.0.1:2000\"|" .env + sed -i "s|^# AGENT_BRIDGE_SECRET=\"\"|AGENT_BRIDGE_SECRET=\"$(openssl rand -base64 32)\"|" .env + sed -i "s|^ALLOWED_SIGN_IN=\"\"|ALLOWED_SIGN_IN=\"localhost\"|" .env +fi + +if ! docker info >/dev/null 2>&1; then + nohup dockerd >/tmp/dockerd.log 2>&1 & + for _ in $(seq 1 30); do + docker info >/dev/null 2>&1 && break + sleep 1 + done +fi + +docker compose up -d +for _ in $(seq 1 60); do + [ "$(docker inspect -f '{{.State.Health.Status}}' crm-postgres 2>/dev/null)" = "healthy" ] && break + sleep 1 +done + +bun install +bun run --filter=@crm/db db:deploy +bun run --filter=@crm/db db:seed diff --git a/apps/api/package.json b/apps/api/package.json index 151aec0f8..b3c9816fc 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -11,7 +11,7 @@ "build": "bun build src/main.ts --target=bun --outdir dist --packages=external --sourcemap", "check-types": "tsc --noEmit", "dev": "concurrently -n api,trpc -c blue,magenta \"bun --watch src/main.ts\" \"bun run dev:trpc\"", - "dev:trpc": "nestjs-trpc watch -e src/app.module.ts -r \"**/*.router.ts\" -o src/generated", + "dev:trpc": "bun scripts/trpc-watch.ts", "dev:session": "bun scripts/dev-session.ts", "lint": "biome check .", "postinstall": "node scripts/chmod-trpc-binary.mjs", diff --git a/apps/api/scripts/dev-config.ts b/apps/api/scripts/dev-config.ts new file mode 100644 index 000000000..d0284e817 --- /dev/null +++ b/apps/api/scripts/dev-config.ts @@ -0,0 +1,6 @@ +export const TRPC_WATCH = { + debounceMs: 150, + routerSuffix: ".router.ts", + sourceDir: "src", + generatedDir: "generated", +} as const; diff --git a/apps/api/scripts/trpc-watch.ts b/apps/api/scripts/trpc-watch.ts new file mode 100644 index 000000000..0a914e158 --- /dev/null +++ b/apps/api/scripts/trpc-watch.ts @@ -0,0 +1,44 @@ +import { spawn } from "node:child_process"; +import { watch } from "node:fs"; +import { sep } from "node:path"; +import { TRPC_WATCH } from "./dev-config"; + +let timer: ReturnType | null = null; +let running = false; +let queued = false; + +function generate(): void { + if (running) { + queued = true; + return; + } + running = true; + const child = spawn("bun", ["run", "trpc:generate"], { stdio: "inherit" }); + child.on("exit", () => { + running = false; + if (queued) { + queued = false; + generate(); + } + }); +} + +function schedule(): void { + if (timer) clearTimeout(timer); + timer = setTimeout(generate, TRPC_WATCH.debounceMs); +} + +function isRouterChange(file: string | null): boolean { + if (!file) return false; + if (file.startsWith(`${TRPC_WATCH.generatedDir}${sep}`)) return false; + return file.endsWith(TRPC_WATCH.routerSuffix); +} + +watch(TRPC_WATCH.sourceDir, { recursive: true }, (_event, file) => { + if (isRouterChange(file)) schedule(); +}); + +generate(); +console.log( + `[trpc] watching ${TRPC_WATCH.sourceDir} for *${TRPC_WATCH.routerSuffix} writes`, +); diff --git a/package.json b/package.json index 534ec9d00..f3590e38a 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "typescript": "5.9.2" }, "engines": { - "node": ">=22" + "node": ">=24" }, "packageManager": "bun@1.3.12", "devEngines": { diff --git a/turbo.json b/turbo.json index 110f9bd94..bd5ae18d1 100644 --- a/turbo.json +++ b/turbo.json @@ -33,7 +33,12 @@ "CRM_TELEMETRY_DISABLED", "DO_NOT_TRACK", "VERCEL", - "VERCEL_GIT_COMMIT_SHA" + "VERCEL_GIT_COMMIT_SHA", + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + "NODE_EXTRA_CA_CERTS", + "SSL_CERT_FILE" ], "tasks": { From 11069d5fca60c2220ab6b3a0daffc9eacaefde20 Mon Sep 17 00:00:00 2001 From: teknewmcc26 Date: Wed, 2 Sep 2026 18:14:43 +0200 Subject: [PATCH 03/19] chore(claude): install Agent Reach on remote session start (#1) Adds a SessionStart hook for Claude Code on the web that installs Agent Reach and its upstream tools outside the repo, registers Exa, LinkedIn and XiaoHongShu in mcporter, and puts the tool directories on PATH. Documents the hook in docs/setup.md. Claude-Session: https://claude.ai/code/session_01ENFcs9QNwdwtwEhffoxyGu Co-authored-by: Claude --- .claude/hooks/session-start.sh | 98 ++++++++++++++++++++++++++++++++++ .claude/settings.json | 12 +++++ docs/setup.md | 19 +++++++ 3 files changed, 129 insertions(+) create mode 100755 .claude/hooks/session-start.sh diff --git a/.claude/hooks/session-start.sh b/.claude/hooks/session-start.sh new file mode 100755 index 000000000..7339d6d6b --- /dev/null +++ b/.claude/hooks/session-start.sh @@ -0,0 +1,98 @@ +#!/bin/bash +set -uo pipefail + +if [ "${CLAUDE_CODE_REMOTE:-}" != "true" ]; then + exit 0 +fi + +AGENT_REACH_HOME="$HOME/.agent-reach" +AGENT_REACH_TOOLS="$AGENT_REACH_HOME/tools" +AGENT_REACH_VENV="$HOME/.agent-reach-venv" +AGENT_REACH_REPO="https://github.com/Panniantong/agent-reach.git" +XHS_REPO="https://github.com/xpzouying/xiaohongshu-mcp.git" +XHS_PORT=18060 +YTDLP_CONFIG="$HOME/.config/yt-dlp/config" +MCPORTER_CONFIG="$HOME/.mcporter/mcporter.json" +LOCAL_BIN="$HOME/.local/bin" + +export PATH="$AGENT_REACH_VENV/bin:$LOCAL_BIN:$PATH" + +log() { echo "[agent-reach hook] $*"; } + +step() { + local name="$1" + shift + if "$@"; then + log "ok: $name" + else + log "failed: $name (continuing)" + fi +} + +install_agent_reach() { + mkdir -p "$AGENT_REACH_TOOLS" + if [ ! -d "$AGENT_REACH_TOOLS/agent-reach/.git" ]; then + git clone -q --depth 1 "$AGENT_REACH_REPO" "$AGENT_REACH_TOOLS/agent-reach" + else + git -C "$AGENT_REACH_TOOLS/agent-reach" pull -q --ff-only || true + fi + [ -x "$AGENT_REACH_VENV/bin/pip" ] || python3 -m venv "$AGENT_REACH_VENV" + "$AGENT_REACH_VENV/bin/pip" install -q "$AGENT_REACH_TOOLS/agent-reach" +} + +install_apt_tools() { + local missing=() + command -v gh >/dev/null || missing+=(gh) + command -v ffmpeg >/dev/null || missing+=(ffmpeg) + [ "${#missing[@]}" -eq 0 ] && return 0 + apt-get update -q >/dev/null 2>&1 + DEBIAN_FRONTEND=noninteractive apt-get install -y -q "${missing[@]}" >/dev/null 2>&1 +} + +install_mcporter() { + command -v mcporter >/dev/null || npm install -g mcporter >/dev/null 2>&1 +} + +mcporter_has() { + [ -f "$MCPORTER_CONFIG" ] && grep -q "\"$1\"" "$MCPORTER_CONFIG" +} + +configure_mcporter() { + mcporter_has exa || mcporter config add exa https://mcp.exa.ai/mcp --scope home >/dev/null + mcporter_has linkedin || mcporter config add linkedin --command uvx --arg mcp-server-linkedin@latest --env UV_HTTP_TIMEOUT=300 --scope home >/dev/null + mcporter_has xiaohongshu || mcporter config add xiaohongshu "http://localhost:$XHS_PORT/mcp" --scope home >/dev/null +} + +configure_ytdlp() { + mkdir -p "$(dirname "$YTDLP_CONFIG")" + grep -qxF -- '--js-runtimes node' "$YTDLP_CONFIG" 2>/dev/null || printf '%s\n' '--js-runtimes node' >> "$YTDLP_CONFIG" +} + +install_channels() { + agent-reach install --env=auto --system --channels=all >/dev/null 2>&1 || true + return 0 +} + +build_xiaohongshu() { + [ -x "$AGENT_REACH_TOOLS/xiaohongshu-mcp" ] && return 0 + command -v go >/dev/null || return 1 + [ -d "$AGENT_REACH_TOOLS/xiaohongshu-mcp-src/.git" ] || git clone -q --depth 1 "$XHS_REPO" "$AGENT_REACH_TOOLS/xiaohongshu-mcp-src" + (cd "$AGENT_REACH_TOOLS/xiaohongshu-mcp-src" && CGO_ENABLED=0 go build -o "$AGENT_REACH_TOOLS/xiaohongshu-mcp" . && CGO_ENABLED=0 go build -o "$AGENT_REACH_TOOLS/xiaohongshu-login" ./cmd/login) >/dev/null 2>&1 +} + +export_path() { + [ -n "${CLAUDE_ENV_FILE:-}" ] || return 0 + echo "export PATH=\"$AGENT_REACH_VENV/bin:$LOCAL_BIN:\$PATH\"" >> "$CLAUDE_ENV_FILE" +} + +step "agent-reach package" install_agent_reach +step "gh + ffmpeg" install_apt_tools +step "mcporter" install_mcporter +step "mcporter servers" configure_mcporter +step "yt-dlp config" configure_ytdlp +step "optional channels" install_channels +step "xiaohongshu-mcp build" build_xiaohongshu +step "PATH" export_path + +agent-reach doctor 2>/dev/null | grep -E '^状态' || true +exit 0 diff --git a/.claude/settings.json b/.claude/settings.json index 97251ad97..abeca8504 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -4,5 +4,17 @@ }, "enabledPlugins": { "paper-desktop@paper": true + }, + "hooks": { + "SessionStart": [ + { + "hooks": [ + { + "type": "command", + "command": "$CLAUDE_PROJECT_DIR/.claude/hooks/session-start.sh" + } + ] + } + ] } } diff --git a/docs/setup.md b/docs/setup.md index 03b8912f7..30aa939f1 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -175,3 +175,22 @@ A rebuild drops the database and re-runs every migration, and it says which of t two reasons fired. Force one with `bun run db:test --reset`. Nothing else in the repo may drop a database, and this may only because the `_test` suffix is checked first. + +## Agent Reach in Claude Code on the web + +`.claude/hooks/session-start.sh` runs on every remote session start and installs +[Agent Reach](https://github.com/Panniantong/agent-reach) outside the repo: the +CLI in `~/.agent-reach-venv`, the upstream tools in `~/.agent-reach/tools` and +`~/.local/bin`, mcporter with Exa, LinkedIn and XiaoHongShu registered, gh and +ffmpeg from apt. It does nothing on a local machine. Every step is best effort: +a failed step logs and the session still starts. + +The proxy blocks GitHub archive and release downloads, so the hook clones over +git and builds `xiaohongshu-mcp` with Go instead of downloading a binary. The +hook adds both bin directories to `PATH` for the session, so `agent-reach +doctor`, `twitter`, `rdt`, `bili`, `yt-dlp` and `mcporter` work from any shell. + +Channels that need a login (Twitter, Reddit, XiaoHongShu, Xueqiu, LinkedIn, +Groq for podcasts) still need their cookies or keys pasted into +`agent-reach configure` each session. Facebook and Instagram need a desktop +Chrome and never work in the container. From 21c53e55d2d5fd08bec95c29305a86194b5076be Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 2 Sep 2026 18:13:43 +0000 Subject: [PATCH 04/19] chore(claude): register the CRM dev hook on remote session start Claude-Session: https://claude.ai/code/session_011fuk1HeytBt9RU12xKY8WB --- .claude/settings.json | 4 ++++ docs/setup.md | 5 +++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.claude/settings.json b/.claude/settings.json index abeca8504..7f426f2fa 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -12,6 +12,10 @@ { "type": "command", "command": "$CLAUDE_PROJECT_DIR/.claude/hooks/session-start.sh" + }, + { + "type": "command", + "command": "$CLAUDE_PROJECT_DIR/.claude/hooks/crm-dev.sh" } ] } diff --git a/docs/setup.md b/docs/setup.md index 184f92e73..03ffb3610 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -197,8 +197,9 @@ Chrome and never work in the container. ## The CRM itself in Claude Code on the web -`.claude/hooks/crm-dev.sh` prepares the repo once it is listed as a second -`SessionStart` command in `.claude/settings.json`, after the Agent Reach hook: Node 24 in `/opt/node24` (eve refuses Node 22), a `.env` from +`.claude/hooks/crm-dev.sh` is the second `SessionStart` command in +`.claude/settings.json`. It runs after the Agent Reach hook and prepares the +repo: Node 24 in `/opt/node24` (eve refuses Node 22), a `.env` from `.env.example` with generated secrets when none exists, `dockerd` when no daemon answers, `docker compose up -d`, `bun install`, `migrate deploy` and the seed. It does nothing on a local machine. The `.env` it writes sets `ALLOWED_SIGN_IN` From 1b2ccaef32d2c1e90180cc1fc09b515f5f3116cd Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 2 Sep 2026 22:20:51 +0000 Subject: [PATCH 05/19] fix(deploy): daily crons so the Hobby plan accepts the build Vercel's Hobby plan rejects any cron more frequent than once a day, which failed both the API and the agent builds. The mailbox sync and the agent dispatch now run daily, and build-func.mjs copies the API crons from vercel.json instead of keeping a second list. Claude-Session: https://claude.ai/code/session_011fuk1HeytBt9RU12xKY8WB --- apps/agent/agent/schedules/dispatch.ts | 2 +- apps/api/scripts/build-func.mjs | 2 +- apps/api/vercel.json | 2 +- docs/environment.md | 8 ++++++-- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/apps/agent/agent/schedules/dispatch.ts b/apps/agent/agent/schedules/dispatch.ts index 1aa080fe6..ea4cc60fb 100644 --- a/apps/agent/agent/schedules/dispatch.ts +++ b/apps/agent/agent/schedules/dispatch.ts @@ -10,7 +10,7 @@ import { brief, drainAll, taskAuth } from "../lib/dispatch"; import { reconcileStaleTasks } from "../lib/stale-tasks"; export default defineSchedule({ - cron: "* * * * *", + cron: "0 6 * * *", async run({ receive, waitUntil, appAuth }) { waitUntil( Promise.all([ diff --git a/apps/api/scripts/build-func.mjs b/apps/api/scripts/build-func.mjs index b06f0c017..5c24f67d2 100644 --- a/apps/api/scripts/build-func.mjs +++ b/apps/api/scripts/build-func.mjs @@ -177,7 +177,7 @@ writeFileSync( JSON.stringify({ version: 3, routes: [{ src: "/(.*)", dest: "/api/index" }], - crons: [{ path: "/internal/sync/google", schedule: "*/5 * * * *" }], + crons: JSON.parse(readFileSync(join(apiDir, "vercel.json"), "utf8")).crons, }), ); diff --git a/apps/api/vercel.json b/apps/api/vercel.json index ffce1ccc0..fc8d8f153 100644 --- a/apps/api/vercel.json +++ b/apps/api/vercel.json @@ -3,7 +3,7 @@ "crons": [ { "path": "/internal/sync/mailboxes", - "schedule": "*/5 * * * *" + "schedule": "0 8 * * *" }, { "path": "/internal/sync/rates", diff --git a/docs/environment.md b/docs/environment.md index 22417c60e..4d6309f5c 100644 --- a/docs/environment.md +++ b/docs/environment.md @@ -174,8 +174,12 @@ imports nothing, Calendar reads from `now`, and Outlook records `now` as its cur **`CRON_SECRET`** (min 16 chars) guards `POST /internal/sync/mailboxes` and `/internal/sync/rates`; both **fail closed when unset**. `/internal/sync/google` is kept as an alias of the first, so an existing deployment's cron does not break on -deploy. **Crons live in `apps/api/vercel.json`** — mailboxes `*/5 * * * *`, rates -daily. Minute-level schedules need a Pro plan; on Hobby it silently becomes daily. +deploy. **Crons live in `apps/api/vercel.json`**, and `build-func.mjs` copies them +into the Build Output config, so that file is the only place to edit. Every cron +there, and the agent's `schedules/dispatch.ts`, runs **once a day**: Vercel's Hobby +plan rejects the build for anything more frequent. On Pro, mailboxes can go back to +`*/5 * * * *` and the dispatch to `* * * * *`; the API's poke covers new tasks +between ticks either way. Deliberate absences: **no `GOOGLE_SYNC_ENABLED`** (a switch that can disable a mandatory feature is only ever wrong), **no `GOOGLE_WORKSPACE_DOMAIN`** (`ALLOWED_SIGN_IN` already From 28897b663b0caf4ab5d3995c3ede65e8b0148539 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 2 Sep 2026 22:36:48 +0000 Subject: [PATCH 06/19] fix(app): let the Vercel build see API_URL and APP_URL The app's build task replaced the root env list instead of extending it, so Turbo hid API_URL from next.config.ts on Vercel and the API proxy pointed at localhost. Claude-Session: https://claude.ai/code/session_011fuk1HeytBt9RU12xKY8WB --- apps/app/turbo.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/app/turbo.json b/apps/app/turbo.json index cea0ca0ed..3051a65cd 100644 --- a/apps/app/turbo.json +++ b/apps/app/turbo.json @@ -6,7 +6,7 @@ "dependsOn": ["^build"], "inputs": ["$TURBO_DEFAULT$", ".env*"], "outputs": [".next/**", "!.next/cache/**", "!.next/dev/**"], - "env": ["NEXT_PUBLIC_API_URL", "NEXT_PUBLIC_AUTH_URL"], + "env": ["$TURBO_EXTENDS$", "NEXT_PUBLIC_API_URL", "NEXT_PUBLIC_AUTH_URL"], "passThroughEnv": [ "AGENT_BRIDGE_SECRET", "AGENT_URL", From 9077b990927dec28074540df62e0eddadca9aee9 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 2 Sep 2026 22:45:11 +0000 Subject: [PATCH 07/19] feat(auth): optional email and password sign-in PASSWORD_SIGN_IN="true" enables Better Auth's emailAndPassword and adds a form to the sign-in page. Sign-up goes through the same allow-list hook as the social providers, so only an ALLOWED_SIGN_IN address can create an account. Claude-Session: https://claude.ai/code/session_011fuk1HeytBt9RU12xKY8WB --- .env.example | 6 ++ apps/api/src/config/env.validation.ts | 4 + apps/api/src/sso/sso.contracts.ts | 1 + apps/api/src/sso/sso.service.ts | 2 + apps/api/test/auth.e2e.spec.ts | 1 + apps/api/turbo.json | 2 + apps/app/app/(landing)/sign-in/page.tsx | 12 ++- .../(landing)/sign-in/password-sign-in.tsx | 74 +++++++++++++++++++ docs/environment.md | 11 ++- packages/auth/src/auth.ts | 2 +- packages/auth/src/env.ts | 5 ++ packages/auth/src/index.ts | 1 + turbo.json | 1 + 13 files changed, 115 insertions(+), 7 deletions(-) create mode 100644 apps/app/app/(landing)/sign-in/password-sign-in.tsx diff --git a/.env.example b/.env.example index 12fac543c..34874a019 100644 --- a/.env.example +++ b/.env.example @@ -57,6 +57,12 @@ GOOGLE_CLIENT_SECRET="" # SLACK_CLIENT_ID="" # SLACK_CLIENT_SECRET="" +# Optional. Adds an email + password form to the sign-in page, next to the +# social buttons. Only addresses that pass ALLOWED_SIGN_IN can create an +# account, exactly like the social sign-ins, and the password must be at +# least 8 characters. The only value that turns it on is "true". +# PASSWORD_SIGN_IN="true" + # Which Entra tenant may sign in. "common" (the default) accepts any work, # school or personal Microsoft account and leans on ALLOWED_SIGN_IN to decide # who actually gets in; your own tenant's GUID refuses everyone else at diff --git a/apps/api/src/config/env.validation.ts b/apps/api/src/config/env.validation.ts index 08cb676c2..1261e7599 100644 --- a/apps/api/src/config/env.validation.ts +++ b/apps/api/src/config/env.validation.ts @@ -68,6 +68,10 @@ export class EnvironmentVariables { @IsString() MICROSOFT_TENANT_ID?: string; + @IsOptional() + @IsString() + PASSWORD_SIGN_IN?: string; + @IsOptional() @IsString() SLACK_CLIENT_ID?: string; diff --git a/apps/api/src/sso/sso.contracts.ts b/apps/api/src/sso/sso.contracts.ts index d36d63e5c..afaa0d894 100644 --- a/apps/api/src/sso/sso.contracts.ts +++ b/apps/api/src/sso/sso.contracts.ts @@ -35,6 +35,7 @@ const ssoPublicProviderOutput = z.object({ export const ssoSignInOptionsOutput = z.object({ google: z.boolean(), microsoft: z.boolean(), + password: z.boolean(), providers: z.array(ssoPublicProviderOutput), }); diff --git a/apps/api/src/sso/sso.service.ts b/apps/api/src/sso/sso.service.ts index 340efcb65..b54cbe8e2 100644 --- a/apps/api/src/sso/sso.service.ts +++ b/apps/api/src/sso/sso.service.ts @@ -3,6 +3,7 @@ import { canConfigureSso, isGoogleConfigured, isMicrosoftConfigured, + isPasswordSignInEnabled, ssoCallbackBase, ssoCallbackURL, ssoProviderName, @@ -126,6 +127,7 @@ export class SsoService { return { google: isGoogleConfigured(), microsoft: isMicrosoftConfigured(), + password: isPasswordSignInEnabled(), providers: rows.map((row) => ({ providerId: row.providerId, name: ssoProviderName(row.providerId), diff --git a/apps/api/test/auth.e2e.spec.ts b/apps/api/test/auth.e2e.spec.ts index d5d79131d..950ec5231 100644 --- a/apps/api/test/auth.e2e.spec.ts +++ b/apps/api/test/auth.e2e.spec.ts @@ -67,6 +67,7 @@ describe("Auth (e2e)", () => { expect(response.body.result.data).toEqual({ google: true, microsoft: microsoftConfigured, + password: process.env.PASSWORD_SIGN_IN === "true", providers: [], }); }); diff --git a/apps/api/turbo.json b/apps/api/turbo.json index 04699fe51..5bf076837 100644 --- a/apps/api/turbo.json +++ b/apps/api/turbo.json @@ -31,6 +31,7 @@ "MICROSOFT_CLIENT_ID", "MICROSOFT_CLIENT_SECRET", "MICROSOFT_TENANT_ID", + "PASSWORD_SIGN_IN", "PORT", "REDIS_URL" ] @@ -49,6 +50,7 @@ "MICROSOFT_CLIENT_ID", "MICROSOFT_CLIENT_SECRET", "MICROSOFT_TENANT_ID", + "PASSWORD_SIGN_IN", "PORT", "REDIS_URL" ] diff --git a/apps/app/app/(landing)/sign-in/page.tsx b/apps/app/app/(landing)/sign-in/page.tsx index 5b0e451af..8f5346ae7 100644 --- a/apps/app/app/(landing)/sign-in/page.tsx +++ b/apps/app/app/(landing)/sign-in/page.tsx @@ -5,6 +5,7 @@ import { Suspense } from "react"; import { AuthHeading, AuthShell } from "@/components/auth-shell"; import { getSession } from "@/lib/session"; import { getServerQueryClient, getServerTrpc } from "@/lib/trpc/server"; +import { PasswordSignIn } from "./password-sign-in"; import { SocialSignIn } from "./social-sign-in"; import { type SsoProvider, SsoSignIn } from "./sso-sign-in"; @@ -15,6 +16,7 @@ export const metadata: Metadata = { type SignInOptions = { google: boolean; microsoft: boolean; + password: boolean; providers: SsoProvider[]; }; @@ -75,6 +77,7 @@ async function SignIn({ if (options?.microsoft ?? false) configured.push("microsoft"); const providers = options?.providers ?? []; + const showPassword = options?.password ?? false; const insisted = configured.find((provider) => provider === method); const showSso = providers.length > 0 && insisted === undefined; @@ -85,7 +88,7 @@ async function SignIn({ ? configured : []; - if (!showSso && social.length === 0) { + if (!showSso && social.length === 0 && !showPassword) { return ( <> Set GOOGLE_CLIENT_ID and GOOGLE_CLIENT_SECRET — or MICROSOFT_CLIENT_ID - and MICROSOFT_CLIENT_SECRET — in the root .env file and restart. Your - own identity provider can be added from Settings once somebody is - signed in. + and MICROSOFT_CLIENT_SECRET, or PASSWORD_SIGN_IN — in the root .env + file and restart. Your own identity provider can be added from + Settings once somebody is signed in.

); @@ -110,6 +113,7 @@ async function SignIn({ description="Sign in with your account to continue." /> + {showPassword ? : null} {showSso ? : null} {social.map((provider) => ( diff --git a/apps/app/app/(landing)/sign-in/password-sign-in.tsx b/apps/app/app/(landing)/sign-in/password-sign-in.tsx new file mode 100644 index 000000000..0d4445300 --- /dev/null +++ b/apps/app/app/(landing)/sign-in/password-sign-in.tsx @@ -0,0 +1,74 @@ +"use client"; + +import { signIn } from "@crm/auth/client"; +import { Button } from "@crm/ui/components/button"; +import { Field, FieldLabel } from "@crm/ui/components/field"; +import { Input } from "@crm/ui/components/input"; +import { Spinner } from "@crm/ui/components/spinner"; +import { useRouter } from "next/navigation"; +import { type FormEvent, useId, useState } from "react"; +import { toast } from "sonner"; + +export function PasswordSignIn() { + const router = useRouter(); + const emailId = useId(); + const passwordId = useId(); + const [email, setEmail] = useState(""); + const [password, setPassword] = useState(""); + const [pending, setPending] = useState(false); + + async function handleSubmit(event: FormEvent) { + event.preventDefault(); + setPending(true); + + const { error } = await signIn.email({ email, password }); + + if (error) { + setPending(false); + toast.error(error.message ?? "Could not sign in."); + return; + } + + router.replace("/"); + router.refresh(); + } + + return ( +
{ + handleSubmit(event).catch(() => { + setPending(false); + toast.error("Could not reach the sign-in service."); + }); + }} + > + + Email + setEmail(event.target.value)} + /> + + + Password + setPassword(event.target.value)} + /> + + +
+ ); +} diff --git a/docs/environment.md b/docs/environment.md index 22417c60e..edab83665 100644 --- a/docs/environment.md +++ b/docs/environment.md @@ -43,8 +43,15 @@ the three that is genuinely optional on its own — set it to your tenant's GUID refuse other tenants at Microsoft instead of at `ALLOWED_SIGN_IN`. There is **no Microsoft equivalent of `hd`**: `tenantId` is the whole of it. -**Neither pair is required, but an install wants one of them or an SSO provider** — -with none, the sign-in page says so by name rather than rendering nothing. +**`PASSWORD_SIGN_IN="true"`** adds an email + password form to the sign-in page +(`emailAndPassword` in `auth.ts`). Sign-up goes through the same +`user.create.before` hook as the social providers, so only an address that passes +`ALLOWED_SIGN_IN` can create an account. Off by default; only the literal `true` +turns it on. + +**Neither pair is required, but an install wants one of them, a password form, or +an SSO provider** — with none, the sign-in page says so by name rather than +rendering nothing. **`ALLOWED_SIGN_IN`** — comma-separated whole domains or single addresses (bare addresses exist for a solo self-hoster, where `gmail.com` would be an open door). **One diff --git a/packages/auth/src/auth.ts b/packages/auth/src/auth.ts index c59c98254..32d3d3cdb 100644 --- a/packages/auth/src/auth.ts +++ b/packages/auth/src/auth.ts @@ -78,7 +78,7 @@ export const auth = betterAuth({ }), emailAndPassword: { - enabled: false, + enabled: env.passwordSignIn, }, socialProviders, diff --git a/packages/auth/src/env.ts b/packages/auth/src/env.ts index 9813d7a44..4875a0962 100644 --- a/packages/auth/src/env.ts +++ b/packages/auth/src/env.ts @@ -62,6 +62,7 @@ export const env = { google: googleCredentials(), microsoft: microsoftCredentials(), slack: slackCredentials(), + passwordSignIn: optional("PASSWORD_SIGN_IN") === "true", cookieDomain: optional("AUTH_COOKIE_DOMAIN"), trustedOrigins: [...new Set([...appUrls, apiUrl])], isProduction: process.env.NODE_ENV === "production", @@ -79,4 +80,8 @@ export function isSlackConfigured(): boolean { return env.slack !== undefined; } +export function isPasswordSignInEnabled(): boolean { + return env.passwordSignIn; +} + export { apiUrl, appUrl }; diff --git a/packages/auth/src/index.ts b/packages/auth/src/index.ts index 31fbcbaad..cb09172eb 100644 --- a/packages/auth/src/index.ts +++ b/packages/auth/src/index.ts @@ -11,6 +11,7 @@ export { appUrl, isGoogleConfigured, isMicrosoftConfigured, + isPasswordSignInEnabled, isSlackConfigured, } from "./env"; export { diff --git a/turbo.json b/turbo.json index bd5ae18d1..90971be16 100644 --- a/turbo.json +++ b/turbo.json @@ -17,6 +17,7 @@ "SLACK_CLIENT_ID", "SLACK_CLIENT_SECRET", "MICROSOFT_TENANT_ID", + "PASSWORD_SIGN_IN", "AUTH_COOKIE_DOMAIN", "CRON_SECRET", "REDIS_URL", From c42c22a9d95efd384a51e8f670bd8d76d76f1c96 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 2 Sep 2026 23:30:31 +0000 Subject: [PATCH 08/19] feat(agent): GLEIF M&A sourcing tools and skill MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three free tools on the public GLEIF register — search entities by name, read one by LEI with its direct parent, list the subsidiaries an entity consolidates filtered by country or region — parsed at the boundary into GleifEntity. The gleif-mna-sourcing skill carries the method: a parent place and a child place make a scenario, the subsidiaries in the child place are the targets, and the people who run them come from web research under the existing egress rules. Claude-Session: https://claude.ai/code/session_011fuk1HeytBt9RU12xKY8WB --- apps/agent/agent/lib/gleif-config.ts | 71 +++++ apps/agent/agent/lib/gleif.ts | 253 ++++++++++++++++++ .../agent/skills/gleif-mna-sourcing/SKILL.md | 79 ++++++ apps/agent/agent/tools/gleif_get_entity.ts | 33 +++ .../agent/tools/gleif_list_subsidiaries.ts | 48 ++++ .../agent/tools/gleif_search_entities.ts | 54 ++++ apps/agent/test/fixtures/gleif.json | 130 +++++++++ apps/agent/test/gleif.spec.ts | 218 +++++++++++++++ docs/agent.md | 13 + 9 files changed, 899 insertions(+) create mode 100644 apps/agent/agent/lib/gleif-config.ts create mode 100644 apps/agent/agent/lib/gleif.ts create mode 100644 apps/agent/agent/skills/gleif-mna-sourcing/SKILL.md create mode 100644 apps/agent/agent/tools/gleif_get_entity.ts create mode 100644 apps/agent/agent/tools/gleif_list_subsidiaries.ts create mode 100644 apps/agent/agent/tools/gleif_search_entities.ts create mode 100644 apps/agent/test/fixtures/gleif.json create mode 100644 apps/agent/test/gleif.spec.ts diff --git a/apps/agent/agent/lib/gleif-config.ts b/apps/agent/agent/lib/gleif-config.ts new file mode 100644 index 000000000..448fbc253 --- /dev/null +++ b/apps/agent/agent/lib/gleif-config.ts @@ -0,0 +1,71 @@ +const SECOND_MS = 1_000; + +export const GLEIF = { + api: { + baseUrl: "https://api.gleif.org/api/v1", + timeoutMs: 20 * SECOND_MS, + pageSize: 100, + maxPages: 10, + }, + + search: { + defaultLimit: 10, + maxLimit: 50, + }, + + regions: { + UE: [ + "AT", + "BE", + "BG", + "CY", + "CZ", + "DE", + "DK", + "EE", + "ES", + "FI", + "FR", + "GR", + "HR", + "HU", + "IE", + "IT", + "LT", + "LU", + "LV", + "MT", + "NL", + "PL", + "PT", + "RO", + "SE", + "SI", + "SK", + ], + ASIE: [ + "BD", + "BN", + "CN", + "HK", + "ID", + "IN", + "JP", + "KH", + "KR", + "LA", + "LK", + "MM", + "MN", + "MO", + "MY", + "NP", + "PH", + "PK", + "SG", + "TH", + "TW", + "VN", + ], + }, +} as const; diff --git a/apps/agent/agent/lib/gleif.ts b/apps/agent/agent/lib/gleif.ts new file mode 100644 index 000000000..7fece22c8 --- /dev/null +++ b/apps/agent/agent/lib/gleif.ts @@ -0,0 +1,253 @@ +import { z } from "zod"; +import { GLEIF } from "./gleif-config"; + +export type Outcome = { ok: true; data: T } | { ok: false; reason: string }; + +const legalName = z.object({ name: z.string().trim().min(1) }); + +const legalAddress = z.object({ + country: z.string().trim().length(2), + city: z.string().trim().min(1).nullable().optional(), +}); + +const otherName = z.object({ + name: z.string().trim().min(1), + type: z.string().nullable().optional(), +}); + +const record = z.object({ + id: z.string().trim().length(20), + attributes: z.object({ + entity: z.object({ + legalName, + legalAddress, + otherNames: z.array(otherName).nullable().optional(), + transliteratedOtherNames: z.array(otherName).nullable().optional(), + status: z.string().nullable().optional(), + category: z.string().nullable().optional(), + jurisdiction: z.string().nullable().optional(), + }), + registration: z + .object({ status: z.string().nullable().optional() }) + .optional(), + }), +}); + +const pagination = z.object({ + currentPage: z.number().int(), + lastPage: z.number().int(), + total: z.number().int(), +}); + +const page = z.object({ + data: z.array(record), + meta: z.object({ pagination }).optional(), +}); + +const single = z.object({ data: record.nullable() }); + +export const gleifEntity = record.transform(({ id, attributes }) => ({ + lei: id, + name: attributes.entity.legalName.name, + alternativeNames: [ + ...new Set( + [ + ...(attributes.entity.otherNames ?? []), + ...(attributes.entity.transliteratedOtherNames ?? []), + ] + .map((other) => other.name) + .filter((name) => name !== attributes.entity.legalName.name), + ), + ], + country: attributes.entity.legalAddress.country.toUpperCase(), + city: attributes.entity.legalAddress.city ?? null, + status: attributes.entity.status ?? null, + registrationStatus: attributes.registration?.status ?? null, + category: attributes.entity.category ?? null, + jurisdiction: attributes.entity.jurisdiction ?? null, +})); + +export type GleifEntity = z.infer; + +export type Subsidiaries = { + parent: string; + children: GleifEntity[]; + total: number; + truncated: boolean; +}; + +export const ENTITY_CATEGORIES = [ + "GENERAL", + "FUND", + "BRANCH", + "SOLE_PROPRIETOR", + "RESIDENT_GOVERNMENT_ENTITY", + "INTERNATIONAL_ORGANIZATION", +] as const; + +export type EntityCategory = (typeof ENTITY_CATEGORIES)[number]; + +type Query = Record; + +export function resolveCountries(spec: string | undefined): string[] { + if (!spec) return []; + const regions: Record = GLEIF.regions; + return [ + ...new Set( + spec + .split(",") + .map((part) => part.trim().toUpperCase()) + .filter(Boolean) + .flatMap((part) => regions[part] ?? [part]) + .filter((code) => /^[A-Z]{2}$/.test(code)), + ), + ]; +} + +function normalizeLei(lei: string): string { + return encodeURIComponent(lei.trim().toUpperCase()); +} + +async function request( + path: string, + query: Query, + shape: Shape, + notFound: z.infer, +): Promise>> { + const url = new URL(`${GLEIF.api.baseUrl}${path}`); + for (const [key, value] of Object.entries(query)) { + if (value !== undefined) url.searchParams.set(key, value); + } + + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), GLEIF.api.timeoutMs); + + try { + const response = await fetch(url, { + headers: { accept: "application/vnd.api+json" }, + signal: controller.signal, + }); + + if (response.status === 404) return { ok: true, data: notFound }; + if (!response.ok) return { ok: false, reason: `HTTP ${response.status}` }; + + const parsed = shape.safeParse(await response.json()); + return parsed.success + ? { ok: true, data: parsed.data } + : { + ok: false, + reason: `Unreadable GLEIF response: ${parsed.error.message}`, + }; + } catch (error) { + const aborted = error instanceof Error && error.name === "AbortError"; + return { + ok: false, + reason: aborted + ? `GLEIF timed out after ${GLEIF.api.timeoutMs}ms.` + : error instanceof Error + ? error.message + : String(error), + }; + } finally { + clearTimeout(timer); + } +} + +export async function searchEntities(input: { + name: string; + country?: string; + category?: EntityCategory | "ANY"; + activeOnly?: boolean; + limit?: number; +}): Promise> { + const limit = Math.min( + input.limit ?? GLEIF.search.defaultLimit, + GLEIF.search.maxLimit, + ); + const category = input.category ?? "GENERAL"; + + const response = await request( + "/lei-records", + { + "filter[entity.legalName]": input.name.trim(), + "filter[entity.legalAddress.country]": input.country + ?.trim() + .toUpperCase(), + "filter[entity.status]": + (input.activeOnly ?? true) ? "ACTIVE" : undefined, + "filter[entity.category]": category === "ANY" ? undefined : category, + "page[size]": String(limit), + "page[number]": "1", + }, + page, + { data: [] }, + ); + if (!response.ok) return response; + + return { + ok: true, + data: { + entities: response.data.data.map((row) => gleifEntity.parse(row)), + total: response.data.meta?.pagination.total ?? response.data.data.length, + }, + }; +} + +async function one(path: string): Promise> { + const response = await request(path, {}, single, { data: null }); + if (!response.ok) return response; + + return { + ok: true, + data: response.data.data ? gleifEntity.parse(response.data.data) : null, + }; +} + +export function getEntity(lei: string): Promise> { + return one(`/lei-records/${normalizeLei(lei)}`); +} + +export function directParent( + lei: string, +): Promise> { + return one(`/lei-records/${normalizeLei(lei)}/direct-parent`); +} + +export async function directChildren( + lei: string, + options: { countries?: string[] } = {}, +): Promise> { + const parent = lei.trim().toUpperCase(); + const wanted = new Set(options.countries ?? []); + const children: GleifEntity[] = []; + let total = 0; + let truncated = false; + + for (let number = 1; number <= GLEIF.api.maxPages; number++) { + const response = await request( + `/lei-records/${normalizeLei(parent)}/direct-children`, + { + "page[size]": String(GLEIF.api.pageSize), + "page[number]": String(number), + }, + page, + { data: [] }, + ); + if (!response.ok) return response; + + const meta = response.data.meta?.pagination; + total = meta?.total ?? response.data.data.length; + + for (const row of response.data.data) { + const entity = gleifEntity.parse(row); + if (wanted.size === 0 || wanted.has(entity.country)) { + children.push(entity); + } + } + + if (!meta || number >= meta.lastPage) break; + if (number === GLEIF.api.maxPages) truncated = true; + } + + return { ok: true, data: { parent, children, total, truncated } }; +} diff --git a/apps/agent/agent/skills/gleif-mna-sourcing/SKILL.md b/apps/agent/agent/skills/gleif-mna-sourcing/SKILL.md new file mode 100644 index 000000000..8685c84f4 --- /dev/null +++ b/apps/agent/agent/skills/gleif-mna-sourcing/SKILL.md @@ -0,0 +1,79 @@ +--- +description: Use when asked to source M&A targets, list a group's subsidiaries by country, build a cross-border target list, or find the executives of those targets — the GLEIF register plus web research, under the legal rules that keep the list usable. +--- + +# GLEIF M&A sourcing + +The GLEIF register is the public list of legal entities that hold a Legal +Entity Identifier, with the parent–subsidiary relationships they report. It +is the deterministic half of sourcing: who owns what, where. The other half, +who runs each target, is web research and it is where the rules below apply. + +## 1. Fix the scenario before searching + +A scenario is a parent place and a child place: "US parents with subsidiaries +in Asia", "EU groups with a Mexican entity". Ask for both when the request +gives only one. `UE` and `ASIE` are known regions; anything else is ISO codes +separated by commas. + +## 2. Find the parents + +- A named group: `gleif_search_entities` with the name and its country, then + keep the ACTIVE entity that is the group head. `gleif_get_entity` on a + candidate shows its direct parent; a group head has none. +- A whole scenario with no names: ask the rep for a list of parents, or take + the companies already in the CRM (`search_crm`) that sit in the parent + place. GLEIF search is by name, not by country alone. + +## 3. List the targets + +For each parent, `gleif_list_subsidiaries` with `childCountries` set to the +child place. Every row returned is a target: LEI, legal name, country, city. +A local-language legal name usually comes with `alternativeNames`; use the +Latin one when you search the web for the entity, and show both in the list. + +- Rank parents by how many matching subsidiaries they have. More entities in + the child place means more to buy, and more to talk to. +- GLEIF relationships are self-reported. A group with zero children in the + register is not proof it has none. Say so rather than dropping it silently. +- `totalDirectChildren` is the parent's whole footprint; `matched` is the + slice in the child place. Report both. + +## 4. Find who runs each target + +This is not deterministic and it is where a list becomes unusable if you cut +corners. The rules: + +- **Never fetch linkedin.com.** A profile URL comes from a search engine + snippet, which is public. Use `research_person`, `find_contact_socials` and + `resolve_linkedin_profile`; they already respect this. +- **Never invent a LinkedIn URL.** If no snippet shows it, the answer is + "not found". A plausible namesake is worse than a blank. +- **One source per line.** Every executive you name carries the domain you + saw it on and the date. `record_fact` with `web.cited-claim` or + `search.cites-profile` is how that lands on a contact. +- **Subsidiaries rarely have a CEO.** Look for the local title: managing + director, country head, general manager, president director. Several + people per entity is normal. When the entity has no leader of its own, + give the group leader and say it is the group's. +- **Expect blanks.** Ten to twenty percent of group leaders and far more + local ones have no public profile. Do not fill the gap. + +Contact details (email, phone) come only from a provider that carries the +compliance of the source, never from a page you read. If none is configured, +stop at name, title and public profile. + +## 5. Deliver + +Write the list as a table: parent, LEI, target legal name, country, city, +leader, title, profile URL, source. Then the counts: parents scanned, +targets found, leaders found, leaders without a public profile. A target +already in the CRM (`search_crm` by name) is marked as such rather than +duplicated. + +## Quality checks before handing over + +- Every LEI has 20 characters and appears once. +- Every target belongs to the child place asked for. +- Every profile URL was seen in a snippet and matches the named person. +- Every blank is reported as a blank, not as a guess. diff --git a/apps/agent/agent/tools/gleif_get_entity.ts b/apps/agent/agent/tools/gleif_get_entity.ts new file mode 100644 index 000000000..b38d392ac --- /dev/null +++ b/apps/agent/agent/tools/gleif_get_entity.ts @@ -0,0 +1,33 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; +import { directParent, getEntity } from "../lib/gleif"; + +export default defineTool({ + description: + "Read one legal entity from the public GLEIF register by LEI, with its direct parent when one is registered. Free, no key. Use it to confirm a company's legal name, country and status, or to climb from a subsidiary to the group above it.", + inputSchema: z.object({ + lei: z.string().trim().length(20), + }), + async execute({ lei }) { + const [entity, parent] = await Promise.all([ + getEntity(lei), + directParent(lei), + ]); + + if (!entity.ok) return { found: false as const, reason: entity.reason }; + if (!entity.data) { + return { found: false as const, reason: "No entity holds this LEI." }; + } + + return { + found: true as const, + entity: entity.data, + directParent: parent.ok ? parent.data : null, + note: parent.ok + ? parent.data + ? undefined + : "No direct parent is registered. Either it is a group head, or the relationship was never reported." + : `The parent lookup failed: ${parent.reason}`, + }; + }, +}); diff --git a/apps/agent/agent/tools/gleif_list_subsidiaries.ts b/apps/agent/agent/tools/gleif_list_subsidiaries.ts new file mode 100644 index 000000000..43db4a4d8 --- /dev/null +++ b/apps/agent/agent/tools/gleif_list_subsidiaries.ts @@ -0,0 +1,48 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; +import { directChildren, resolveCountries } from "../lib/gleif"; + +export default defineTool({ + description: + "List the direct subsidiaries a legal entity consolidates, from the public GLEIF relationship register, optionally keeping only those in given countries or regions. Returns each subsidiary with its LEI, legal name, country and city. Free, no key. This is the M&A sourcing step: a parent in one country with subsidiaries in another is a cross-border target.", + inputSchema: z.object({ + lei: z + .string() + .trim() + .length(20) + .describe("The parent's LEI, from gleif_search_entities."), + childCountries: z + .string() + .trim() + .optional() + .describe( + "Keep subsidiaries in these places only. A region name ('UE', 'ASIE') or ISO codes separated by commas ('US,CA'). Empty keeps all.", + ), + }), + async execute({ lei, childCountries }) { + const countries = resolveCountries(childCountries); + const result = await directChildren(lei, { countries }); + + if (!result.ok) { + return { + parent: lei, + subsidiaries: [], + matched: 0, + reason: result.reason, + }; + } + + return { + parent: result.data.parent, + countries, + totalDirectChildren: result.data.total, + matched: result.data.children.length, + subsidiaries: result.data.children, + note: result.data.truncated + ? "The parent has more direct subsidiaries than were read. The counts are a floor." + : result.data.total === 0 + ? "GLEIF records no direct subsidiary for this entity. Relationships are self-reported, so a group can exist without any." + : undefined, + }; + }, +}); diff --git a/apps/agent/agent/tools/gleif_search_entities.ts b/apps/agent/agent/tools/gleif_search_entities.ts new file mode 100644 index 000000000..4f6ecba17 --- /dev/null +++ b/apps/agent/agent/tools/gleif_search_entities.ts @@ -0,0 +1,54 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; +import { ENTITY_CATEGORIES, searchEntities } from "../lib/gleif"; +import { GLEIF } from "../lib/gleif-config"; + +export default defineTool({ + description: + "Find legal entities in the public GLEIF register by name, optionally within one country. Returns each match with its LEI, legal name, country, city and status. Free, no key. Use it to identify a group parent before listing its subsidiaries with gleif_list_subsidiaries, or to check a company's legal identity.", + inputSchema: z.object({ + name: z + .string() + .trim() + .min(2) + .describe("Part of the legal name. 'Renault', 'Siemens Energy'."), + country: z + .string() + .trim() + .length(2) + .optional() + .describe("ISO 3166-1 alpha-2 code of the legal address. 'FR', 'US'."), + category: z + .enum([...ENTITY_CATEGORIES, "ANY"]) + .default("GENERAL") + .describe( + "GLEIF entity category. GENERAL is an operating company or holding; FUND, BRANCH and the others are rarely M&A targets. ANY removes the filter.", + ), + activeOnly: z + .boolean() + .default(true) + .describe("Only entities whose GLEIF status is ACTIVE."), + limit: z + .number() + .int() + .min(1) + .max(GLEIF.search.maxLimit) + .default(GLEIF.search.defaultLimit), + }), + async execute(input) { + const result = await searchEntities(input); + + if (!result.ok) return { found: 0, entities: [], reason: result.reason }; + + return { + found: result.data.total, + entities: result.data.entities, + note: + result.data.total === 0 + ? "Nothing in GLEIF matches. Only entities that hold an LEI are listed; try a shorter name or drop the country." + : result.data.total > result.data.entities.length + ? `Showing ${result.data.entities.length} of ${result.data.total}. Narrow the name or add a country.` + : undefined, + }; + }, +}); diff --git a/apps/agent/test/fixtures/gleif.json b/apps/agent/test/fixtures/gleif.json new file mode 100644 index 000000000..0d2e32695 --- /dev/null +++ b/apps/agent/test/fixtures/gleif.json @@ -0,0 +1,130 @@ +{ + "search": { + "data": [ + { + "id": "969500F7JLTX36OUI695", + "attributes": { + "entity": { + "legalName": { + "name": "RENAULT" + }, + "legalAddress": { + "country": "FR", + "city": "BOULOGNE-BILLANCOURT" + }, + "status": "ACTIVE", + "category": "GENERAL", + "jurisdiction": "FR" + }, + "registration": { + "status": "ISSUED" + } + } + }, + { + "id": "969500HC1NCZMYE1TU11", + "attributes": { + "entity": { + "legalName": { + "name": "RENAULT INVEST" + }, + "legalAddress": { + "country": "FR", + "city": "SURESNES" + }, + "status": "ACTIVE", + "category": "GENERAL", + "jurisdiction": "FR" + }, + "registration": { + "status": "ISSUED" + } + } + } + ], + "meta": { + "pagination": { + "currentPage": 1, + "perPage": 3, + "from": 1, + "to": 3, + "total": 22, + "lastPage": 8 + } + } + }, + "children": { + "data": [ + { + "id": "549300WSRTFGXLOGXV17", + "attributes": { + "entity": { + "legalName": { + "name": "AUTOFIN" + }, + "legalAddress": { + "country": "BE", + "city": "BRUSSELS" + }, + "status": "ACTIVE", + "category": "GENERAL", + "jurisdiction": "BE" + }, + "registration": { + "status": "LAPSED" + } + } + }, + { + "id": "5493004RF65W82IYO343", + "attributes": { + "entity": { + "legalName": { + "name": "RCI FINANCIAL SERVICES" + }, + "legalAddress": { + "country": "BE", + "city": "BRUSSELS" + }, + "status": "ACTIVE", + "category": "GENERAL", + "jurisdiction": "BE" + }, + "registration": { + "status": "LAPSED" + } + } + }, + { + "id": "969500V06Q2Q3ELCWD59", + "attributes": { + "entity": { + "legalName": { + "name": "RENAULT SAS" + }, + "legalAddress": { + "country": "FR", + "city": "BOULOGNE-BILLANCOURT" + }, + "status": "ACTIVE", + "category": "GENERAL", + "jurisdiction": "FR" + }, + "registration": { + "status": "ISSUED" + } + } + } + ], + "meta": { + "pagination": { + "currentPage": 1, + "perPage": 5, + "from": 1, + "to": 3, + "total": 3, + "lastPage": 1 + } + } + } +} diff --git a/apps/agent/test/gleif.spec.ts b/apps/agent/test/gleif.spec.ts new file mode 100644 index 000000000..9bd92581c --- /dev/null +++ b/apps/agent/test/gleif.spec.ts @@ -0,0 +1,218 @@ +import { afterEach, describe, expect, it } from "bun:test"; +import { + directChildren, + directParent, + getEntity, + resolveCountries, + searchEntities, +} from "../agent/lib/gleif"; +import { GLEIF } from "../agent/lib/gleif-config"; +import fixtures from "./fixtures/gleif.json"; + +const realFetch = globalThis.fetch; + +const requested: string[] = []; + +function replies(reply: (url: URL) => { status?: number; body: unknown }) { + globalThis.fetch = (async (input: URL | RequestInfo) => { + const url = new URL(String(input instanceof Request ? input.url : input)); + requested.push(url.toString()); + const { status, body } = reply(url); + return new Response(JSON.stringify(body), { + status: status ?? 200, + headers: { "content-type": "application/vnd.api+json" }, + }); + }) as typeof fetch; +} + +afterEach(() => { + globalThis.fetch = realFetch; + requested.length = 0; +}); + +describe("resolveCountries", () => { + it("expands a region name", () => { + expect(resolveCountries("UE")).toEqual([...GLEIF.regions.UE]); + }); + + it("accepts ISO codes, mixed case, and drops junk", () => { + expect(resolveCountries(" us, ca ,Asie,xyz")).toEqual([ + "US", + "CA", + ...GLEIF.regions.ASIE, + ]); + }); + + it("is empty for nothing", () => { + expect(resolveCountries(undefined)).toEqual([]); + expect(resolveCountries("")).toEqual([]); + }); +}); + +describe("searchEntities", () => { + it("parses a search page into entities and sends the filters", async () => { + replies(() => ({ body: fixtures.search })); + + const result = await searchEntities({ name: "Renault", country: "fr" }); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.data.total).toBe(22); + expect(result.data.entities[0]).toEqual({ + lei: "969500F7JLTX36OUI695", + name: "RENAULT", + alternativeNames: [], + country: "FR", + city: "BOULOGNE-BILLANCOURT", + status: "ACTIVE", + registrationStatus: "ISSUED", + category: "GENERAL", + jurisdiction: "FR", + }); + + const url = new URL(requested[0] ?? ""); + expect(url.searchParams.get("filter[entity.legalName]")).toBe("Renault"); + expect(url.searchParams.get("filter[entity.legalAddress.country]")).toBe( + "FR", + ); + expect(url.searchParams.get("filter[entity.status]")).toBe("ACTIVE"); + expect(url.searchParams.get("filter[entity.category]")).toBe("GENERAL"); + }); + + it("drops the category filter on ANY", async () => { + replies(() => ({ body: fixtures.search })); + + await searchEntities({ name: "Renault", category: "ANY" }); + + const url = new URL(requested[0] ?? ""); + expect(url.searchParams.has("filter[entity.category]")).toBe(false); + }); + + it("reports an HTTP failure as a reason, never a throw", async () => { + replies(() => ({ status: 503, body: {} })); + + const result = await searchEntities({ name: "Renault" }); + + expect(result).toEqual({ ok: false, reason: "HTTP 503" }); + }); + + it("refuses a response that is not the GLEIF shape", async () => { + replies(() => ({ body: { data: [{ id: "short" }] } })); + + const result = await searchEntities({ name: "Renault" }); + + expect(result.ok).toBe(false); + }); +}); + +describe("directChildren", () => { + it("keeps only the wanted countries and reports the whole footprint", async () => { + replies(() => ({ body: fixtures.children })); + + const result = await directChildren("969500F7JLTX36OUI695", { + countries: ["BE"], + }); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.data.total).toBe(3); + expect(result.data.children.map((child) => child.name)).toEqual([ + "AUTOFIN", + "RCI FINANCIAL SERVICES", + ]); + expect(result.data.truncated).toBe(false); + }); + + it("walks every page and flags a cut-off at the page cap", async () => { + const child = fixtures.children.data[0]; + replies((url) => { + const number = Number(url.searchParams.get("page[number]")); + return { + body: { + data: [{ ...child, id: `${number}`.padStart(20, "0") }], + meta: { + pagination: { + currentPage: number, + lastPage: GLEIF.api.maxPages + 1, + total: GLEIF.api.maxPages + 1, + }, + }, + }, + }; + }); + + const result = await directChildren("969500F7JLTX36OUI695"); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(requested).toHaveLength(GLEIF.api.maxPages); + expect(result.data.children).toHaveLength(GLEIF.api.maxPages); + expect(result.data.truncated).toBe(true); + }); +}); + +describe("getEntity and directParent", () => { + it("returns null for a 404 instead of failing", async () => { + replies(() => ({ status: 404, body: { errors: [] } })); + + expect(await getEntity("969500F7JLTX36OUI695")).toEqual({ + ok: true, + data: null, + }); + expect(await directParent("969500F7JLTX36OUI695")).toEqual({ + ok: true, + data: null, + }); + }); + + it("collects the other names a local-language entity carries", async () => { + const local = fixtures.search.data[1]; + replies(() => ({ + body: { + data: { + ...local, + attributes: { + ...local.attributes, + entity: { + ...local.attributes.entity, + legalName: { name: "ドットマティクス株式会社" }, + otherNames: [ + { + type: "ALTERNATIVE_LANGUAGE_LEGAL_NAME", + name: "Dotmatics K.K.", + }, + { + type: "PREVIOUS_LEGAL_NAME", + name: "ドットマティクス株式会社", + }, + ], + transliteratedOtherNames: [ + { + type: "AUTO_ASCII_TRANSLITERATED_LEGAL_NAME", + name: "Dotmatics K.K.", + }, + ], + }, + }, + }, + }, + })); + + const result = await getEntity("969500HC1NCZMYE1TU11"); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.data?.alternativeNames).toEqual(["Dotmatics K.K."]); + }); + + it("parses a single record", async () => { + replies(() => ({ body: { data: fixtures.search.data[1] } })); + + const result = await getEntity("969500hc1nczmye1tu11"); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.data?.name).toBe("RENAULT INVEST"); + expect(requested[0]).toContain("/lei-records/969500HC1NCZMYE1TU11"); + }); +}); diff --git a/docs/agent.md b/docs/agent.md index e1fc98353..ca6b18c75 100644 --- a/docs/agent.md +++ b/docs/agent.md @@ -219,6 +219,19 @@ missing key removes a place to look. **Never an error, never throws.** `capabilitiesFrom()`/`markdownFor()` are the pure halves. `contextDevKey()` is the only resolver, and `lib/context-dev.ts` memoises its client on the key string. +### The GLEIF register needs no key + +`lib/gleif.ts` reads the public GLEIF API — legal entities by name, one entity by +LEI, the direct subsidiaries an entity consolidates — and is always on. The three +`gleif_*` tools are free: no budget is charged. Every response is parsed with Zod +at the boundary into `GleifEntity`; a shape the register does not promise is a +failed outcome with a reason, never a throw. Region names (`UE`, `ASIE`) and the +page cap live in `lib/gleif-config.ts`. The `gleif-mna-sourcing` skill is the +method: a parent place and a child place make a scenario, subsidiaries in the +child place are the targets, and the people who run them come from web research +under the same egress rules as everything else — never a LinkedIn fetch, never an +invented URL, one source per line. + ## Budget and scheduling - `lib/focus.ts` — per-session budget in `defineState`; running out is a normal ending. From 4917357476a5ba88eb98b51c02da842f874a22e9 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 3 Sep 2026 04:11:55 +0000 Subject: [PATCH 09/19] fix(telemetry): allow the gleif tool names The tool allowlist mirrors apps/agent/agent/tools, and its test checks that every file is listed. Claude-Session: https://claude.ai/code/session_011fuk1HeytBt9RU12xKY8WB --- packages/telemetry/src/allowlist.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/telemetry/src/allowlist.ts b/packages/telemetry/src/allowlist.ts index 5880dc2bd..f944d9de4 100644 --- a/packages/telemetry/src/allowlist.ts +++ b/packages/telemetry/src/allowlist.ts @@ -121,6 +121,9 @@ export const AGENT_TOOLS = [ "find_contact_socials", "get_contact_work_history", "get_linkedin_profile", + "gleif_get_entity", + "gleif_list_subsidiaries", + "gleif_search_entities", "identify_contact", "list_deals", "list_fields", From 0e0d4cc35530633ae38e671f5dcebdcdd39ef2b2 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 3 Sep 2026 04:13:38 +0000 Subject: [PATCH 10/19] fix(app): narrate the gleif tools in the agent transcript TOOL_VERBS mirrors apps/agent/agent/tools, and its test checks that every tool has a sentence. Claude-Session: https://claude.ai/code/session_011fuk1HeytBt9RU12xKY8WB --- apps/app/lib/agent-transcript.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apps/app/lib/agent-transcript.ts b/apps/app/lib/agent-transcript.ts index 2898c35bb..189d0ca36 100644 --- a/apps/app/lib/agent-transcript.ts +++ b/apps/app/lib/agent-transcript.ts @@ -63,6 +63,9 @@ const VERBS: ToolVerbs = { search_crm: "Looked the record up in the CRM", resolve_linkedin_profile: "Searched for their LinkedIn profile", get_linkedin_profile: "Read a LinkedIn profile", + gleif_search_entities: "Searched the GLEIF register for a company", + gleif_get_entity: "Read a company's GLEIF record", + gleif_list_subsidiaries: "Listed a group's subsidiaries from GLEIF", get_contact_work_history: "Read their work history", fetch_contact_photo: "Fetched their profile picture", find_contact_socials: "Searched for their other profiles", From 3063dd131de5622ca218dbf360fac3da614cd81a Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 3 Sep 2026 05:19:22 +0000 Subject: [PATCH 11/19] feat(agent): add companies with their source, and look up work email add_company is the agent's one company write path: dedupe by domain then by name and country, the row, the company.created event task the API would have written, a timeline note naming the source with a link, the LEI as a company field, and the brand and profile enrichment queued. find_work_email asks Hunter, under an optional HUNTER_API_KEY, for a contact's address with the public pages it was seen on, fills the email only when the record has none, and writes the candidate, score and sources to the timeline. The gleif-mna-sourcing skill uses both. Claude-Session: https://claude.ai/code/session_011fuk1HeytBt9RU12xKY8WB --- .env.example | 6 + apps/agent/agent/lib/capabilities.ts | 6 + apps/agent/agent/lib/companies.ts | 191 ++++++++++++++++++ apps/agent/agent/lib/dispatch-config.ts | 5 + apps/agent/agent/lib/hunter-config.ts | 10 + apps/agent/agent/lib/hunter.ts | 143 +++++++++++++ .../agent/skills/gleif-mna-sourcing/SKILL.md | 19 +- apps/agent/agent/tools/add_company.ts | 57 ++++++ apps/agent/agent/tools/find_work_email.ts | 125 ++++++++++++ apps/agent/test/capabilities.spec.ts | 6 +- apps/agent/test/companies.integration.spec.ts | 137 +++++++++++++ apps/agent/test/hunter.spec.ts | 144 +++++++++++++ apps/agent/turbo.json | 2 + apps/app/lib/agent-transcript.ts | 2 + docs/agent.md | 17 ++ docs/environment.md | 1 + packages/telemetry/src/allowlist.ts | 2 + turbo.json | 1 + 18 files changed, 867 insertions(+), 7 deletions(-) create mode 100644 apps/agent/agent/lib/companies.ts create mode 100644 apps/agent/agent/lib/hunter-config.ts create mode 100644 apps/agent/agent/lib/hunter.ts create mode 100644 apps/agent/agent/tools/add_company.ts create mode 100644 apps/agent/agent/tools/find_work_email.ts create mode 100644 apps/agent/test/companies.integration.spec.ts create mode 100644 apps/agent/test/hunter.spec.ts diff --git a/.env.example b/.env.example index 34874a019..788aa4162 100644 --- a/.env.example +++ b/.env.example @@ -131,6 +131,12 @@ GOOGLE_CLIENT_SECRET="" # knowing before a call. https://perplexity.ai/settings/api # PERPLEXITY_API_KEY="" +# Hunter — finds a person's work email from their name and their employer's +# domain, with the public pages it was seen on, and checks whether an address +# is deliverable. The free plan covers a few dozen lookups a month. +# https://hunter.io/api-keys +# HUNTER_API_KEY="" + # GitHub — raises the rate limit when matching contacts to GitHub profiles. # Any classic token with no scopes will do. # GITHUB_TOKEN="" diff --git a/apps/agent/agent/lib/capabilities.ts b/apps/agent/agent/lib/capabilities.ts index 2e587e955..d63696899 100644 --- a/apps/agent/agent/lib/capabilities.ts +++ b/apps/agent/agent/lib/capabilities.ts @@ -66,6 +66,12 @@ export function capabilitiesFrom( "a person read back from a LinkedIn URL you already hold — their real name, bio, current title and employer, every earlier role with its dates, their education and their other public profiles, all self-reported and so authoritative on identity", enabled: contextDev !== null, }, + { + ...fromEnv("HUNTER_API_KEY"), + label: "Work email lookup", + gives: + "a person's work email address from their name and their employer's domain, with the public pages Hunter saw it on, and a deliverability check on an address you already hold", + }, { ...fromEnv("BLOB_READ_WRITE_TOKEN"), label: "Picture storage", diff --git a/apps/agent/agent/lib/companies.ts b/apps/agent/agent/lib/companies.ts new file mode 100644 index 000000000..9aa37f8b5 --- /dev/null +++ b/apps/agent/agent/lib/companies.ts @@ -0,0 +1,191 @@ +import { ActivityType, db, type Prisma } from "@crm/db"; +import { PRIORITY } from "@crm/db/agent-tasks"; +import { DISPATCH } from "./dispatch-config"; +import { createField, listFields, writeField } from "./fields"; +import { hostOf } from "./names"; +import { scheduleTask } from "./tasks"; + +export const LEI_FIELD_LABEL = "LEI"; + +export type CompanySource = { label: string; url: string }; + +export type NewCompany = { + name: string; + website?: string | null; + countryCode?: string | null; + country?: string | null; + city?: string | null; + lei?: string | null; + source: CompanySource; +}; + +export type CreatedCompany = { + created: boolean; + id: string; + name: string; + domain: string | null; + reason?: string; +}; + +function domainFrom(website: string | null | undefined): string | null { + if (!website) return null; + const host = hostOf(website); + return host.includes(".") ? host : null; +} + +async function existingCompany( + name: string, + domain: string | null, + countryCode: string | null, +): Promise<{ id: string; name: string; domain: string | null } | null> { + const select = { id: true, name: true, domain: true }; + + if (domain) { + const byDomain = await db.company.findFirst({ + where: { domain, archivedAt: null }, + select, + }); + if (byDomain) return byDomain; + } + + const where: Prisma.CompanyWhereInput = { + name: { equals: name, mode: "insensitive" }, + archivedAt: null, + }; + if (countryCode) where.countryCode = countryCode; + + return db.company.findFirst({ where, select }); +} + +async function authorId(): Promise { + const user = await db.user.findFirst({ + orderBy: { createdAt: "asc" }, + select: { id: true }, + }); + return user?.id ?? null; +} + +async function recordLei(companyId: string, lei: string): Promise { + const fields = await listFields("COMPANY"); + const existing = fields.find( + (field) => field.label.toUpperCase() === LEI_FIELD_LABEL, + ); + const key = existing + ? existing.key + : await createField({ + entity: "COMPANY", + label: LEI_FIELD_LABEL, + type: "TEXT", + agentBrief: + "The 20-character Legal Entity Identifier from the GLEIF register.", + }).then((field) => ("created" in field ? null : field.key)); + + if (key) + await writeField({ + entity: "COMPANY", + recordId: companyId, + key, + value: lei, + }); +} + +export async function createCompany( + input: NewCompany, +): Promise { + const name = input.name.trim(); + const domain = domainFrom(input.website); + const countryCode = input.countryCode?.trim().toUpperCase() || null; + + const existing = await existingCompany(name, domain, countryCode); + if (existing) { + return { + created: false, + ...existing, + reason: domain + ? `${existing.name} already uses the domain ${domain}.` + : `${existing.name} is already in the CRM.`, + }; + } + + const occurredAt = new Date(); + const created = await db.$transaction(async (tx) => { + const company = await tx.company.create({ + data: { + name, + domain, + website: domain ? `https://${domain}` : null, + countryCode, + country: input.country?.trim() || null, + city: input.city?.trim() || null, + }, + select: { id: true, name: true, domain: true }, + }); + + const payload: Prisma.InputJsonObject = { + type: "company.created", + record: { kind: "company", id: company.id }, + occurredAt: occurredAt.toISOString(), + data: { name: company.name, domain: company.domain }, + }; + await tx.agentTask.create({ + data: { + companyId: company.id, + kind: "agent-event", + reason: "company.created", + payload, + priority: PRIORITY.event, + budget: 1, + dueAt: occurredAt, + }, + }); + + return company; + }); + + const author = await authorId(); + if (author) { + await db.activity.create({ + data: { + type: ActivityType.ENRICHMENT, + subject: `Added from ${input.source.label}`, + body: [ + `${created.name} was added by the agent from ${input.source.label}.`, + input.lei ? `LEI ${input.lei.trim().toUpperCase()}.` : null, + `Source: ${input.source.url}`, + ] + .filter(Boolean) + .join(" "), + occurredAt, + companyId: created.id, + createdById: author, + meta: { + source: input.source.label, + sourceUrl: input.source.url, + agent: "sourcing", + }, + }, + select: { id: true }, + }); + } + + if (input.lei) await recordLei(created.id, input.lei.trim().toUpperCase()); + + await scheduleTask({ + companyId: created.id, + kind: "brand", + reason: "New company", + dueAt: occurredAt, + priority: PRIORITY.brand, + budget: DISPATCH.newCompany.brandBudget, + }); + await scheduleTask({ + companyId: created.id, + kind: "company-profile", + reason: "New company", + dueAt: occurredAt, + priority: PRIORITY.companyProfile, + budget: DISPATCH.newCompany.profileBudget, + }); + + return { created: true, ...created }; +} diff --git a/apps/agent/agent/lib/dispatch-config.ts b/apps/agent/agent/lib/dispatch-config.ts index 4382b4a80..957df9944 100644 --- a/apps/agent/agent/lib/dispatch-config.ts +++ b/apps/agent/agent/lib/dispatch-config.ts @@ -32,6 +32,11 @@ export const DISPATCH = { leaseMs: 10 * MINUTE_MS, }, + newCompany: { + brandBudget: 2, + profileBudget: 4, + }, + reconcile: { scan: 200, retire: 100, diff --git a/apps/agent/agent/lib/hunter-config.ts b/apps/agent/agent/lib/hunter-config.ts new file mode 100644 index 000000000..865568172 --- /dev/null +++ b/apps/agent/agent/lib/hunter-config.ts @@ -0,0 +1,10 @@ +const SECOND_MS = 1_000; + +export const HUNTER = { + api: { + baseUrl: "https://api.hunter.io/v2", + timeoutMs: 20 * SECOND_MS, + }, + minScore: 50, + maxSources: 5, +} as const; diff --git a/apps/agent/agent/lib/hunter.ts b/apps/agent/agent/lib/hunter.ts new file mode 100644 index 000000000..22af6db5f --- /dev/null +++ b/apps/agent/agent/lib/hunter.ts @@ -0,0 +1,143 @@ +import { z } from "zod"; +import { HUNTER } from "./hunter-config"; + +export type Outcome = { ok: true; data: T } | { ok: false; reason: string }; + +export const HUNTER_API_KEY = "HUNTER_API_KEY"; + +const source = z.object({ + uri: z.string().trim().min(1), + domain: z.string().trim().min(1).nullable().optional(), + extracted_on: z.string().trim().min(1).nullable().optional(), +}); + +const finder = z.object({ + data: z.object({ + email: z.string().trim().email().nullable(), + score: z.number().nullable().optional(), + first_name: z.string().nullable().optional(), + last_name: z.string().nullable().optional(), + position: z.string().nullable().optional(), + sources: z.array(source).nullable().optional(), + }), +}); + +const verifier = z.object({ + data: z.object({ + status: z.string().trim().min(1), + score: z.number().nullable().optional(), + }), +}); + +export type EmailSource = { + url: string; + domain: string | null; + seenOn: string | null; +}; + +export type WorkEmail = { + email: string | null; + score: number; + position: string | null; + sources: EmailSource[]; +}; + +export type Verification = { status: string; score: number | null }; + +export function hunterEnabled(): boolean { + return Boolean(process.env[HUNTER_API_KEY]?.trim()); +} + +async function request( + path: string, + query: Record, + shape: Shape, +): Promise>> { + const apiKey = process.env[HUNTER_API_KEY]?.trim(); + if (!apiKey) return { ok: false, reason: `No ${HUNTER_API_KEY}.` }; + + const url = new URL(`${HUNTER.api.baseUrl}${path}`); + for (const [key, value] of Object.entries(query)) { + if (value !== undefined) url.searchParams.set(key, value); + } + url.searchParams.set("api_key", apiKey); + + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), HUNTER.api.timeoutMs); + + try { + const response = await fetch(url, { signal: controller.signal }); + if (!response.ok) return { ok: false, reason: `HTTP ${response.status}` }; + + const parsed = shape.safeParse(await response.json()); + return parsed.success + ? { ok: true, data: parsed.data } + : { + ok: false, + reason: `Unreadable Hunter response: ${parsed.error.message}`, + }; + } catch (error) { + const aborted = error instanceof Error && error.name === "AbortError"; + return { + ok: false, + reason: aborted + ? `Hunter timed out after ${HUNTER.api.timeoutMs}ms.` + : error instanceof Error + ? error.message + : String(error), + }; + } finally { + clearTimeout(timer); + } +} + +export async function findWorkEmail(input: { + firstName: string; + lastName: string; + domain: string; +}): Promise> { + const response = await request( + "/email-finder", + { + domain: input.domain.trim().toLowerCase(), + first_name: input.firstName.trim(), + last_name: input.lastName.trim(), + }, + finder, + ); + if (!response.ok) return response; + + const { data } = response.data; + return { + ok: true, + data: { + email: data.email, + score: data.score ?? 0, + position: data.position ?? null, + sources: (data.sources ?? []).slice(0, HUNTER.maxSources).map((s) => ({ + url: s.uri, + domain: s.domain ?? null, + seenOn: s.extracted_on ?? null, + })), + }, + }; +} + +export async function verifyEmail( + email: string, +): Promise> { + const response = await request( + "/email-verifier", + { email: email.trim().toLowerCase() }, + verifier, + ); + if (!response.ok) return response; + + return { + ok: true, + data: { + status: response.data.data.status, + score: response.data.data.score ?? null, + }, + }; +} diff --git a/apps/agent/agent/skills/gleif-mna-sourcing/SKILL.md b/apps/agent/agent/skills/gleif-mna-sourcing/SKILL.md index 8685c84f4..2ed5b8454 100644 --- a/apps/agent/agent/skills/gleif-mna-sourcing/SKILL.md +++ b/apps/agent/agent/skills/gleif-mna-sourcing/SKILL.md @@ -59,17 +59,24 @@ corners. The rules: - **Expect blanks.** Ten to twenty percent of group leaders and far more local ones have no public profile. Do not fill the gap. -Contact details (email, phone) come only from a provider that carries the -compliance of the source, never from a page you read. If none is configured, -stop at name, title and public profile. +Contact details come only from a provider that carries the compliance of the +source, never from a page you read. `find_work_email` is that provider when +`HUNTER_API_KEY` is set: it returns the address with the public pages it was +seen on and writes them to the contact's timeline. Without it, stop at name, +title and public profile. ## 5. Deliver Write the list as a table: parent, LEI, target legal name, country, city, leader, title, profile URL, source. Then the counts: parents scanned, -targets found, leaders found, leaders without a public profile. A target -already in the CRM (`search_crm` by name) is marked as such rather than -duplicated. +targets found, leaders found, leaders without a public profile. + +When the rep wants targets in the CRM, `add_company` each one with its LEI +and the GLEIF record as the source (`https://search.gleif.org/#/record/`). +The tool returns the existing company when there is one, so a target already +in the CRM is never duplicated. The company's brand and profile enrichment +queue up on their own; the leaders you found go on as contacts through the +usual `record_fact` path, with their sources. ## Quality checks before handing over diff --git a/apps/agent/agent/tools/add_company.ts b/apps/agent/agent/tools/add_company.ts new file mode 100644 index 000000000..62e8ec197 --- /dev/null +++ b/apps/agent/agent/tools/add_company.ts @@ -0,0 +1,57 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; +import { createCompany } from "../lib/companies"; + +export default defineTool({ + description: + "Add a company to the CRM with the source it came from. Use it for a sourcing target or any company a rep asks for that search_crm cannot find. A company that already exists, by domain or by name in the same country, is returned rather than duplicated. The source is written to the company's timeline, a company.created event fires, and the brand and profile enrichment queue up on their own. Free.", + inputSchema: z.object({ + name: z + .string() + .trim() + .min(1) + .max(200) + .describe("The legal or trading name."), + website: z + .string() + .trim() + .optional() + .describe("Their site or domain when you know it. 'siemens.com'."), + countryCode: z + .string() + .trim() + .length(2) + .optional() + .describe("ISO 3166-1 alpha-2 code. 'JP'."), + country: z + .string() + .trim() + .optional() + .describe("Country name, for display."), + city: z.string().trim().optional(), + lei: z + .string() + .trim() + .length(20) + .optional() + .describe("The Legal Entity Identifier, when it came from GLEIF."), + source: z + .object({ + label: z + .string() + .trim() + .min(1) + .max(80) + .describe("Where this came from. 'GLEIF register', 'their website'."), + url: z + .string() + .trim() + .url() + .describe("The page a rep can open to check."), + }) + .describe("Every record the agent creates names its source."), + }), + async execute(input) { + return createCompany(input); + }, +}); diff --git a/apps/agent/agent/tools/find_work_email.ts b/apps/agent/agent/tools/find_work_email.ts new file mode 100644 index 000000000..adfe45b2a --- /dev/null +++ b/apps/agent/agent/tools/find_work_email.ts @@ -0,0 +1,125 @@ +import { ActivityType, db } from "@crm/db"; +import { defineTool } from "eve/tools"; +import { z } from "zod"; +import { unavailable } from "../lib/capabilities"; +import { focusOn, spend } from "../lib/focus"; +import { findWorkEmail, HUNTER_API_KEY, hunterEnabled } from "../lib/hunter"; +import { HUNTER } from "../lib/hunter-config"; +import { domainOf } from "../lib/names"; + +export default defineTool({ + description: + "Find a contact's work email address from their name and their employer's domain, through Hunter, with the public pages the address was seen on. Writes the candidate and its sources to the contact's timeline, and fills the email in when the record has none. Needs HUNTER_API_KEY; without it the tool says so and nothing is charged.", + inputSchema: z.object({ + contactId: z.string(), + }), + async execute({ contactId }) { + if (!hunterEnabled()) return unavailable(HUNTER_API_KEY); + + focusOn({ contactId }); + + const contact = await db.contact.findUnique({ + where: { id: contactId }, + select: { + id: true, + firstName: true, + lastName: true, + email: true, + company: { + select: { id: true, name: true, domain: true, ownerId: true }, + }, + }, + }); + + if (!contact) return { ok: false as const, reason: "No such contact." }; + if (!contact.lastName) { + return { + ok: false as const, + reason: "The contact has no last name yet. Identify them first.", + }; + } + + const domain = + contact.company?.domain ?? + (contact.email ? domainOf(contact.email) : null); + if (!domain) { + return { + ok: false as const, + reason: + "No employer domain to search. Set the company's website first.", + }; + } + + const charge = spend(1); + if (!charge.ok) return { ok: false as const, reason: charge.reason }; + + const result = await findWorkEmail({ + firstName: contact.firstName, + lastName: contact.lastName, + domain, + }); + if (!result.ok) return { ok: false as const, reason: result.reason }; + + const found = result.data; + if (!found.email || found.score < HUNTER.minScore) { + return { + ok: true as const, + email: null, + score: found.score, + note: "Hunter has no address it trusts for this person. Leave the email blank rather than guessing a pattern.", + }; + } + + const author = + contact.company?.ownerId ?? + (await db.user.findFirst({ select: { id: true } }))?.id ?? + null; + + const filled = !contact.email; + if (filled) { + await db.contact.update({ + where: { id: contact.id }, + data: { email: found.email }, + }); + } + + if (author) { + await db.activity.create({ + data: { + type: ActivityType.ENRICHMENT, + subject: filled ? "Work email found" : "Work email candidate", + body: [ + `${found.email} (Hunter score ${found.score}).`, + found.sources.length > 0 + ? `Seen on: ${found.sources.map((s) => s.url).join(", ")}` + : "Hunter gave no public page for it.", + filled ? null : `The record keeps ${contact.email}.`, + ] + .filter(Boolean) + .join(" "), + occurredAt: new Date(), + contactId: contact.id, + companyId: contact.company?.id ?? null, + createdById: author, + meta: { + source: "hunter.io", + endpoint: "email-finder", + score: found.score, + sources: found.sources.map((s) => s.url), + agent: "people-research", + }, + }, + select: { id: true }, + }); + } + + return { + ok: true as const, + email: found.email, + score: found.score, + position: found.position, + sources: found.sources, + filled, + }; + }, +}); diff --git a/apps/agent/test/capabilities.spec.ts b/apps/agent/test/capabilities.spec.ts index db3dd6cf1..3ef07c423 100644 --- a/apps/agent/test/capabilities.spec.ts +++ b/apps/agent/test/capabilities.spec.ts @@ -9,7 +9,11 @@ import { unavailable, } from "../agent/lib/capabilities"; -const KEYS = ["PERPLEXITY_API_KEY", "BLOB_READ_WRITE_TOKEN"] as const; +const KEYS = [ + "PERPLEXITY_API_KEY", + "HUNTER_API_KEY", + "BLOB_READ_WRITE_TOKEN", +] as const; const saved: Record = {}; diff --git a/apps/agent/test/companies.integration.spec.ts b/apps/agent/test/companies.integration.spec.ts new file mode 100644 index 000000000..c44d09966 --- /dev/null +++ b/apps/agent/test/companies.integration.spec.ts @@ -0,0 +1,137 @@ +import { afterEach, beforeEach, describe, expect, it } from "bun:test"; +import { db } from "@crm/db"; +import { createCompany, LEI_FIELD_LABEL } from "../agent/lib/companies"; + +const SUFFIX = "companies-spec"; +const SOURCE = { + label: "GLEIF register", + url: "https://search.gleif.org/#/record/W38RGI023J3WT1HWRP32", +}; + +const USER_ID = "companies-spec-user"; + +async function clear() { + const companies = await db.company.findMany({ + where: { name: { contains: SUFFIX } }, + select: { id: true }, + }); + const ids = companies.map((company) => company.id); + await db.activity.deleteMany({ where: { companyId: { in: ids } } }); + await db.agentTask.deleteMany({ where: { companyId: { in: ids } } }); + await db.fieldValue.deleteMany({ where: { companyId: { in: ids } } }); + await db.company.deleteMany({ where: { id: { in: ids } } }); + await db.fieldDefinition.deleteMany({ + where: { entity: "COMPANY", label: LEI_FIELD_LABEL }, + }); +} + +beforeEach(async () => { + await clear(); + await db.user.upsert({ + where: { id: USER_ID }, + create: { + id: USER_ID, + name: "Companies Spec", + email: `${USER_ID}@example.test`, + }, + update: {}, + }); +}); + +afterEach(async () => { + await clear(); + await db.user.deleteMany({ where: { id: USER_ID } }); +}); + +describe("createCompany", () => { + it("creates the company, its event, its source note, its LEI and its enrichment", async () => { + const result = await createCompany({ + name: `Siemens ${SUFFIX}`, + website: "https://www.siemens.com/global/", + countryCode: "de", + country: "Germany", + city: "Munich", + lei: "W38RGI023J3WT1HWRP32", + source: SOURCE, + }); + + expect(result.created).toBe(true); + expect(result.domain).toBe("siemens.com"); + + const company = await db.company.findUniqueOrThrow({ + where: { id: result.id }, + select: { countryCode: true, website: true, city: true }, + }); + expect(company.countryCode).toBe("DE"); + expect(company.website).toBe("https://siemens.com"); + expect(company.city).toBe("Munich"); + + const tasks = await db.agentTask.findMany({ + where: { companyId: result.id }, + select: { kind: true, reason: true, payload: true }, + }); + expect(tasks.map((task) => task.kind).sort()).toEqual([ + "agent-event", + "brand", + "company-profile", + ]); + const event = tasks.find((task) => task.kind === "agent-event"); + expect(event?.payload).toMatchObject({ + type: "company.created", + record: { kind: "company", id: result.id }, + }); + + const activity = await db.activity.findFirst({ + where: { companyId: result.id }, + select: { subject: true, body: true, meta: true }, + }); + expect(activity?.subject).toBe("Added from GLEIF register"); + expect(activity?.body).toContain(SOURCE.url); + expect(activity?.meta).toMatchObject({ sourceUrl: SOURCE.url }); + + const lei = await db.fieldValue.findFirst({ + where: { companyId: result.id, field: { label: LEI_FIELD_LABEL } }, + select: { text: true }, + }); + expect(lei?.text).toBe("W38RGI023J3WT1HWRP32"); + }); + + it("returns the existing company instead of a duplicate", async () => { + const first = await createCompany({ + name: `Renault ${SUFFIX}`, + countryCode: "FR", + source: SOURCE, + }); + const second = await createCompany({ + name: `renault ${SUFFIX}`, + countryCode: "fr", + source: SOURCE, + }); + + expect(second.created).toBe(false); + expect(second.id).toBe(first.id); + expect(second.reason).toContain("already"); + + expect( + await db.company.count({ + where: { name: { contains: `Renault ${SUFFIX}` }, archivedAt: null }, + }), + ).toBe(1); + }); + + it("matches on domain before name", async () => { + const first = await createCompany({ + name: `Acme ${SUFFIX}`, + website: "acme-companies-spec.test", + source: SOURCE, + }); + const second = await createCompany({ + name: `Acme Holdings ${SUFFIX}`, + website: "https://www.acme-companies-spec.test/about", + source: SOURCE, + }); + + expect(second.created).toBe(false); + expect(second.id).toBe(first.id); + }); +}); diff --git a/apps/agent/test/hunter.spec.ts b/apps/agent/test/hunter.spec.ts new file mode 100644 index 000000000..45a7774ec --- /dev/null +++ b/apps/agent/test/hunter.spec.ts @@ -0,0 +1,144 @@ +import { afterEach, beforeEach, describe, expect, it } from "bun:test"; +import { + findWorkEmail, + HUNTER_API_KEY, + hunterEnabled, + verifyEmail, +} from "../agent/lib/hunter"; + +const realFetch = globalThis.fetch; +const savedKey = process.env[HUNTER_API_KEY]; +const requested: string[] = []; + +function replies(status: number, json: string) { + globalThis.fetch = (async (input: URL | RequestInfo) => { + requested.push(String(input instanceof Request ? input.url : input)); + return new Response(json, { + status, + headers: { "content-type": "application/json" }, + }); + }) as typeof fetch; +} + +beforeEach(() => { + process.env[HUNTER_API_KEY] = "hunter-test-key"; +}); + +afterEach(() => { + globalThis.fetch = realFetch; + requested.length = 0; + if (savedKey === undefined) delete process.env[HUNTER_API_KEY]; + else process.env[HUNTER_API_KEY] = savedKey; +}); + +describe("hunterEnabled", () => { + it("is off without a key, and blank counts as unset", () => { + delete process.env[HUNTER_API_KEY]; + expect(hunterEnabled()).toBe(false); + process.env[HUNTER_API_KEY] = " "; + expect(hunterEnabled()).toBe(false); + }); +}); + +describe("findWorkEmail", () => { + it("refuses without a key, before any request", async () => { + delete process.env[HUNTER_API_KEY]; + + const result = await findWorkEmail({ + firstName: "Ada", + lastName: "Lovelace", + domain: "example.com", + }); + + expect(result.ok).toBe(false); + expect(requested).toHaveLength(0); + }); + + it("returns the address, its score and the pages it was seen on", async () => { + replies( + 200, + JSON.stringify({ + data: { + email: "ada.lovelace@example.com", + score: 92, + position: "CTO", + sources: [ + { + domain: "example.com", + uri: "https://example.com/team", + extracted_on: "2026-05-01", + }, + { + domain: "news.test", + uri: "https://news.test/ada", + extracted_on: null, + }, + ], + }, + }), + ); + + const result = await findWorkEmail({ + firstName: "Ada", + lastName: "Lovelace", + domain: "Example.com", + }); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.data.email).toBe("ada.lovelace@example.com"); + expect(result.data.score).toBe(92); + expect(result.data.sources.map((s) => s.url)).toEqual([ + "https://example.com/team", + "https://news.test/ada", + ]); + + const url = new URL(requested[0] ?? ""); + expect(url.searchParams.get("domain")).toBe("example.com"); + expect(url.searchParams.get("first_name")).toBe("Ada"); + expect(url.searchParams.get("api_key")).toBe("hunter-test-key"); + }); + + it("keeps a null address as a null, not a failure", async () => { + replies( + 200, + JSON.stringify({ data: { email: null, score: null, sources: [] } }), + ); + + const result = await findWorkEmail({ + firstName: "Ada", + lastName: "Lovelace", + domain: "example.com", + }); + + expect(result).toEqual({ + ok: true, + data: { email: null, score: 0, position: null, sources: [] }, + }); + }); + + it("reports an HTTP failure as a reason", async () => { + replies(429, JSON.stringify({ errors: [] })); + + const result = await findWorkEmail({ + firstName: "Ada", + lastName: "Lovelace", + domain: "example.com", + }); + + expect(result).toEqual({ ok: false, reason: "HTTP 429" }); + }); +}); + +describe("verifyEmail", () => { + it("reads the status and score", async () => { + replies(200, JSON.stringify({ data: { status: "valid", score: 97 } })); + + const result = await verifyEmail("Ada.Lovelace@Example.com"); + + expect(result).toEqual({ ok: true, data: { status: "valid", score: 97 } }); + expect(new URL(requested[0] ?? "").searchParams.get("email")).toBe( + "ada.lovelace@example.com", + ); + }); +}); diff --git a/apps/agent/turbo.json b/apps/agent/turbo.json index 6432b1236..f6a5c6d61 100644 --- a/apps/agent/turbo.json +++ b/apps/agent/turbo.json @@ -20,6 +20,7 @@ "BLOB_READ_WRITE_TOKEN", "DATABASE_URL", "GITHUB_TOKEN", + "HUNTER_API_KEY", "PERPLEXITY_API_KEY" ] }, @@ -33,6 +34,7 @@ "BLOB_READ_WRITE_TOKEN", "DATABASE_URL", "GITHUB_TOKEN", + "HUNTER_API_KEY", "PERPLEXITY_API_KEY" ] }, diff --git a/apps/app/lib/agent-transcript.ts b/apps/app/lib/agent-transcript.ts index 189d0ca36..2bd8a00b2 100644 --- a/apps/app/lib/agent-transcript.ts +++ b/apps/app/lib/agent-transcript.ts @@ -63,6 +63,8 @@ const VERBS: ToolVerbs = { search_crm: "Looked the record up in the CRM", resolve_linkedin_profile: "Searched for their LinkedIn profile", get_linkedin_profile: "Read a LinkedIn profile", + add_company: "Added a company to the CRM", + find_work_email: "Looked up a work email address", gleif_search_entities: "Searched the GLEIF register for a company", gleif_get_entity: "Read a company's GLEIF record", gleif_list_subsidiaries: "Listed a group's subsidiaries from GLEIF", diff --git a/docs/agent.md b/docs/agent.md index ca6b18c75..966b64c46 100644 --- a/docs/agent.md +++ b/docs/agent.md @@ -219,6 +219,23 @@ missing key removes a place to look. **Never an error, never throws.** `capabilitiesFrom()`/`markdownFor()` are the pure halves. `contextDevKey()` is the only resolver, and `lib/context-dev.ts` memoises its client on the key string. +### The agent can add a company, and every one it adds names its source + +`lib/companies.ts` is the one write path: dedupe by domain then by name and +country, the row, the `company.created` event task the API would have written, +an `ENRICHMENT` activity that says where the company came from and links the +page, the LEI as a company field when there is one, then the same `brand` and +`company-profile` tasks `companyCreated` queues on the API side. `add_company` +requires a source; a company without one cannot be created by the agent. + +### Work email is a provider, never a guess + +`lib/hunter.ts` (`HUNTER_API_KEY`) finds an address from a name and an employer +domain and hands back the public pages it was seen on. `find_work_email` fills +the email only when the record has none, and always writes the candidate, its +score and its sources to the timeline. Below `HUNTER.minScore` nothing is +written: a pattern guess is worse than a blank. + ### The GLEIF register needs no key `lib/gleif.ts` reads the public GLEIF API — legal entities by name, one entity by diff --git a/docs/environment.md b/docs/environment.md index 4ced8d483..f86833a30 100644 --- a/docs/environment.md +++ b/docs/environment.md @@ -119,6 +119,7 @@ single place that knows what is set. | Variable | What it adds | | --- | --- | | `PERPLEXITY_API_KEY` | Open-web research with citations; finds a LinkedIn slug | +| `HUNTER_API_KEY` | A work email from name plus employer domain, with the pages it was seen on; a deliverability check | | `GITHUB_TOKEN` | Raises the GitHub rate limit from 60/hour | | `BLOB_READ_WRITE_TOKEN` | Mirrors logos and photos into Blob | | `AI_GATEWAY_API_KEY` | The model. Not needed on Vercel (OIDC) | diff --git a/packages/telemetry/src/allowlist.ts b/packages/telemetry/src/allowlist.ts index f944d9de4..d57d44244 100644 --- a/packages/telemetry/src/allowlist.ts +++ b/packages/telemetry/src/allowlist.ts @@ -114,11 +114,13 @@ export function permitted( } export const AGENT_TOOLS = [ + "add_company", "agent", "archive_field", "enrich_company", "fetch_contact_photo", "find_contact_socials", + "find_work_email", "get_contact_work_history", "get_linkedin_profile", "gleif_get_entity", diff --git a/turbo.json b/turbo.json index 90971be16..519cfcc08 100644 --- a/turbo.json +++ b/turbo.json @@ -25,6 +25,7 @@ "PORT", "PRISMA_LOG_QUERIES", "PERPLEXITY_API_KEY", + "HUNTER_API_KEY", "GITHUB_TOKEN", "BLOB_READ_WRITE_TOKEN", "AI_GATEWAY_API_KEY", From eb8587d96c39511fb9b1207b62255009251fdd8a Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 3 Sep 2026 05:40:52 +0000 Subject: [PATCH 12/19] feat(agent): contact details from Hunter, Apollo, Lusha, Dropcontact or ZoomInfo One registry, one Provider contract, one fetchJson that parses every response at the boundary. lookupContactDetails asks the configured providers in order and stops at the first answer above the confidence threshold. find_contact_details replaces find_work_email: it fills the email and the phone only where the record has none, and always writes the candidate, its confidence and its source to the timeline. Claude-Session: https://claude.ai/code/session_011fuk1HeytBt9RU12xKY8WB --- .env.example | 28 +- apps/agent/agent/lib/apollo.ts | 103 +++++ apps/agent/agent/lib/capabilities.ts | 32 +- .../agent/agent/lib/contact-details-config.ts | 36 ++ .../agent/lib/contact-details-providers.ts | 14 + apps/agent/agent/lib/contact-details.ts | 128 ++++++ apps/agent/agent/lib/dropcontact.ts | 128 ++++++ apps/agent/agent/lib/hunter-config.ts | 10 - apps/agent/agent/lib/hunter.ts | 136 +++---- apps/agent/agent/lib/lusha.ts | 100 +++++ apps/agent/agent/lib/zoominfo.ts | 150 +++++++ .../agent/skills/gleif-mna-sourcing/SKILL.md | 8 +- .../agent/agent/tools/find_contact_details.ts | 171 ++++++++ apps/agent/agent/tools/find_work_email.ts | 125 ------ apps/agent/test/capabilities.spec.ts | 5 + apps/agent/test/contact-details.spec.ts | 382 ++++++++++++++++++ apps/agent/test/hunter.spec.ts | 48 +-- apps/agent/turbo.json | 10 + apps/app/lib/agent-transcript.ts | 2 +- docs/agent.md | 24 +- docs/environment.md | 6 +- packages/telemetry/src/allowlist.ts | 2 +- turbo.json | 5 + 23 files changed, 1396 insertions(+), 257 deletions(-) create mode 100644 apps/agent/agent/lib/apollo.ts create mode 100644 apps/agent/agent/lib/contact-details-config.ts create mode 100644 apps/agent/agent/lib/contact-details-providers.ts create mode 100644 apps/agent/agent/lib/contact-details.ts create mode 100644 apps/agent/agent/lib/dropcontact.ts delete mode 100644 apps/agent/agent/lib/hunter-config.ts create mode 100644 apps/agent/agent/lib/lusha.ts create mode 100644 apps/agent/agent/lib/zoominfo.ts create mode 100644 apps/agent/agent/tools/find_contact_details.ts delete mode 100644 apps/agent/agent/tools/find_work_email.ts create mode 100644 apps/agent/test/contact-details.spec.ts diff --git a/.env.example b/.env.example index 788aa4162..05669d5da 100644 --- a/.env.example +++ b/.env.example @@ -131,11 +131,33 @@ GOOGLE_CLIENT_SECRET="" # knowing before a call. https://perplexity.ai/settings/api # PERPLEXITY_API_KEY="" -# Hunter — finds a person's work email from their name and their employer's -# domain, with the public pages it was seen on, and checks whether an address -# is deliverable. The free plan covers a few dozen lookups a month. +# Contact-data providers. Each one is optional and the agent asks the ones +# that are set, in this order, stopping at the first confident answer: +# Hunter, Apollo, Lusha, Dropcontact, ZoomInfo. Every answer is written to +# the contact's timeline with its confidence and its source. +# +# Hunter — a work email with the public pages it was seen on, and a +# deliverability check. The free plan covers a few dozen lookups a month. # https://hunter.io/api-keys # HUNTER_API_KEY="" +# +# Apollo — a work email with its verification status, title and work phone. +# https://app.apollo.io/#/settings/integrations/api +# APOLLO_API_KEY="" +# +# Lusha — work email, direct and mobile phones, title. +# https://dashboard.lusha.com/enrich/api +# LUSHA_API_KEY="" +# +# Dropcontact — work email with its qualification, phone, title. GDPR-compliant +# by construction; the lookup is asynchronous and takes a few seconds. +# https://app.dropcontact.com/api +# DROPCONTACT_API_KEY="" +# +# ZoomInfo — work email, direct and mobile phones, title. Needs a ZoomInfo API +# user; both halves must be set, and a token is minted from them and cached. +# ZOOMINFO_USERNAME="" +# ZOOMINFO_PASSWORD="" # GitHub — raises the rate limit when matching contacts to GitHub profiles. # Any classic token with no scopes will do. diff --git a/apps/agent/agent/lib/apollo.ts b/apps/agent/agent/lib/apollo.ts new file mode 100644 index 000000000..78973c023 --- /dev/null +++ b/apps/agent/agent/lib/apollo.ts @@ -0,0 +1,103 @@ +import { z } from "zod"; +import { + type ContactDetails, + fetchJson, + keyed, + type Outcome, + type Person, + type Provider, +} from "./contact-details"; +import { CONTACT_DETAILS } from "./contact-details-config"; + +export const APOLLO_API_KEY = "APOLLO_API_KEY"; + +const phone = z.object({ + sanitized_number: z.string().trim().min(1).nullable().optional(), + raw_number: z.string().trim().min(1).nullable().optional(), + type: z.string().nullable().optional(), +}); + +const match = z.object({ + person: z + .object({ + id: z.string().nullable().optional(), + email: z.string().trim().email().nullable().optional(), + email_status: z.string().nullable().optional(), + title: z.string().nullable().optional(), + linkedin_url: z.string().nullable().optional(), + phone_numbers: z.array(phone).nullable().optional(), + }) + .nullable() + .optional(), +}); + +function confidenceOf(status: string | null | undefined): number { + const scale = CONTACT_DETAILS.apollo.confidence; + switch (status) { + case "verified": + return scale.verified; + case "likely_to_engage": + case "likely": + return scale.likely; + case "guessed": + case "extrapolated": + return scale.guessed; + default: + return scale.unknown; + } +} + +export async function apolloMatch( + person: Person, +): Promise> { + const apiKey = process.env[APOLLO_API_KEY]?.trim(); + if (!apiKey) return { ok: false, reason: `No ${APOLLO_API_KEY}.` }; + + const response = await fetchJson( + new URL(`${CONTACT_DETAILS.apollo.baseUrl}/people/match`), + { + method: "POST", + headers: { + "content-type": "application/json", + "x-api-key": apiKey, + }, + body: JSON.stringify({ + first_name: person.firstName.trim(), + last_name: person.lastName.trim(), + domain: person.domain.trim().toLowerCase(), + organization_name: person.companyName ?? undefined, + reveal_personal_emails: false, + reveal_phone_number: false, + }), + }, + match, + "Apollo", + ); + if (!response.ok) return response; + + const found = response.data.person; + return { + ok: true, + data: { + provider: "apollo", + email: found?.email ?? null, + confidence: found?.email ? confidenceOf(found.email_status) : 0, + phones: (found?.phone_numbers ?? []).flatMap((entry) => { + const number = entry.sanitized_number ?? entry.raw_number; + return number ? [{ number, type: entry.type ?? null }] : []; + }), + title: found?.title ?? null, + linkedinUrl: found?.linkedin_url ?? null, + sources: [], + reference: found?.id ?? null, + }, + }; +} + +export const apollo: Provider = { + id: "apollo", + label: "Apollo", + keys: [APOLLO_API_KEY], + enabled: keyed(APOLLO_API_KEY), + find: apolloMatch, +}; diff --git a/apps/agent/agent/lib/capabilities.ts b/apps/agent/agent/lib/capabilities.ts index d63696899..1c67716ce 100644 --- a/apps/agent/agent/lib/capabilities.ts +++ b/apps/agent/agent/lib/capabilities.ts @@ -68,9 +68,37 @@ export function capabilitiesFrom( }, { ...fromEnv("HUNTER_API_KEY"), - label: "Work email lookup", + label: "Contact details via Hunter", gives: - "a person's work email address from their name and their employer's domain, with the public pages Hunter saw it on, and a deliverability check on an address you already hold", + "a person's work email from their name and their employer's domain, with the public pages Hunter saw it on, and a deliverability check on an address you already hold", + }, + { + ...fromEnv("APOLLO_API_KEY"), + label: "Contact details via Apollo", + gives: + "a person's work email with Apollo's verification status, their title and a work phone", + }, + { + ...fromEnv("LUSHA_API_KEY"), + label: "Contact details via Lusha", + gives: "a person's work email, direct and mobile phones and their title", + }, + { + ...fromEnv("DROPCONTACT_API_KEY"), + label: "Contact details via Dropcontact", + gives: + "a person's work email with its qualification, a phone and their title, from a GDPR-compliant source", + }, + { + id: "ZOOMINFO_USERNAME", + from: "ZOOMINFO_USERNAME + ZOOMINFO_PASSWORD", + label: "Contact details via ZoomInfo", + gives: + "a person's work email, direct and mobile phones and their title from ZoomInfo's database", + enabled: Boolean( + process.env.ZOOMINFO_USERNAME?.trim() && + process.env.ZOOMINFO_PASSWORD?.trim(), + ), }, { ...fromEnv("BLOB_READ_WRITE_TOKEN"), diff --git a/apps/agent/agent/lib/contact-details-config.ts b/apps/agent/agent/lib/contact-details-config.ts new file mode 100644 index 000000000..9e9ba6079 --- /dev/null +++ b/apps/agent/agent/lib/contact-details-config.ts @@ -0,0 +1,36 @@ +const SECOND_MS = 1_000; +const MINUTE_MS = 60 * SECOND_MS; + +export const CONTACT_DETAILS = { + timeoutMs: 20 * SECOND_MS, + minConfidence: 50, + maxSources: 5, + order: ["hunter", "apollo", "lusha", "dropcontact", "zoominfo"], + + hunter: { + baseUrl: "https://api.hunter.io/v2", + }, + + apollo: { + baseUrl: "https://api.apollo.io/api/v1", + confidence: { verified: 95, likely: 70, guessed: 55, unknown: 30 }, + }, + + lusha: { + baseUrl: "https://api.lusha.com", + confidence: { work: 85, other: 60 }, + }, + + dropcontact: { + baseUrl: "https://api.dropcontact.io", + pollMs: 3 * SECOND_MS, + maxPolls: 10, + confidence: { nominative: 90, catchAll: 55, other: 40 }, + }, + + zoominfo: { + baseUrl: "https://api.zoominfo.com", + tokenTtlMs: 55 * MINUTE_MS, + confidence: { matched: 80 }, + }, +} as const; diff --git a/apps/agent/agent/lib/contact-details-providers.ts b/apps/agent/agent/lib/contact-details-providers.ts new file mode 100644 index 000000000..8e97497c4 --- /dev/null +++ b/apps/agent/agent/lib/contact-details-providers.ts @@ -0,0 +1,14 @@ +import { apollo } from "./apollo"; +import type { Provider } from "./contact-details"; +import { dropcontact } from "./dropcontact"; +import { hunter } from "./hunter"; +import { lusha } from "./lusha"; +import { zoominfo } from "./zoominfo"; + +export const CONTACT_DETAILS_PROVIDERS: readonly Provider[] = [ + hunter, + apollo, + lusha, + dropcontact, + zoominfo, +]; diff --git a/apps/agent/agent/lib/contact-details.ts b/apps/agent/agent/lib/contact-details.ts new file mode 100644 index 000000000..145b291c7 --- /dev/null +++ b/apps/agent/agent/lib/contact-details.ts @@ -0,0 +1,128 @@ +import type { z } from "zod"; +import { CONTACT_DETAILS } from "./contact-details-config"; + +export type Outcome = { ok: true; data: T } | { ok: false; reason: string }; + +export type ProviderId = (typeof CONTACT_DETAILS.order)[number]; + +export type DetailSource = { + url: string; + domain: string | null; + seenOn: string | null; +}; + +export type Phone = { number: string; type: string | null }; + +export type ContactDetails = { + provider: ProviderId; + email: string | null; + confidence: number; + phones: Phone[]; + title: string | null; + linkedinUrl: string | null; + sources: DetailSource[]; + reference: string | null; +}; + +export type Person = { + firstName: string; + lastName: string; + domain: string; + companyName: string | null; +}; + +export type Provider = { + id: ProviderId; + label: string; + keys: readonly string[]; + enabled: () => boolean; + find: (person: Person) => Promise>; +}; + +export type Lookup = + | { outcome: "found"; details: ContactDetails; tried: ProviderId[] } + | { outcome: "none"; tried: ProviderId[]; reasons: string[] } + | { outcome: "unconfigured" }; + +export function keyed(...names: string[]): () => boolean { + return () => names.every((name) => Boolean(process.env[name]?.trim())); +} + +export async function fetchJson( + url: URL, + init: RequestInit, + shape: Shape, + label: string, +): Promise>> { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), CONTACT_DETAILS.timeoutMs); + + try { + const response = await fetch(url, { ...init, signal: controller.signal }); + if (!response.ok) return { ok: false, reason: `HTTP ${response.status}` }; + + const parsed = shape.safeParse(await response.json()); + return parsed.success + ? { ok: true, data: parsed.data } + : { + ok: false, + reason: `Unreadable ${label} response: ${parsed.error.message}`, + }; + } catch (error) { + const aborted = error instanceof Error && error.name === "AbortError"; + return { + ok: false, + reason: aborted + ? `${label} timed out after ${CONTACT_DETAILS.timeoutMs}ms.` + : error instanceof Error + ? error.message + : String(error), + }; + } finally { + clearTimeout(timer); + } +} + +export function configuredProviders(all: readonly Provider[]): Provider[] { + return CONTACT_DETAILS.order.flatMap((id) => { + const provider = all.find((candidate) => candidate.id === id); + return provider?.enabled() ? [provider] : []; + }); +} + +export async function lookupContactDetails( + person: Person, + all: readonly Provider[], +): Promise { + const providers = configuredProviders(all); + if (providers.length === 0) return { outcome: "unconfigured" }; + + const tried: ProviderId[] = []; + const reasons: string[] = []; + + for (const provider of providers) { + tried.push(provider.id); + const result = await provider.find(person); + + if (!result.ok) { + reasons.push(`${provider.label}: ${result.reason}`); + continue; + } + + const { data } = result; + if (data.email && data.confidence >= CONTACT_DETAILS.minConfidence) { + return { outcome: "found", details: data, tried }; + } + if (data.phones.length > 0 && !data.email) { + return { outcome: "found", details: data, tried }; + } + + reasons.push( + data.email + ? `${provider.label}: ${data.email} at confidence ${data.confidence}, below ${CONTACT_DETAILS.minConfidence}` + : `${provider.label}: nothing for this person`, + ); + } + + return { outcome: "none", tried, reasons }; +} diff --git a/apps/agent/agent/lib/dropcontact.ts b/apps/agent/agent/lib/dropcontact.ts new file mode 100644 index 000000000..779e88301 --- /dev/null +++ b/apps/agent/agent/lib/dropcontact.ts @@ -0,0 +1,128 @@ +import { z } from "zod"; +import { + type ContactDetails, + fetchJson, + keyed, + type Outcome, + type Person, + type Provider, +} from "./contact-details"; +import { CONTACT_DETAILS } from "./contact-details-config"; + +export const DROPCONTACT_API_KEY = "DROPCONTACT_API_KEY"; + +const submitted = z.object({ + request_id: z.string().trim().min(1), +}); + +const email = z.object({ + email: z.string().trim().email(), + qualification: z.string().nullable().optional(), +}); + +const row = z.object({ + email: z.array(email).nullable().optional(), + phone: z.string().nullable().optional(), + mobile_phone: z.string().nullable().optional(), + job: z.string().nullable().optional(), + linkedin: z.string().nullable().optional(), +}); + +const batch = z.object({ + success: z.boolean(), + data: z.array(row).nullable().optional(), +}); + +function confidenceOf(qualification: string | null | undefined): number { + const scale = CONTACT_DETAILS.dropcontact.confidence; + const value = (qualification ?? "").toLowerCase(); + if (value.startsWith("nominative")) return scale.nominative; + if (value.startsWith("catch_all")) return scale.catchAll; + return scale.other; +} + +function wait(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +export async function dropcontactEnrich( + person: Person, +): Promise> { + const apiKey = process.env[DROPCONTACT_API_KEY]?.trim(); + if (!apiKey) return { ok: false, reason: `No ${DROPCONTACT_API_KEY}.` }; + + const headers = { + "content-type": "application/json", + "X-Access-Token": apiKey, + }; + const base = CONTACT_DETAILS.dropcontact.baseUrl; + + const submit = await fetchJson( + new URL(`${base}/batch`), + { + method: "POST", + headers, + body: JSON.stringify({ + data: [ + { + first_name: person.firstName.trim(), + last_name: person.lastName.trim(), + website: person.domain.trim().toLowerCase(), + company: person.companyName ?? undefined, + }, + ], + siren: false, + }), + }, + submitted, + "Dropcontact", + ); + if (!submit.ok) return submit; + + for (let poll = 0; poll < CONTACT_DETAILS.dropcontact.maxPolls; poll++) { + await wait(CONTACT_DETAILS.dropcontact.pollMs); + + const result = await fetchJson( + new URL(`${base}/batch/${encodeURIComponent(submit.data.request_id)}`), + { headers }, + batch, + "Dropcontact", + ); + if (!result.ok) return result; + if (!result.data.success) continue; + + const found = result.data.data?.[0] ?? null; + const best = found?.email?.[0] ?? null; + const phones = [found?.phone, found?.mobile_phone].flatMap( + (number, index) => + number ? [{ number, type: index === 0 ? "work" : "mobile" }] : [], + ); + + return { + ok: true, + data: { + provider: "dropcontact", + email: best?.email ?? null, + confidence: best ? confidenceOf(best.qualification) : 0, + phones, + title: found?.job ?? null, + linkedinUrl: found?.linkedin ?? null, + sources: [], + reference: submit.data.request_id, + }, + }; + } + + return { + ok: false, + reason: `Dropcontact did not finish within ${CONTACT_DETAILS.dropcontact.maxPolls} polls.`, + }; +} + +export const dropcontact: Provider = { + id: "dropcontact", + label: "Dropcontact", + keys: [DROPCONTACT_API_KEY], + enabled: keyed(DROPCONTACT_API_KEY), + find: dropcontactEnrich, +}; diff --git a/apps/agent/agent/lib/hunter-config.ts b/apps/agent/agent/lib/hunter-config.ts deleted file mode 100644 index 865568172..000000000 --- a/apps/agent/agent/lib/hunter-config.ts +++ /dev/null @@ -1,10 +0,0 @@ -const SECOND_MS = 1_000; - -export const HUNTER = { - api: { - baseUrl: "https://api.hunter.io/v2", - timeoutMs: 20 * SECOND_MS, - }, - minScore: 50, - maxSources: 5, -} as const; diff --git a/apps/agent/agent/lib/hunter.ts b/apps/agent/agent/lib/hunter.ts index 22af6db5f..081405bb4 100644 --- a/apps/agent/agent/lib/hunter.ts +++ b/apps/agent/agent/lib/hunter.ts @@ -1,7 +1,13 @@ import { z } from "zod"; -import { HUNTER } from "./hunter-config"; - -export type Outcome = { ok: true; data: T } | { ok: false; reason: string }; +import { + type ContactDetails, + fetchJson, + keyed, + type Outcome, + type Person, + type Provider, +} from "./contact-details"; +import { CONTACT_DETAILS } from "./contact-details-config"; export const HUNTER_API_KEY = "HUNTER_API_KEY"; @@ -15,9 +21,8 @@ const finder = z.object({ data: z.object({ email: z.string().trim().email().nullable(), score: z.number().nullable().optional(), - first_name: z.string().nullable().optional(), - last_name: z.string().nullable().optional(), position: z.string().nullable().optional(), + linkedin_url: z.string().nullable().optional(), sources: z.array(source).nullable().optional(), }), }); @@ -29,81 +34,35 @@ const verifier = z.object({ }), }); -export type EmailSource = { - url: string; - domain: string | null; - seenOn: string | null; -}; - -export type WorkEmail = { - email: string | null; - score: number; - position: string | null; - sources: EmailSource[]; -}; - export type Verification = { status: string; score: number | null }; export function hunterEnabled(): boolean { - return Boolean(process.env[HUNTER_API_KEY]?.trim()); + return keyed(HUNTER_API_KEY)(); } -async function request( - path: string, - query: Record, - shape: Shape, -): Promise>> { - const apiKey = process.env[HUNTER_API_KEY]?.trim(); - if (!apiKey) return { ok: false, reason: `No ${HUNTER_API_KEY}.` }; - - const url = new URL(`${HUNTER.api.baseUrl}${path}`); +function endpoint(path: string, query: Record): URL { + const url = new URL(`${CONTACT_DETAILS.hunter.baseUrl}${path}`); for (const [key, value] of Object.entries(query)) { - if (value !== undefined) url.searchParams.set(key, value); - } - url.searchParams.set("api_key", apiKey); - - const controller = new AbortController(); - const timer = setTimeout(() => controller.abort(), HUNTER.api.timeoutMs); - - try { - const response = await fetch(url, { signal: controller.signal }); - if (!response.ok) return { ok: false, reason: `HTTP ${response.status}` }; - - const parsed = shape.safeParse(await response.json()); - return parsed.success - ? { ok: true, data: parsed.data } - : { - ok: false, - reason: `Unreadable Hunter response: ${parsed.error.message}`, - }; - } catch (error) { - const aborted = error instanceof Error && error.name === "AbortError"; - return { - ok: false, - reason: aborted - ? `Hunter timed out after ${HUNTER.api.timeoutMs}ms.` - : error instanceof Error - ? error.message - : String(error), - }; - } finally { - clearTimeout(timer); + url.searchParams.set(key, value); } + url.searchParams.set("api_key", process.env[HUNTER_API_KEY]?.trim() ?? ""); + return url; } -export async function findWorkEmail(input: { - firstName: string; - lastName: string; - domain: string; -}): Promise> { - const response = await request( - "/email-finder", - { - domain: input.domain.trim().toLowerCase(), - first_name: input.firstName.trim(), - last_name: input.lastName.trim(), - }, +export async function findWorkEmail( + person: Person, +): Promise> { + if (!hunterEnabled()) return { ok: false, reason: `No ${HUNTER_API_KEY}.` }; + + const response = await fetchJson( + endpoint("/email-finder", { + domain: person.domain.trim().toLowerCase(), + first_name: person.firstName.trim(), + last_name: person.lastName.trim(), + }), + {}, finder, + "Hunter", ); if (!response.ok) return response; @@ -111,14 +70,20 @@ export async function findWorkEmail(input: { return { ok: true, data: { + provider: "hunter", email: data.email, - score: data.score ?? 0, - position: data.position ?? null, - sources: (data.sources ?? []).slice(0, HUNTER.maxSources).map((s) => ({ - url: s.uri, - domain: s.domain ?? null, - seenOn: s.extracted_on ?? null, - })), + confidence: data.score ?? 0, + phones: [], + title: data.position ?? null, + linkedinUrl: data.linkedin_url ?? null, + sources: (data.sources ?? []) + .slice(0, CONTACT_DETAILS.maxSources) + .map((s) => ({ + url: s.uri, + domain: s.domain ?? null, + seenOn: s.extracted_on ?? null, + })), + reference: null, }, }; } @@ -126,10 +91,13 @@ export async function findWorkEmail(input: { export async function verifyEmail( email: string, ): Promise> { - const response = await request( - "/email-verifier", - { email: email.trim().toLowerCase() }, + if (!hunterEnabled()) return { ok: false, reason: `No ${HUNTER_API_KEY}.` }; + + const response = await fetchJson( + endpoint("/email-verifier", { email: email.trim().toLowerCase() }), + {}, verifier, + "Hunter", ); if (!response.ok) return response; @@ -141,3 +109,11 @@ export async function verifyEmail( }, }; } + +export const hunter: Provider = { + id: "hunter", + label: "Hunter", + keys: [HUNTER_API_KEY], + enabled: hunterEnabled, + find: findWorkEmail, +}; diff --git a/apps/agent/agent/lib/lusha.ts b/apps/agent/agent/lib/lusha.ts new file mode 100644 index 000000000..269acc975 --- /dev/null +++ b/apps/agent/agent/lib/lusha.ts @@ -0,0 +1,100 @@ +import { z } from "zod"; +import { + type ContactDetails, + fetchJson, + keyed, + type Outcome, + type Person, + type Provider, +} from "./contact-details"; +import { CONTACT_DETAILS } from "./contact-details-config"; + +export const LUSHA_API_KEY = "LUSHA_API_KEY"; + +const email = z.object({ + email: z.string().trim().email(), + emailType: z.string().nullable().optional(), +}); + +const phone = z.object({ + number: z.string().trim().min(1), + phoneType: z.string().nullable().optional(), +}); + +const contact = z.object({ + jobTitle: z.string().nullable().optional(), + emailAddresses: z.array(email).nullable().optional(), + phoneNumbers: z.array(phone).nullable().optional(), + socialLinks: z + .object({ linkedin: z.string().nullable().optional() }) + .nullable() + .optional(), + id: z.union([z.string(), z.number()]).nullable().optional(), +}); + +const person = z.object({ + data: z + .union([z.object({ contact }), contact]) + .nullable() + .optional(), +}); + +function unwrap( + value: z.infer["data"], +): z.infer | null { + if (!value) return null; + return "contact" in value ? value.contact : value; +} + +export async function lushaPerson( + who: Person, +): Promise> { + const apiKey = process.env[LUSHA_API_KEY]?.trim(); + if (!apiKey) return { ok: false, reason: `No ${LUSHA_API_KEY}.` }; + + const url = new URL(`${CONTACT_DETAILS.lusha.baseUrl}/v2/person`); + url.searchParams.set("firstName", who.firstName.trim()); + url.searchParams.set("lastName", who.lastName.trim()); + url.searchParams.set("companyDomain", who.domain.trim().toLowerCase()); + + const response = await fetchJson( + url, + { headers: { api_key: apiKey } }, + person, + "Lusha", + ); + if (!response.ok) return response; + + const found = unwrap(response.data.data); + const work = (found?.emailAddresses ?? []).find( + (entry) => (entry.emailType ?? "").toLowerCase() === "work", + ); + const best = work ?? (found?.emailAddresses ?? [])[0] ?? null; + const scale = CONTACT_DETAILS.lusha.confidence; + + return { + ok: true, + data: { + provider: "lusha", + email: best?.email ?? null, + confidence: best ? (work ? scale.work : scale.other) : 0, + phones: (found?.phoneNumbers ?? []).map((entry) => ({ + number: entry.number, + type: entry.phoneType ?? null, + })), + title: found?.jobTitle ?? null, + linkedinUrl: found?.socialLinks?.linkedin ?? null, + sources: [], + reference: + found?.id === undefined || found.id === null ? null : String(found.id), + }, + }; +} + +export const lusha: Provider = { + id: "lusha", + label: "Lusha", + keys: [LUSHA_API_KEY], + enabled: keyed(LUSHA_API_KEY), + find: lushaPerson, +}; diff --git a/apps/agent/agent/lib/zoominfo.ts b/apps/agent/agent/lib/zoominfo.ts new file mode 100644 index 000000000..dd63e1bbc --- /dev/null +++ b/apps/agent/agent/lib/zoominfo.ts @@ -0,0 +1,150 @@ +import { z } from "zod"; +import { + type ContactDetails, + fetchJson, + keyed, + type Outcome, + type Person, + type Phone, + type Provider, +} from "./contact-details"; +import { CONTACT_DETAILS } from "./contact-details-config"; + +export const ZOOMINFO_USERNAME = "ZOOMINFO_USERNAME"; +export const ZOOMINFO_PASSWORD = "ZOOMINFO_PASSWORD"; + +const authenticated = z.object({ jwt: z.string().trim().min(1) }); + +const hit = z.object({ + id: z.union([z.string(), z.number()]).nullable().optional(), + email: z.string().trim().email().nullable().optional(), + phone: z.string().nullable().optional(), + directPhone: z.string().nullable().optional(), + mobilePhone: z.string().nullable().optional(), + jobTitle: z.string().nullable().optional(), +}); + +const enriched = z.object({ + success: z.boolean().optional(), + data: z + .object({ + result: z + .array(z.object({ data: z.array(hit).nullable().optional() })) + .nullable() + .optional(), + }) + .nullable() + .optional(), +}); + +let session: { username: string; jwt: string; expiresAt: number } | null = null; + +async function token(): Promise> { + const username = process.env[ZOOMINFO_USERNAME]?.trim(); + const password = process.env[ZOOMINFO_PASSWORD]?.trim(); + if (!username || !password) { + return { + ok: false, + reason: `No ${ZOOMINFO_USERNAME} and ${ZOOMINFO_PASSWORD}.`, + }; + } + + if (session?.username === username && session.expiresAt > Date.now()) { + return { ok: true, data: session.jwt }; + } + + const response = await fetchJson( + new URL(`${CONTACT_DETAILS.zoominfo.baseUrl}/authenticate`), + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ username, password }), + }, + authenticated, + "ZoomInfo", + ); + if (!response.ok) return response; + + session = { + username, + jwt: response.data.jwt, + expiresAt: Date.now() + CONTACT_DETAILS.zoominfo.tokenTtlMs, + }; + return { ok: true, data: session.jwt }; +} + +export function forgetZoomInfoSession(): void { + session = null; +} + +export async function zoominfoEnrich( + person: Person, +): Promise> { + const jwt = await token(); + if (!jwt.ok) return jwt; + + const response = await fetchJson( + new URL(`${CONTACT_DETAILS.zoominfo.baseUrl}/enrich/contact`), + { + method: "POST", + headers: { + "content-type": "application/json", + authorization: `Bearer ${jwt.data}`, + }, + body: JSON.stringify({ + matchPersonInput: [ + { + firstName: person.firstName.trim(), + lastName: person.lastName.trim(), + companyName: person.companyName ?? undefined, + companyWebsite: person.domain.trim().toLowerCase(), + }, + ], + outputFields: [ + "id", + "email", + "phone", + "directPhone", + "mobilePhone", + "jobTitle", + ], + }), + }, + enriched, + "ZoomInfo", + ); + if (!response.ok) return response; + + const found = response.data.data?.result?.[0]?.data?.[0] ?? null; + const phones: Phone[] = []; + if (found?.directPhone) + phones.push({ number: found.directPhone, type: "direct" }); + if (found?.mobilePhone) + phones.push({ number: found.mobilePhone, type: "mobile" }); + if (found?.phone) phones.push({ number: found.phone, type: "work" }); + + return { + ok: true, + data: { + provider: "zoominfo", + email: found?.email ?? null, + confidence: found?.email + ? CONTACT_DETAILS.zoominfo.confidence.matched + : 0, + phones, + title: found?.jobTitle ?? null, + linkedinUrl: null, + sources: [], + reference: + found?.id === undefined || found?.id === null ? null : String(found.id), + }, + }; +} + +export const zoominfo: Provider = { + id: "zoominfo", + label: "ZoomInfo", + keys: [ZOOMINFO_USERNAME, ZOOMINFO_PASSWORD], + enabled: keyed(ZOOMINFO_USERNAME, ZOOMINFO_PASSWORD), + find: zoominfoEnrich, +}; diff --git a/apps/agent/agent/skills/gleif-mna-sourcing/SKILL.md b/apps/agent/agent/skills/gleif-mna-sourcing/SKILL.md index 2ed5b8454..89c20c7e7 100644 --- a/apps/agent/agent/skills/gleif-mna-sourcing/SKILL.md +++ b/apps/agent/agent/skills/gleif-mna-sourcing/SKILL.md @@ -60,10 +60,10 @@ corners. The rules: local ones have no public profile. Do not fill the gap. Contact details come only from a provider that carries the compliance of the -source, never from a page you read. `find_work_email` is that provider when -`HUNTER_API_KEY` is set: it returns the address with the public pages it was -seen on and writes them to the contact's timeline. Without it, stop at name, -title and public profile. +source, never from a page you read. `find_contact_details` asks the configured +providers (Hunter, Apollo, Lusha, Dropcontact, ZoomInfo) in order and writes +the answer, its confidence and its source to the contact's timeline. Without +any of them, stop at name, title and public profile. ## 5. Deliver diff --git a/apps/agent/agent/tools/find_contact_details.ts b/apps/agent/agent/tools/find_contact_details.ts new file mode 100644 index 000000000..b5279688f --- /dev/null +++ b/apps/agent/agent/tools/find_contact_details.ts @@ -0,0 +1,171 @@ +import { ActivityType, db } from "@crm/db"; +import { defineTool } from "eve/tools"; +import { z } from "zod"; +import { + type ContactDetails, + configuredProviders, + lookupContactDetails, +} from "../lib/contact-details"; +import { CONTACT_DETAILS_PROVIDERS } from "../lib/contact-details-providers"; +import { focusOn, spend } from "../lib/focus"; +import { domainOf } from "../lib/names"; + +function describeSources(details: ContactDetails): string { + if (details.sources.length > 0) { + return `Seen on: ${details.sources.map((s) => s.url).join(", ")}`; + } + return details.reference + ? `Attested by ${details.provider} (record ${details.reference}).` + : `Attested by ${details.provider}, which gave no public page.`; +} + +export default defineTool({ + description: + "Find a contact's work email and phone from their name and their employer's domain, through the configured contact-data providers in order (Hunter, Apollo, Lusha, Dropcontact, ZoomInfo), stopping at the first confident answer. Fills the email and phone only where the record has none, and always writes the candidate, its confidence and its source to the contact's timeline. Without any provider key the tool says so and nothing is charged.", + inputSchema: z.object({ + contactId: z.string(), + }), + async execute({ contactId }) { + const providers = configuredProviders(CONTACT_DETAILS_PROVIDERS); + if (providers.length === 0) { + return { + ok: false as const, + configured: false as const, + reason: + "No contact-data provider is configured on this install (HUNTER_API_KEY, APOLLO_API_KEY, LUSHA_API_KEY, DROPCONTACT_API_KEY or ZOOMINFO_USERNAME with ZOOMINFO_PASSWORD). This is not a failure and retrying will not help — say in your write-up that contact details could not be checked.", + }; + } + + focusOn({ contactId }); + + const contact = await db.contact.findUnique({ + where: { id: contactId }, + select: { + id: true, + firstName: true, + lastName: true, + email: true, + phone: true, + company: { + select: { id: true, name: true, domain: true, ownerId: true }, + }, + }, + }); + + if (!contact) return { ok: false as const, reason: "No such contact." }; + if (!contact.lastName) { + return { + ok: false as const, + reason: "The contact has no last name yet. Identify them first.", + }; + } + + const domain = + contact.company?.domain ?? + (contact.email ? domainOf(contact.email) : null); + if (!domain) { + return { + ok: false as const, + reason: + "No employer domain to search. Set the company's website first.", + }; + } + + const charge = spend(1); + if (!charge.ok) return { ok: false as const, reason: charge.reason }; + + const lookup = await lookupContactDetails( + { + firstName: contact.firstName, + lastName: contact.lastName, + domain, + companyName: contact.company?.name ?? null, + }, + CONTACT_DETAILS_PROVIDERS, + ); + + if (lookup.outcome === "unconfigured") { + return { ok: false as const, reason: "No provider is configured." }; + } + if (lookup.outcome === "none") { + return { + ok: true as const, + email: null, + phones: [], + tried: lookup.tried, + reasons: lookup.reasons, + note: "No provider has an address it trusts for this person. Leave the email blank rather than guessing a pattern.", + }; + } + + const { details } = lookup; + const author = + contact.company?.ownerId ?? + (await db.user.findFirst({ select: { id: true } }))?.id ?? + null; + + const phone = details.phones[0]?.number ?? null; + const filledEmail = Boolean(details.email) && !contact.email; + const filledPhone = Boolean(phone) && !contact.phone; + + if (filledEmail || filledPhone) { + await db.contact.update({ + where: { id: contact.id }, + data: { + email: filledEmail ? details.email : undefined, + phone: filledPhone ? phone : undefined, + }, + }); + } + + if (author) { + await db.activity.create({ + data: { + type: ActivityType.ENRICHMENT, + subject: filledEmail + ? "Contact details found" + : "Contact details candidate", + body: [ + details.email + ? `${details.email} (confidence ${details.confidence}).` + : null, + phone ? `Phone ${phone}.` : null, + describeSources(details), + details.email && !filledEmail + ? `The record keeps ${contact.email}.` + : null, + ] + .filter(Boolean) + .join(" "), + occurredAt: new Date(), + contactId: contact.id, + companyId: contact.company?.id ?? null, + createdById: author, + meta: { + source: details.provider, + confidence: details.confidence, + reference: details.reference, + sources: details.sources.map((s) => s.url), + tried: lookup.tried, + agent: "people-research", + }, + }, + select: { id: true }, + }); + } + + return { + ok: true as const, + provider: details.provider, + email: details.email, + confidence: details.confidence, + phones: details.phones, + title: details.title, + linkedinUrl: details.linkedinUrl, + sources: details.sources, + filledEmail, + filledPhone, + tried: lookup.tried, + }; + }, +}); diff --git a/apps/agent/agent/tools/find_work_email.ts b/apps/agent/agent/tools/find_work_email.ts deleted file mode 100644 index adfe45b2a..000000000 --- a/apps/agent/agent/tools/find_work_email.ts +++ /dev/null @@ -1,125 +0,0 @@ -import { ActivityType, db } from "@crm/db"; -import { defineTool } from "eve/tools"; -import { z } from "zod"; -import { unavailable } from "../lib/capabilities"; -import { focusOn, spend } from "../lib/focus"; -import { findWorkEmail, HUNTER_API_KEY, hunterEnabled } from "../lib/hunter"; -import { HUNTER } from "../lib/hunter-config"; -import { domainOf } from "../lib/names"; - -export default defineTool({ - description: - "Find a contact's work email address from their name and their employer's domain, through Hunter, with the public pages the address was seen on. Writes the candidate and its sources to the contact's timeline, and fills the email in when the record has none. Needs HUNTER_API_KEY; without it the tool says so and nothing is charged.", - inputSchema: z.object({ - contactId: z.string(), - }), - async execute({ contactId }) { - if (!hunterEnabled()) return unavailable(HUNTER_API_KEY); - - focusOn({ contactId }); - - const contact = await db.contact.findUnique({ - where: { id: contactId }, - select: { - id: true, - firstName: true, - lastName: true, - email: true, - company: { - select: { id: true, name: true, domain: true, ownerId: true }, - }, - }, - }); - - if (!contact) return { ok: false as const, reason: "No such contact." }; - if (!contact.lastName) { - return { - ok: false as const, - reason: "The contact has no last name yet. Identify them first.", - }; - } - - const domain = - contact.company?.domain ?? - (contact.email ? domainOf(contact.email) : null); - if (!domain) { - return { - ok: false as const, - reason: - "No employer domain to search. Set the company's website first.", - }; - } - - const charge = spend(1); - if (!charge.ok) return { ok: false as const, reason: charge.reason }; - - const result = await findWorkEmail({ - firstName: contact.firstName, - lastName: contact.lastName, - domain, - }); - if (!result.ok) return { ok: false as const, reason: result.reason }; - - const found = result.data; - if (!found.email || found.score < HUNTER.minScore) { - return { - ok: true as const, - email: null, - score: found.score, - note: "Hunter has no address it trusts for this person. Leave the email blank rather than guessing a pattern.", - }; - } - - const author = - contact.company?.ownerId ?? - (await db.user.findFirst({ select: { id: true } }))?.id ?? - null; - - const filled = !contact.email; - if (filled) { - await db.contact.update({ - where: { id: contact.id }, - data: { email: found.email }, - }); - } - - if (author) { - await db.activity.create({ - data: { - type: ActivityType.ENRICHMENT, - subject: filled ? "Work email found" : "Work email candidate", - body: [ - `${found.email} (Hunter score ${found.score}).`, - found.sources.length > 0 - ? `Seen on: ${found.sources.map((s) => s.url).join(", ")}` - : "Hunter gave no public page for it.", - filled ? null : `The record keeps ${contact.email}.`, - ] - .filter(Boolean) - .join(" "), - occurredAt: new Date(), - contactId: contact.id, - companyId: contact.company?.id ?? null, - createdById: author, - meta: { - source: "hunter.io", - endpoint: "email-finder", - score: found.score, - sources: found.sources.map((s) => s.url), - agent: "people-research", - }, - }, - select: { id: true }, - }); - } - - return { - ok: true as const, - email: found.email, - score: found.score, - position: found.position, - sources: found.sources, - filled, - }; - }, -}); diff --git a/apps/agent/test/capabilities.spec.ts b/apps/agent/test/capabilities.spec.ts index 3ef07c423..95b7cc8e4 100644 --- a/apps/agent/test/capabilities.spec.ts +++ b/apps/agent/test/capabilities.spec.ts @@ -12,6 +12,11 @@ import { const KEYS = [ "PERPLEXITY_API_KEY", "HUNTER_API_KEY", + "APOLLO_API_KEY", + "LUSHA_API_KEY", + "DROPCONTACT_API_KEY", + "ZOOMINFO_USERNAME", + "ZOOMINFO_PASSWORD", "BLOB_READ_WRITE_TOKEN", ] as const; diff --git a/apps/agent/test/contact-details.spec.ts b/apps/agent/test/contact-details.spec.ts new file mode 100644 index 000000000..a36ee1bf9 --- /dev/null +++ b/apps/agent/test/contact-details.spec.ts @@ -0,0 +1,382 @@ +import { afterEach, beforeEach, describe, expect, it } from "bun:test"; +import { APOLLO_API_KEY, apolloMatch } from "../agent/lib/apollo"; +import { + type ContactDetails, + configuredProviders, + lookupContactDetails, + type Provider, +} from "../agent/lib/contact-details"; +import { CONTACT_DETAILS } from "../agent/lib/contact-details-config"; +import { + DROPCONTACT_API_KEY, + dropcontactEnrich, +} from "../agent/lib/dropcontact"; +import { LUSHA_API_KEY, lushaPerson } from "../agent/lib/lusha"; +import { + forgetZoomInfoSession, + ZOOMINFO_PASSWORD, + ZOOMINFO_USERNAME, + zoominfoEnrich, +} from "../agent/lib/zoominfo"; + +const KEYS = [ + APOLLO_API_KEY, + LUSHA_API_KEY, + DROPCONTACT_API_KEY, + ZOOMINFO_USERNAME, + ZOOMINFO_PASSWORD, +] as const; + +const saved: Record = {}; +const realFetch = globalThis.fetch; +const requests: { url: URL; method: string; body: string }[] = []; + +const ADA = { + firstName: "Ada", + lastName: "Lovelace", + domain: "example.com", + companyName: "Example", +}; + +function replies(answer: (url: URL) => { status?: number; json: string }) { + globalThis.fetch = (async (input: URL | RequestInfo, init?: RequestInit) => { + const url = new URL(String(input instanceof Request ? input.url : input)); + requests.push({ + url, + method: init?.method ?? "GET", + body: String(init?.body ?? ""), + }); + const { status, json } = answer(url); + return new Response(json, { + status: status ?? 200, + headers: { "content-type": "application/json" }, + }); + }) as typeof fetch; +} + +function details(overrides: Partial): ContactDetails { + return { + provider: "hunter", + email: null, + confidence: 0, + phones: [], + title: null, + linkedinUrl: null, + sources: [], + reference: null, + ...overrides, + }; +} + +function fake( + id: Provider["id"], + enabled: boolean, + find: Provider["find"], +): Provider { + return { id, label: id, keys: [], enabled: () => enabled, find }; +} + +beforeEach(() => { + for (const key of KEYS) { + saved[key] = process.env[key]; + process.env[key] = `${key}-test`; + } + forgetZoomInfoSession(); +}); + +afterEach(() => { + globalThis.fetch = realFetch; + requests.length = 0; + for (const key of KEYS) { + if (saved[key] === undefined) delete process.env[key]; + else process.env[key] = saved[key]; + } +}); + +describe("configuredProviders", () => { + it("keeps the configured order and drops what is off", () => { + const all = [ + fake("zoominfo", true, async () => ({ ok: true, data: details({}) })), + fake("hunter", false, async () => ({ ok: true, data: details({}) })), + fake("lusha", true, async () => ({ ok: true, data: details({}) })), + ]; + + expect(configuredProviders(all).map((p) => p.id)).toEqual([ + "lusha", + "zoominfo", + ]); + }); +}); + +describe("lookupContactDetails", () => { + it("says so when nothing is configured", async () => { + expect(await lookupContactDetails(ADA, [])).toEqual({ + outcome: "unconfigured", + }); + }); + + it("stops at the first confident answer and names what it tried", async () => { + const calls: string[] = []; + const all = [ + fake("hunter", true, async () => { + calls.push("hunter"); + return { + ok: true, + data: details({ email: "a@example.com", confidence: 20 }), + }; + }), + fake("apollo", true, async () => { + calls.push("apollo"); + return { ok: false, reason: "HTTP 500" }; + }), + fake("lusha", true, async () => { + calls.push("lusha"); + return { + ok: true, + data: details({ + provider: "lusha", + email: "ada@example.com", + confidence: 85, + }), + }; + }), + fake("zoominfo", true, async () => { + calls.push("zoominfo"); + return { ok: true, data: details({}) }; + }), + ]; + + const result = await lookupContactDetails(ADA, all); + + expect(result.outcome).toBe("found"); + if (result.outcome !== "found") return; + expect(result.details.email).toBe("ada@example.com"); + expect(result.tried).toEqual(["hunter", "apollo", "lusha"]); + expect(calls).not.toContain("zoominfo"); + }); + + it("accepts a phone-only answer when no address is on offer", async () => { + const all = [ + fake("lusha", true, async () => ({ + ok: true, + data: details({ + provider: "lusha", + phones: [{ number: "+33100000000", type: "work" }], + }), + })), + ]; + + const result = await lookupContactDetails(ADA, all); + + expect(result.outcome).toBe("found"); + }); + + it("reports every reason when nobody is confident", async () => { + const all = [ + fake("hunter", true, async () => ({ + ok: true, + data: details({ email: "a@example.com", confidence: 10 }), + })), + fake("apollo", true, async () => ({ ok: false, reason: "HTTP 401" })), + ]; + + const result = await lookupContactDetails(ADA, all); + + expect(result.outcome).toBe("none"); + if (result.outcome !== "none") return; + expect(result.reasons).toHaveLength(2); + expect(result.reasons[0]).toContain( + `below ${CONTACT_DETAILS.minConfidence}`, + ); + expect(result.reasons[1]).toBe("apollo: HTTP 401"); + }); +}); + +describe("apolloMatch", () => { + it("posts the person and reads email status, title and phones", async () => { + replies(() => ({ + json: JSON.stringify({ + person: { + id: "p1", + email: "ada@example.com", + email_status: "verified", + title: "CTO", + linkedin_url: "https://www.linkedin.com/in/ada", + phone_numbers: [ + { sanitized_number: "+33100000000", type: "work_hq" }, + ], + }, + }), + })); + + const result = await apolloMatch(ADA); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.data).toMatchObject({ + provider: "apollo", + email: "ada@example.com", + confidence: CONTACT_DETAILS.apollo.confidence.verified, + title: "CTO", + reference: "p1", + phones: [{ number: "+33100000000", type: "work_hq" }], + }); + expect(requests[0]?.method).toBe("POST"); + expect(requests[0]?.body).toContain('"domain":"example.com"'); + }); + + it("is a blank, not a failure, when nobody matches", async () => { + replies(() => ({ json: JSON.stringify({ person: null }) })); + + const result = await apolloMatch(ADA); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.data.email).toBeNull(); + expect(result.data.confidence).toBe(0); + }); +}); + +describe("lushaPerson", () => { + it("prefers the work address and keeps the phones", async () => { + replies(() => ({ + json: JSON.stringify({ + data: { + id: 42, + jobTitle: "CTO", + emailAddresses: [ + { email: "ada@personal.test", emailType: "personal" }, + { email: "ada@example.com", emailType: "work" }, + ], + phoneNumbers: [{ number: "+33100000000", phoneType: "direct" }], + socialLinks: { linkedin: "https://www.linkedin.com/in/ada" }, + }, + }), + })); + + const result = await lushaPerson(ADA); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.data).toMatchObject({ + provider: "lusha", + email: "ada@example.com", + confidence: CONTACT_DETAILS.lusha.confidence.work, + reference: "42", + phones: [{ number: "+33100000000", type: "direct" }], + }); + expect(requests[0]?.url.searchParams.get("companyDomain")).toBe( + "example.com", + ); + }); + + it("accepts the wrapped contact shape too", async () => { + replies(() => ({ + json: JSON.stringify({ + data: { contact: { emailAddresses: [{ email: "ada@example.com" }] } }, + }), + })); + + const result = await lushaPerson(ADA); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.data.email).toBe("ada@example.com"); + expect(result.data.confidence).toBe(CONTACT_DETAILS.lusha.confidence.other); + }); +}); + +describe("dropcontactEnrich", () => { + it("submits a batch, then reads the finished row", async () => { + replies((url) => + url.pathname.endsWith("/batch") + ? { json: JSON.stringify({ request_id: "req-1", success: true }) } + : { + json: JSON.stringify({ + success: true, + data: [ + { + email: [ + { + email: "ada@example.com", + qualification: "nominative@pro", + }, + ], + phone: "+33100000000", + job: "CTO", + linkedin: "https://www.linkedin.com/in/ada", + }, + ], + }), + }, + ); + + const result = await dropcontactEnrich(ADA); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.data).toMatchObject({ + provider: "dropcontact", + email: "ada@example.com", + confidence: CONTACT_DETAILS.dropcontact.confidence.nominative, + reference: "req-1", + phones: [{ number: "+33100000000", type: "work" }], + }); + expect(requests.map((r) => r.method)).toEqual(["POST", "GET"]); + expect(requests[1]?.url.pathname).toBe("/batch/req-1"); + }); +}); + +describe("zoominfoEnrich", () => { + it("authenticates once, then enriches with the bearer token", async () => { + replies((url) => + url.pathname === "/authenticate" + ? { json: JSON.stringify({ jwt: "jwt-1" }) } + : { + json: JSON.stringify({ + success: true, + data: { + result: [ + { + data: [ + { + id: 7, + email: "ada@example.com", + directPhone: "+33100000000", + jobTitle: "CTO", + }, + ], + }, + ], + }, + }), + }, + ); + + const first = await zoominfoEnrich(ADA); + const second = await zoominfoEnrich(ADA); + + expect(first.ok).toBe(true); + if (!first.ok) return; + expect(first.data).toMatchObject({ + provider: "zoominfo", + email: "ada@example.com", + confidence: CONTACT_DETAILS.zoominfo.confidence.matched, + reference: "7", + phones: [{ number: "+33100000000", type: "direct" }], + }); + expect(second.ok).toBe(true); + expect( + requests.filter((r) => r.url.pathname === "/authenticate"), + ).toHaveLength(1); + }); + + it("needs both halves of the credential", async () => { + delete process.env[ZOOMINFO_PASSWORD]; + + const result = await zoominfoEnrich(ADA); + + expect(result.ok).toBe(false); + expect(requests).toHaveLength(0); + }); +}); diff --git a/apps/agent/test/hunter.spec.ts b/apps/agent/test/hunter.spec.ts index 45a7774ec..5646f200f 100644 --- a/apps/agent/test/hunter.spec.ts +++ b/apps/agent/test/hunter.spec.ts @@ -10,6 +10,13 @@ const realFetch = globalThis.fetch; const savedKey = process.env[HUNTER_API_KEY]; const requested: string[] = []; +const ADA = { + firstName: "Ada", + lastName: "Lovelace", + domain: "Example.com", + companyName: "Example", +}; + function replies(status: number, json: string) { globalThis.fetch = (async (input: URL | RequestInfo) => { requested.push(String(input instanceof Request ? input.url : input)); @@ -44,17 +51,13 @@ describe("findWorkEmail", () => { it("refuses without a key, before any request", async () => { delete process.env[HUNTER_API_KEY]; - const result = await findWorkEmail({ - firstName: "Ada", - lastName: "Lovelace", - domain: "example.com", - }); + const result = await findWorkEmail(ADA); expect(result.ok).toBe(false); expect(requested).toHaveLength(0); }); - it("returns the address, its score and the pages it was seen on", async () => { + it("returns the address, its confidence and the pages it was seen on", async () => { replies( 200, JSON.stringify({ @@ -78,16 +81,14 @@ describe("findWorkEmail", () => { }), ); - const result = await findWorkEmail({ - firstName: "Ada", - lastName: "Lovelace", - domain: "Example.com", - }); + const result = await findWorkEmail(ADA); expect(result.ok).toBe(true); if (!result.ok) return; + expect(result.data.provider).toBe("hunter"); expect(result.data.email).toBe("ada.lovelace@example.com"); - expect(result.data.score).toBe(92); + expect(result.data.confidence).toBe(92); + expect(result.data.title).toBe("CTO"); expect(result.data.sources.map((s) => s.url)).toEqual([ "https://example.com/team", "https://news.test/ada", @@ -105,26 +106,27 @@ describe("findWorkEmail", () => { JSON.stringify({ data: { email: null, score: null, sources: [] } }), ); - const result = await findWorkEmail({ - firstName: "Ada", - lastName: "Lovelace", - domain: "example.com", - }); + const result = await findWorkEmail(ADA); expect(result).toEqual({ ok: true, - data: { email: null, score: 0, position: null, sources: [] }, + data: { + provider: "hunter", + email: null, + confidence: 0, + phones: [], + title: null, + linkedinUrl: null, + sources: [], + reference: null, + }, }); }); it("reports an HTTP failure as a reason", async () => { replies(429, JSON.stringify({ errors: [] })); - const result = await findWorkEmail({ - firstName: "Ada", - lastName: "Lovelace", - domain: "example.com", - }); + const result = await findWorkEmail(ADA); expect(result).toEqual({ ok: false, reason: "HTTP 429" }); }); diff --git a/apps/agent/turbo.json b/apps/agent/turbo.json index f6a5c6d61..584af362c 100644 --- a/apps/agent/turbo.json +++ b/apps/agent/turbo.json @@ -21,6 +21,11 @@ "DATABASE_URL", "GITHUB_TOKEN", "HUNTER_API_KEY", + "APOLLO_API_KEY", + "LUSHA_API_KEY", + "DROPCONTACT_API_KEY", + "ZOOMINFO_USERNAME", + "ZOOMINFO_PASSWORD", "PERPLEXITY_API_KEY" ] }, @@ -35,6 +40,11 @@ "DATABASE_URL", "GITHUB_TOKEN", "HUNTER_API_KEY", + "APOLLO_API_KEY", + "LUSHA_API_KEY", + "DROPCONTACT_API_KEY", + "ZOOMINFO_USERNAME", + "ZOOMINFO_PASSWORD", "PERPLEXITY_API_KEY" ] }, diff --git a/apps/app/lib/agent-transcript.ts b/apps/app/lib/agent-transcript.ts index 2bd8a00b2..c7d8d00e2 100644 --- a/apps/app/lib/agent-transcript.ts +++ b/apps/app/lib/agent-transcript.ts @@ -64,7 +64,7 @@ const VERBS: ToolVerbs = { resolve_linkedin_profile: "Searched for their LinkedIn profile", get_linkedin_profile: "Read a LinkedIn profile", add_company: "Added a company to the CRM", - find_work_email: "Looked up a work email address", + find_contact_details: "Looked up their work email and phone", gleif_search_entities: "Searched the GLEIF register for a company", gleif_get_entity: "Read a company's GLEIF record", gleif_list_subsidiaries: "Listed a group's subsidiaries from GLEIF", diff --git a/docs/agent.md b/docs/agent.md index 966b64c46..3b310fe41 100644 --- a/docs/agent.md +++ b/docs/agent.md @@ -228,13 +228,23 @@ page, the LEI as a company field when there is one, then the same `brand` and `company-profile` tasks `companyCreated` queues on the API side. `add_company` requires a source; a company without one cannot be created by the agent. -### Work email is a provider, never a guess - -`lib/hunter.ts` (`HUNTER_API_KEY`) finds an address from a name and an employer -domain and hands back the public pages it was seen on. `find_work_email` fills -the email only when the record has none, and always writes the candidate, its -score and its sources to the timeline. Below `HUNTER.minScore` nothing is -written: a pattern guess is worse than a blank. +### Contact details come from a provider, never a guess + +`lib/contact-details.ts` is the registry: one `Provider` contract, one +`fetchJson` that parses every response with Zod at the boundary, and +`lookupContactDetails`, which asks the configured providers in +`CONTACT_DETAILS.order` and stops at the first answer above +`CONTACT_DETAILS.minConfidence`. The providers are one file each — `hunter.ts`, +`apollo.ts`, `lusha.ts`, `dropcontact.ts`, `zoominfo.ts` — and every one is off +without its key. Confidence is the provider's own signal mapped to one scale: +Hunter's score, Apollo's verification status, Lusha's email type, Dropcontact's +qualification, a ZoomInfo match. + +`find_contact_details` fills the email and the phone only where the record has +none, and always writes the candidate, its confidence and its source to the +timeline: the public pages when the provider has them (Hunter), the provider +and its record id otherwise. A blank beats a pattern guess, so nothing is +written below the threshold. ### The GLEIF register needs no key diff --git a/docs/environment.md b/docs/environment.md index f86833a30..77ecca6ff 100644 --- a/docs/environment.md +++ b/docs/environment.md @@ -119,7 +119,11 @@ single place that knows what is set. | Variable | What it adds | | --- | --- | | `PERPLEXITY_API_KEY` | Open-web research with citations; finds a LinkedIn slug | -| `HUNTER_API_KEY` | A work email from name plus employer domain, with the pages it was seen on; a deliverability check | +| `HUNTER_API_KEY` | Contact details: a work email with the pages it was seen on; a deliverability check | +| `APOLLO_API_KEY` | Contact details: work email with verification status, title, work phone | +| `LUSHA_API_KEY` | Contact details: work email, direct and mobile phones, title | +| `DROPCONTACT_API_KEY` | Contact details: work email with qualification, phone, title; asynchronous | +| `ZOOMINFO_USERNAME` + `ZOOMINFO_PASSWORD` | Contact details from ZoomInfo; a pair, both or neither | | `GITHUB_TOKEN` | Raises the GitHub rate limit from 60/hour | | `BLOB_READ_WRITE_TOKEN` | Mirrors logos and photos into Blob | | `AI_GATEWAY_API_KEY` | The model. Not needed on Vercel (OIDC) | diff --git a/packages/telemetry/src/allowlist.ts b/packages/telemetry/src/allowlist.ts index d57d44244..cb901d6d5 100644 --- a/packages/telemetry/src/allowlist.ts +++ b/packages/telemetry/src/allowlist.ts @@ -119,8 +119,8 @@ export const AGENT_TOOLS = [ "archive_field", "enrich_company", "fetch_contact_photo", + "find_contact_details", "find_contact_socials", - "find_work_email", "get_contact_work_history", "get_linkedin_profile", "gleif_get_entity", diff --git a/turbo.json b/turbo.json index 519cfcc08..3cce91f68 100644 --- a/turbo.json +++ b/turbo.json @@ -26,6 +26,11 @@ "PRISMA_LOG_QUERIES", "PERPLEXITY_API_KEY", "HUNTER_API_KEY", + "APOLLO_API_KEY", + "LUSHA_API_KEY", + "DROPCONTACT_API_KEY", + "ZOOMINFO_USERNAME", + "ZOOMINFO_PASSWORD", "GITHUB_TOKEN", "BLOB_READ_WRITE_TOKEN", "AI_GATEWAY_API_KEY", From a8fb7f9de1a8c1dcb8ac8ed95818f084dd9e893c Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 3 Sep 2026 09:30:33 +0000 Subject: [PATCH 13/19] fix(agent): retireExhausted never exceeds its limit Rewrite the retire statement on the shape claimDue uses: the locked, limited selection is a FROM subquery joined on id, not an IN list. Postgres re-evaluates an IN subquery with LIMIT and SKIP LOCKED inside one UPDATE, so retireExhausted(2) sometimes retired three rows. --- apps/agent/agent/lib/tasks.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/agent/agent/lib/tasks.ts b/apps/agent/agent/lib/tasks.ts index 9d8a912c7..94ce8bfcc 100644 --- a/apps/agent/agent/lib/tasks.ts +++ b/apps/agent/agent/lib/tasks.ts @@ -79,7 +79,7 @@ export async function retireExhausted( UPDATE "agentTask" AS t SET "finishedAt" = ${now}, "outcome" = ${RETIRED_OUTCOME} - WHERE t.id IN ( + FROM ( SELECT c.id FROM "agentTask" AS c WHERE c."finishedAt" IS NULL @@ -88,7 +88,8 @@ export async function retireExhausted( ORDER BY c."dueAt" ASC LIMIT ${limit} FOR UPDATE SKIP LOCKED - ) + ) AS exhausted + WHERE t.id = exhausted.id RETURNING t.id, t."contactId", t."companyId", t."dealId", t.kind; `; } From ccaceb65727160d6ce9240e734037da60514daf7 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 3 Sep 2026 10:57:04 +0000 Subject: [PATCH 14/19] feat(agent): the employer's website as the last contact-details provider website.ts reads the company's own contact, team and legal pages under its robots.txt, with a named user agent, a short timeout and a size cap, and looks for the person by name. A named address on the domain is the answer with the pages it was seen on; a switchboard tel: link counts only when the site names the person. It needs no key, is last in the order, and costs nothing: find_contact_details charges a unit only when a keyed provider is configured. --- .../agent/agent/lib/contact-details-config.ts | 22 +- .../agent/lib/contact-details-providers.ts | 2 + apps/agent/agent/lib/website.ts | 312 ++++++++++++++++++ .../agent/skills/gleif-mna-sourcing/SKILL.md | 11 +- .../agent/agent/tools/find_contact_details.ts | 19 +- apps/agent/test/website.spec.ts | 233 +++++++++++++ docs/agent.md | 20 +- docs/environment.md | 4 + 8 files changed, 602 insertions(+), 21 deletions(-) create mode 100644 apps/agent/agent/lib/website.ts create mode 100644 apps/agent/test/website.spec.ts diff --git a/apps/agent/agent/lib/contact-details-config.ts b/apps/agent/agent/lib/contact-details-config.ts index 9e9ba6079..92956270b 100644 --- a/apps/agent/agent/lib/contact-details-config.ts +++ b/apps/agent/agent/lib/contact-details-config.ts @@ -5,7 +5,7 @@ export const CONTACT_DETAILS = { timeoutMs: 20 * SECOND_MS, minConfidence: 50, maxSources: 5, - order: ["hunter", "apollo", "lusha", "dropcontact", "zoominfo"], + order: ["hunter", "apollo", "lusha", "dropcontact", "zoominfo", "website"], hunter: { baseUrl: "https://api.hunter.io/v2", @@ -33,4 +33,24 @@ export const CONTACT_DETAILS = { tokenTtlMs: 55 * MINUTE_MS, confidence: { matched: 80 }, }, + + website: { + paths: [ + "/", + "/contact", + "/team", + "/about", + "/equipe", + "/notre-equipe", + "/a-propos", + "/mentions-legales", + "/legal", + "/impressum", + ], + maxPages: 8, + maxBytes: 512 * 1024, + timeoutMs: 6 * SECOND_MS, + userAgent: "crm-contact-lookup/1 (+https://github.com/trycompai/crm)", + confidence: { named: 75, phoneOnly: 40 }, + }, } as const; diff --git a/apps/agent/agent/lib/contact-details-providers.ts b/apps/agent/agent/lib/contact-details-providers.ts index 8e97497c4..e271146e5 100644 --- a/apps/agent/agent/lib/contact-details-providers.ts +++ b/apps/agent/agent/lib/contact-details-providers.ts @@ -3,6 +3,7 @@ import type { Provider } from "./contact-details"; import { dropcontact } from "./dropcontact"; import { hunter } from "./hunter"; import { lusha } from "./lusha"; +import { website } from "./website"; import { zoominfo } from "./zoominfo"; export const CONTACT_DETAILS_PROVIDERS: readonly Provider[] = [ @@ -11,4 +12,5 @@ export const CONTACT_DETAILS_PROVIDERS: readonly Provider[] = [ lusha, dropcontact, zoominfo, + website, ]; diff --git a/apps/agent/agent/lib/website.ts b/apps/agent/agent/lib/website.ts new file mode 100644 index 000000000..21243662f --- /dev/null +++ b/apps/agent/agent/lib/website.ts @@ -0,0 +1,312 @@ +import type { + ContactDetails, + DetailSource, + Outcome, + Person, + Phone, + Provider, +} from "./contact-details"; +import { CONTACT_DETAILS } from "./contact-details-config"; +import { domainOf } from "./names"; + +const EMAIL = /[a-z0-9._%+-]+@[a-z0-9.-]+\.[a-z]{2,}/gi; +const MAILTO = /href\s*=\s*["']mailto:([^"'?]+)/gi; +const TEL = /href\s*=\s*["']tel:([^"']+)/gi; +const LINKEDIN_ANCHOR = + /]*href\s*=\s*["']([^"']*linkedin\.com\/in\/[^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi; +const OBFUSCATED_AT = /\s*[([{]\s*at\s*[)\]}]\s*/gi; +const OBFUSCATED_DOT = /\s*[([{]\s*dot\s*[)\]}]\s*/gi; +const DROPPED_BLOCKS = /<(script|style|noscript|svg)\b[^>]*>[\s\S]*?<\/\1>/gi; + +export type WebPage = { + url: string; + html: string; +}; + +type Sighting = { + page: WebPage; + namesPerson: boolean; + emails: string[]; + phones: string[]; + linkedinUrl: string | null; +}; + +export function fold(value: string): string { + return value + .normalize("NFD") + .replace(/[\u0300-\u036f]/g, "") + .toLowerCase(); +} + +function letters(value: string): string { + return fold(value).replace(/[^a-z0-9]/g, ""); +} + +function decodeEntities(html: string): string { + return html + .replace(/&#x([0-9a-f]+);/gi, (_, hex: string) => + String.fromCodePoint(Number.parseInt(hex, 16)), + ) + .replace(/&#(\d+);/g, (_, dec: string) => String.fromCodePoint(Number(dec))) + .replace(/&/g, "&") + .replace(/</g, "<") + .replace(/>/g, ">") + .replace(/"/g, '"') + .replace(/ /g, " "); +} + +export function textOf(html: string): string { + const stripped = html.replace(DROPPED_BLOCKS, " ").replace(/<[^>]+>/g, " "); + return decodeEntities(stripped).replace(/\s+/g, " ").trim(); +} + +function onDomain(email: string, domain: string): boolean { + const host = domainOf(email); + return host === domain || Boolean(host?.endsWith(`.${domain}`)); +} + +export function emailsIn(html: string, domain: string): string[] { + const text = textOf(html) + .replace(OBFUSCATED_AT, "@") + .replace(OBFUSCATED_DOT, "."); + const linked = [...html.matchAll(MAILTO)].map((match) => + decodeEntities(match[1] ?? ""), + ); + const written = text.match(EMAIL) ?? []; + + const found = [...linked, ...written] + .map((email) => email.trim().toLowerCase()) + .filter((email) => onDomain(email, domain)); + + return [...new Set(found)]; +} + +export function phonesIn(html: string): string[] { + const found = [...html.matchAll(TEL)].map((match) => + decodeURIComponent(match[1] ?? "").replace(/[^\d+]/g, ""), + ); + return [...new Set(found.filter((number) => number.length >= 6))]; +} + +function fullNameForms(person: Person): string[] { + const first = fold(person.firstName).trim(); + const last = fold(person.lastName).trim(); + return [`${first} ${last}`, `${last} ${first}`].filter( + (form) => form.trim().length > 0, + ); +} + +export function mentionsPerson(html: string, person: Person): boolean { + const text = fold(textOf(html)).replace(/\s+/g, " "); + return fullNameForms(person).some((form) => text.includes(form)); +} + +export function localPartNamesPerson(local: string, person: Person): boolean { + const first = letters(person.firstName); + const last = letters(person.lastName); + const handle = letters(local); + if (!handle || !last) return false; + + const initial = first.slice(0, 1); + const forms = first + ? [ + `${first}${last}`, + `${last}${first}`, + `${initial}${last}`, + `${last}${initial}`, + ] + : [last]; + + return forms.includes(handle); +} + +export function linkedinFor(html: string, person: Person): string | null { + const forms = fullNameForms(person); + for (const match of html.matchAll(LINKEDIN_ANCHOR)) { + const label = fold(textOf(match[2] ?? "")).replace(/\s+/g, " "); + if (forms.some((form) => label.includes(form))) { + return decodeEntities(match[1] ?? ""); + } + } + return null; +} + +export function robotsAllows(robots: string, path: string): boolean { + let applies = false; + let allowed = true; + + for (const raw of robots.split(/\r?\n/)) { + const line = raw.replace(/#.*$/, "").trim(); + if (!line) continue; + + const colon = line.indexOf(":"); + if (colon < 0) continue; + const field = line.slice(0, colon).trim().toLowerCase(); + const value = line.slice(colon + 1).trim(); + + if (field === "user-agent") { + applies = value === "*"; + continue; + } + if (!applies || field !== "disallow" || !value) continue; + if (path.startsWith(value)) allowed = false; + } + + return allowed; +} + +export async function fetchPage(url: URL): Promise> { + const controller = new AbortController(); + const timer = setTimeout( + () => controller.abort(), + CONTACT_DETAILS.website.timeoutMs, + ); + + try { + const response = await fetch(url, { + headers: { + "user-agent": CONTACT_DETAILS.website.userAgent, + accept: "text/html", + }, + redirect: "follow", + signal: controller.signal, + }); + if (!response.ok) return { ok: false, reason: `HTTP ${response.status}` }; + + const type = response.headers.get("content-type") ?? ""; + if (!type.includes("text/html") && !type.includes("text/plain")) { + return { ok: false, reason: `Not a page: ${type || "no content type"}` }; + } + + const html = (await response.text()).slice( + 0, + CONTACT_DETAILS.website.maxBytes, + ); + return { ok: true, data: { url: url.toString(), html } }; + } catch (error) { + const aborted = error instanceof Error && error.name === "AbortError"; + return { + ok: false, + reason: aborted + ? `Timed out after ${CONTACT_DETAILS.website.timeoutMs}ms.` + : error instanceof Error + ? error.message + : String(error), + }; + } finally { + clearTimeout(timer); + } +} + +async function readRobots(base: URL): Promise { + const robots = await fetchPage(new URL("/robots.txt", base)); + return robots.ok ? robots.data.html : ""; +} + +export async function readSite(person: Person): Promise { + const base = new URL(`https://${person.domain.trim().toLowerCase()}`); + const robots = await readRobots(base); + + const paths = CONTACT_DETAILS.website.paths + .filter((path) => robotsAllows(robots, path)) + .slice(0, CONTACT_DETAILS.website.maxPages); + + const pages = await Promise.all( + paths.map((path) => fetchPage(new URL(path, base))), + ); + + return pages.flatMap((page) => (page.ok ? [page.data] : [])); +} + +function sightingOf(page: WebPage, person: Person): Sighting { + return { + page, + namesPerson: mentionsPerson(page.html, person), + emails: emailsIn(page.html, person.domain.trim().toLowerCase()), + phones: phonesIn(page.html), + linkedinUrl: linkedinFor(page.html, person), + }; +} + +function sourceOf(page: WebPage, seenOn: string): DetailSource { + return { url: page.url, domain: new URL(page.url).hostname, seenOn }; +} + +function phonesOf(sightings: Sighting[]): Phone[] { + const numbers = new Set(sightings.flatMap((s) => s.phones)); + return [...numbers].map((number) => ({ number, type: "main" })); +} + +export function detailsFrom(pages: WebPage[], person: Person): ContactDetails { + const seenOn = new Date().toISOString().slice(0, 10); + const sightings = pages.map((page) => sightingOf(page, person)); + const named = sightings.filter((s) => s.namesPerson); + const linkedinUrl = sightings.find((s) => s.linkedinUrl)?.linkedinUrl ?? null; + + const emailPages = new Map(); + for (const sighting of sightings) { + for (const email of sighting.emails) { + const local = email.slice(0, email.lastIndexOf("@")); + const surnameOnly = letters(local) === letters(person.lastName); + const matches = + localPartNamesPerson(local, person) || + (surnameOnly && sighting.namesPerson); + if (!matches) continue; + emailPages.set(email, [...(emailPages.get(email) ?? []), sighting]); + } + } + + const [email, where] = [...emailPages.entries()][0] ?? [null, []]; + if (email) { + return { + provider: "website", + email, + confidence: CONTACT_DETAILS.website.confidence.named, + phones: phonesOf(where.length > 0 ? where : named), + title: null, + linkedinUrl, + sources: where + .slice(0, CONTACT_DETAILS.maxSources) + .map((s) => sourceOf(s.page, seenOn)), + reference: null, + }; + } + + const phones = phonesOf(named.length > 0 ? sightings : []); + return { + provider: "website", + email: null, + confidence: + phones.length > 0 ? CONTACT_DETAILS.website.confidence.phoneOnly : 0, + phones, + title: null, + linkedinUrl, + sources: named + .slice(0, CONTACT_DETAILS.maxSources) + .map((s) => sourceOf(s.page, seenOn)), + reference: null, + }; +} + +export async function findOnWebsite( + person: Person, +): Promise> { + if (!person.lastName.trim()) { + return { ok: false, reason: "No last name to look for." }; + } + + const pages = await readSite(person); + if (pages.length === 0) { + return { ok: false, reason: `No readable page on ${person.domain}.` }; + } + + return { ok: true, data: detailsFrom(pages, person) }; +} + +export const website: Provider = { + id: "website", + label: "Company website", + keys: [], + enabled: () => true, + find: findOnWebsite, +}; diff --git a/apps/agent/agent/skills/gleif-mna-sourcing/SKILL.md b/apps/agent/agent/skills/gleif-mna-sourcing/SKILL.md index 89c20c7e7..6626edd8a 100644 --- a/apps/agent/agent/skills/gleif-mna-sourcing/SKILL.md +++ b/apps/agent/agent/skills/gleif-mna-sourcing/SKILL.md @@ -59,11 +59,12 @@ corners. The rules: - **Expect blanks.** Ten to twenty percent of group leaders and far more local ones have no public profile. Do not fill the gap. -Contact details come only from a provider that carries the compliance of the -source, never from a page you read. `find_contact_details` asks the configured -providers (Hunter, Apollo, Lusha, Dropcontact, ZoomInfo) in order and writes -the answer, its confidence and its source to the contact's timeline. Without -any of them, stop at name, title and public profile. +Contact details come only from `find_contact_details`, never from a page you +read yourself. It asks the configured providers (Hunter, Apollo, Lusha, +Dropcontact, ZoomInfo) in order, then the target's own website — contact, team +and legal pages, under its `robots.txt` — and writes the answer, its confidence +and its source to the contact's timeline. When it returns nothing, stop at +name, title and public profile. ## 5. Deliver diff --git a/apps/agent/agent/tools/find_contact_details.ts b/apps/agent/agent/tools/find_contact_details.ts index b5279688f..392934865 100644 --- a/apps/agent/agent/tools/find_contact_details.ts +++ b/apps/agent/agent/tools/find_contact_details.ts @@ -21,20 +21,13 @@ function describeSources(details: ContactDetails): string { export default defineTool({ description: - "Find a contact's work email and phone from their name and their employer's domain, through the configured contact-data providers in order (Hunter, Apollo, Lusha, Dropcontact, ZoomInfo), stopping at the first confident answer. Fills the email and phone only where the record has none, and always writes the candidate, its confidence and its source to the contact's timeline. Without any provider key the tool says so and nothing is charged.", + "Find a contact's work email and phone from their name and their employer's domain: the configured contact-data providers first (Hunter, Apollo, Lusha, Dropcontact, ZoomInfo), then the employer's own website (contact, team and legal pages), stopping at the first confident answer. Fills the email and phone only where the record has none, and always writes the candidate, its confidence and its source to the contact's timeline. A website read costs nothing; a provider call costs one unit.", inputSchema: z.object({ contactId: z.string(), }), async execute({ contactId }) { const providers = configuredProviders(CONTACT_DETAILS_PROVIDERS); - if (providers.length === 0) { - return { - ok: false as const, - configured: false as const, - reason: - "No contact-data provider is configured on this install (HUNTER_API_KEY, APOLLO_API_KEY, LUSHA_API_KEY, DROPCONTACT_API_KEY or ZOOMINFO_USERNAME with ZOOMINFO_PASSWORD). This is not a failure and retrying will not help — say in your write-up that contact details could not be checked.", - }; - } + const metered = providers.some((provider) => provider.keys.length > 0); focusOn({ contactId }); @@ -71,8 +64,10 @@ export default defineTool({ }; } - const charge = spend(1); - if (!charge.ok) return { ok: false as const, reason: charge.reason }; + if (metered) { + const charge = spend(1); + if (!charge.ok) return { ok: false as const, reason: charge.reason }; + } const lookup = await lookupContactDetails( { @@ -94,7 +89,7 @@ export default defineTool({ phones: [], tried: lookup.tried, reasons: lookup.reasons, - note: "No provider has an address it trusts for this person. Leave the email blank rather than guessing a pattern.", + note: "No provider has an address it trusts for this person, and the employer's website does not name them. Leave the email blank rather than guessing a pattern.", }; } diff --git a/apps/agent/test/website.spec.ts b/apps/agent/test/website.spec.ts new file mode 100644 index 000000000..c6038e034 --- /dev/null +++ b/apps/agent/test/website.spec.ts @@ -0,0 +1,233 @@ +import { afterEach, beforeEach, describe, expect, it } from "bun:test"; +import { configuredProviders } from "../agent/lib/contact-details"; +import { CONTACT_DETAILS } from "../agent/lib/contact-details-config"; +import { CONTACT_DETAILS_PROVIDERS } from "../agent/lib/contact-details-providers"; +import { + detailsFrom, + emailsIn, + findOnWebsite, + localPartNamesPerson, + phonesIn, + robotsAllows, + textOf, + website, +} from "../agent/lib/website"; + +const realFetch = globalThis.fetch; +const requested: string[] = []; + +const ADA = { + firstName: "Ada", + lastName: "Lovelace", + domain: "example.com", + companyName: "Example", +}; + +type Reply = { status?: number; type?: string; body: string }; + +function serves(pages: Record) { + globalThis.fetch = (async (input: URL | RequestInfo) => { + const url = new URL(String(input instanceof Request ? input.url : input)); + requested.push(url.pathname); + const reply = pages[url.pathname] ?? { status: 404, body: "" }; + return new Response(reply.body, { + status: reply.status ?? 200, + headers: { "content-type": reply.type ?? "text/html; charset=utf-8" }, + }); + }) as typeof fetch; +} + +beforeEach(() => { + requested.length = 0; +}); + +afterEach(() => { + globalThis.fetch = realFetch; +}); + +describe("the website provider", () => { + it("is always on, needs no key, and comes last", () => { + expect(website.keys).toEqual([]); + expect(website.enabled()).toBe(true); + expect(CONTACT_DETAILS.order.at(-1)).toBe("website"); + expect(configuredProviders(CONTACT_DETAILS_PROVIDERS).at(-1)?.id).toBe( + "website", + ); + }); +}); + +describe("textOf", () => { + it("drops scripts and tags and decodes entities", () => { + const html = + "

Ada Lovelace & co @ work

"; + expect(textOf(html)).toBe("Ada Lovelace & co @ work"); + }); +}); + +describe("emailsIn", () => { + it("reads mailto links, written addresses and obfuscated ones, on the domain only", () => { + const html = ` + write +

Press: press@example.com

+

Sales: sales [at] example [dot] com

+

Other: someone@elsewhere.org

+

Sub: team@mail.example.com

+ `; + expect(emailsIn(html, "example.com")).toEqual([ + "ada.lovelace@example.com", + "press@example.com", + "sales@example.com", + "team@mail.example.com", + ]); + }); +}); + +describe("phonesIn", () => { + it("reads tel links and keeps only digits and the plus", () => { + const html = + 'callagainno'; + expect(phonesIn(html)).toEqual(["+33123456789"]); + }); +}); + +describe("localPartNamesPerson", () => { + it("accepts the usual name forms and refuses the rest", () => { + expect(localPartNamesPerson("ada.lovelace", ADA)).toBe(true); + expect(localPartNamesPerson("alovelace", ADA)).toBe(true); + expect(localPartNamesPerson("lovelace.a", ADA)).toBe(true); + expect(localPartNamesPerson("LovelaceAda", ADA)).toBe(true); + expect(localPartNamesPerson("ada", ADA)).toBe(false); + expect(localPartNamesPerson("lovelace", ADA)).toBe(false); + expect(localPartNamesPerson("contact", ADA)).toBe(false); + }); + + it("folds accents", () => { + const person = { ...ADA, firstName: "Éloïse", lastName: "Müller" }; + expect(localPartNamesPerson("eloise.muller", person)).toBe(true); + }); +}); + +describe("robotsAllows", () => { + it("honours the wildcard group only", () => { + const robots = [ + "User-agent: Googlebot", + "Disallow: /team", + "", + "User-agent: *", + "Disallow: /private", + "Disallow: /legal # old", + ].join("\n"); + expect(robotsAllows(robots, "/team")).toBe(true); + expect(robotsAllows(robots, "/private/x")).toBe(false); + expect(robotsAllows(robots, "/legal")).toBe(false); + expect(robotsAllows("", "/anything")).toBe(true); + }); +}); + +describe("detailsFrom", () => { + it("returns the named address with the pages it was seen on and the phones there", () => { + const details = detailsFrom( + [ + { + url: "https://example.com/", + html: 'infox', + }, + { + url: "https://example.com/team", + html: '

Ada Lovelace

mailcallAda Lovelace', + }, + ], + ADA, + ); + + expect(details.provider).toBe("website"); + expect(details.email).toBe("ada.lovelace@example.com"); + expect(details.confidence).toBe(CONTACT_DETAILS.website.confidence.named); + expect(details.sources.map((s) => s.url)).toEqual([ + "https://example.com/team", + ]); + expect(details.phones).toEqual([{ number: "+15550199", type: "main" }]); + expect(details.linkedinUrl).toBe("https://www.linkedin.com/in/ada"); + }); + + it("accepts a surname-only address when the page names the person", () => { + const details = detailsFrom( + [ + { + url: "https://example.com/contact", + html: "

Ada Lovelace, directrice. lovelace@example.com

", + }, + ], + ADA, + ); + expect(details.email).toBe("lovelace@example.com"); + }); + + it("gives the switchboard only when the site names the person", () => { + const pages = [ + { + url: "https://example.com/", + html: 'call', + }, + { + url: "https://example.com/team", + html: "

Ada Lovelace

", + }, + ]; + const named = detailsFrom(pages, ADA); + expect(named.email).toBeNull(); + expect(named.phones).toEqual([{ number: "+15550100", type: "main" }]); + expect(named.confidence).toBe(CONTACT_DETAILS.website.confidence.phoneOnly); + expect(named.sources.map((s) => s.url)).toEqual([ + "https://example.com/team", + ]); + + const stranger = detailsFrom(pages, { ...ADA, lastName: "Byron" }); + expect(stranger.email).toBeNull(); + expect(stranger.phones).toEqual([]); + expect(stranger.confidence).toBe(0); + }); +}); + +describe("findOnWebsite", () => { + it("reads robots.txt, skips what it forbids, and ignores pages that are not HTML", async () => { + serves({ + "/robots.txt": { + type: "text/plain", + body: "User-agent: *\nDisallow: /team\n", + }, + "/": { body: "

Example

" }, + "/contact": { + body: '

Ada Lovelace

m', + }, + "/about": { type: "application/pdf", body: "%PDF" }, + }); + + const result = await findOnWebsite(ADA); + + expect(requested).toContain("/robots.txt"); + expect(requested).toContain("/contact"); + expect(requested).not.toContain("/team"); + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.data.email).toBe("a.lovelace@example.com"); + expect(result.data.sources.map((s) => s.url)).toEqual([ + "https://example.com/contact", + ]); + }); + + it("reports a site with no readable page as a reason, not a throw", async () => { + serves({}); + expect(await findOnWebsite(ADA)).toEqual({ + ok: false, + reason: "No readable page on example.com.", + }); + }); + + it("refuses without a last name, before any request", async () => { + serves({ "/": { body: "

x

" } }); + const result = await findOnWebsite({ ...ADA, lastName: " " }); + expect(result.ok).toBe(false); + expect(requested).toHaveLength(0); + }); +}); diff --git a/docs/agent.md b/docs/agent.md index 3b310fe41..6b1415abb 100644 --- a/docs/agent.md +++ b/docs/agent.md @@ -240,11 +240,25 @@ without its key. Confidence is the provider's own signal mapped to one scale: Hunter's score, Apollo's verification status, Lusha's email type, Dropcontact's qualification, a ZoomInfo match. +`website.ts` is the last provider and needs no key: it reads the employer's own +site — the paths in `CONTACT_DETAILS.website.paths`, fetched together, after +`robots.txt`, with a named user agent, a short timeout and a size cap — and +looks for the person by name. An address on the domain whose local part is a +form of the name (`ada.lovelace`, `alovelace`, `lovelacea`) is the answer at +`confidence.named`; a surname-only address counts when the same page names the +person; a `tel:` link on a site that names the person is the switchboard, typed +`main`, at `confidence.phoneOnly`. A `linkedin.com/in/` link is reported only +when the site itself labels it with the person's name — the site is read, never +LinkedIn. Nothing is charged for it. The sources are the pages the address was +seen on, with the day it was read, so the timeline entry reads the same as +Hunter's. + `find_contact_details` fills the email and the phone only where the record has none, and always writes the candidate, its confidence and its source to the -timeline: the public pages when the provider has them (Hunter), the provider -and its record id otherwise. A blank beats a pattern guess, so nothing is -written below the threshold. +timeline: the public pages when the provider has them (Hunter, the website), +the provider and its record id otherwise. A blank beats a pattern guess, so +nothing is written below the threshold. One unit is charged when a keyed +provider is configured; a website-only lookup is free. ### The GLEIF register needs no key diff --git a/docs/environment.md b/docs/environment.md index 77ecca6ff..862d2d134 100644 --- a/docs/environment.md +++ b/docs/environment.md @@ -129,6 +129,10 @@ single place that knows what is set. | `AI_GATEWAY_API_KEY` | The model. Not needed on Vercel (OIDC) | | `AGENT_BRIDGE_SECRET` | The rep-facing Agent panel — see `agent.md` | +The employer's own website is a contact-details source with no variable at all: +`find_contact_details` reads it after the keyed providers, under its `robots.txt`, +and charges nothing for it. `agent.md` has the rules. + `BLOB_READ_WRITE_TOKEN` is also in `env.validation.ts` and `apps/api/turbo.json` because the API and the seed write pictures too. The Next.js app is deliberately excluded — recognising our URL for the image optimizer needs no token. From 5ca8375865a6b23ca0fe2bf40882df407f4261d6 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 3 Sep 2026 14:18:19 +0000 Subject: [PATCH 15/19] feat(agent): SEC EDGAR research through an external edgartools service services/edgar is a FastAPI service on edgartools, the first external service the CRM talks to: EDGAR_URL, a bearer EDGAR_SECRET, /health, JSON parsed with Zod on the CRM side. Docker Compose runs it; a Colab notebook or another machine can run it behind a tunnel. It answers company profiles, filings, full-text search, 5%+ holders from Schedule 13D/13G, insider transactions from Forms 3/4/5, the latest DEF 14A with its executives and their pay, and a CEO-pay comparison. Eight free sec_* tools read it. add_company keeps a CIK, a ticker and a SIC as fields the way it keeps a LEI. add_contact is the one agent-side contact write path, so executives named in a filing become contacts with the filing as their source. The sec-us-research skill is the method. --- .env.example | 16 + apps/agent/agent/lib/capabilities.ts | 6 + apps/agent/agent/lib/companies.ts | 56 ++- apps/agent/agent/lib/contacts.ts | 163 +++++++ apps/agent/agent/lib/edgar-config.ts | 32 ++ apps/agent/agent/lib/edgar.ts | 224 +++++++++ .../agent/skills/sec-us-research/SKILL.md | 77 +++ apps/agent/agent/tools/add_company.ts | 29 +- apps/agent/agent/tools/add_contact.ts | 35 ++ .../agent/tools/sec_compare_compensation.ts | 31 ++ apps/agent/agent/tools/sec_get_company.ts | 41 ++ apps/agent/agent/tools/sec_get_proxy.ts | 40 ++ apps/agent/agent/tools/sec_list_filings.ts | 50 ++ apps/agent/agent/tools/sec_list_insiders.ts | 39 ++ apps/agent/agent/tools/sec_list_owners.ts | 49 ++ .../agent/agent/tools/sec_search_companies.ts | 40 ++ apps/agent/agent/tools/sec_search_filings.ts | 53 ++ apps/agent/test/capabilities.spec.ts | 1 + apps/agent/test/companies.integration.spec.ts | 52 +- apps/agent/test/contacts.integration.spec.ts | 132 +++++ apps/agent/test/edgar.spec.ts | 247 ++++++++++ apps/agent/turbo.json | 8 +- apps/app/lib/agent-transcript.ts | 9 + docker-compose.yml | 18 + docs/agent.md | 27 + docs/environment.md | 13 + docs/setup.md | 9 +- packages/telemetry/src/allowlist.ts | 9 + packages/validation/package.json | 1 + packages/validation/src/edgar.ts | 202 ++++++++ services/edgar/.dockerignore | 6 + services/edgar/Dockerfile | 26 + services/edgar/README.md | 87 ++++ services/edgar/colab.ipynb | 54 ++ services/edgar/edgar_service/__init__.py | 1 + .../__pycache__/__init__.cpython-311.pyc | Bin 0 -> 180 bytes .../__pycache__/app.cpython-311.pyc | Bin 0 -> 8471 bytes .../__pycache__/auth.cpython-311.pyc | Bin 0 -> 1515 bytes .../__pycache__/config.cpython-311.pyc | Bin 0 -> 1833 bytes .../__pycache__/sec.cpython-311.pyc | Bin 0 -> 31070 bytes .../__pycache__/values.cpython-311.pyc | Bin 0 -> 3840 bytes services/edgar/edgar_service/app.py | 117 +++++ services/edgar/edgar_service/auth.py | 24 + services/edgar/edgar_service/config.py | 33 ++ services/edgar/edgar_service/sec.py | 460 ++++++++++++++++++ services/edgar/edgar_service/values.py | 62 +++ services/edgar/pyproject.toml | 24 + services/edgar/requirements.txt | 4 + services/edgar/tests/__init__.py | 0 .../__pycache__/__init__.cpython-311.pyc | Bin 0 -> 156 bytes .../conftest.cpython-311-pytest-9.1.1.pyc | Bin 0 -> 1717 bytes .../test_app.cpython-311-pytest-9.1.1.pyc | Bin 0 -> 16648 bytes .../test_sec.cpython-311-pytest-9.1.1.pyc | Bin 0 -> 25178 bytes services/edgar/tests/conftest.py | 21 + services/edgar/tests/smoke_live.py | 37 ++ services/edgar/tests/test_app.py | 67 +++ services/edgar/tests/test_sec.py | 163 +++++++ turbo.json | 3 + 58 files changed, 2881 insertions(+), 17 deletions(-) create mode 100644 apps/agent/agent/lib/contacts.ts create mode 100644 apps/agent/agent/lib/edgar-config.ts create mode 100644 apps/agent/agent/lib/edgar.ts create mode 100644 apps/agent/agent/skills/sec-us-research/SKILL.md create mode 100644 apps/agent/agent/tools/add_contact.ts create mode 100644 apps/agent/agent/tools/sec_compare_compensation.ts create mode 100644 apps/agent/agent/tools/sec_get_company.ts create mode 100644 apps/agent/agent/tools/sec_get_proxy.ts create mode 100644 apps/agent/agent/tools/sec_list_filings.ts create mode 100644 apps/agent/agent/tools/sec_list_insiders.ts create mode 100644 apps/agent/agent/tools/sec_list_owners.ts create mode 100644 apps/agent/agent/tools/sec_search_companies.ts create mode 100644 apps/agent/agent/tools/sec_search_filings.ts create mode 100644 apps/agent/test/contacts.integration.spec.ts create mode 100644 apps/agent/test/edgar.spec.ts create mode 100644 packages/validation/src/edgar.ts create mode 100644 services/edgar/.dockerignore create mode 100644 services/edgar/Dockerfile create mode 100644 services/edgar/README.md create mode 100644 services/edgar/colab.ipynb create mode 100644 services/edgar/edgar_service/__init__.py create mode 100644 services/edgar/edgar_service/__pycache__/__init__.cpython-311.pyc create mode 100644 services/edgar/edgar_service/__pycache__/app.cpython-311.pyc create mode 100644 services/edgar/edgar_service/__pycache__/auth.cpython-311.pyc create mode 100644 services/edgar/edgar_service/__pycache__/config.cpython-311.pyc create mode 100644 services/edgar/edgar_service/__pycache__/sec.cpython-311.pyc create mode 100644 services/edgar/edgar_service/__pycache__/values.cpython-311.pyc create mode 100644 services/edgar/edgar_service/app.py create mode 100644 services/edgar/edgar_service/auth.py create mode 100644 services/edgar/edgar_service/config.py create mode 100644 services/edgar/edgar_service/sec.py create mode 100644 services/edgar/edgar_service/values.py create mode 100644 services/edgar/pyproject.toml create mode 100644 services/edgar/requirements.txt create mode 100644 services/edgar/tests/__init__.py create mode 100644 services/edgar/tests/__pycache__/__init__.cpython-311.pyc create mode 100644 services/edgar/tests/__pycache__/conftest.cpython-311-pytest-9.1.1.pyc create mode 100644 services/edgar/tests/__pycache__/test_app.cpython-311-pytest-9.1.1.pyc create mode 100644 services/edgar/tests/__pycache__/test_sec.cpython-311-pytest-9.1.1.pyc create mode 100644 services/edgar/tests/conftest.py create mode 100644 services/edgar/tests/smoke_live.py create mode 100644 services/edgar/tests/test_app.py create mode 100644 services/edgar/tests/test_sec.py diff --git a/.env.example b/.env.example index 05669d5da..a2a870821 100644 --- a/.env.example +++ b/.env.example @@ -136,6 +136,22 @@ GOOGLE_CLIENT_SECRET="" # Hunter, Apollo, Lusha, Dropcontact, ZoomInfo. Every answer is written to # the contact's timeline with its confidence and its source. # +# SEC EDGAR research — US public companies from their SEC filings: profile, +# filings, 5%+ shareholders, insider transactions, the proxy statement with +# its executives and their pay. Served by services/edgar (Python, edgartools) +# and free. `docker compose up -d` runs it on port 2100; a Colab notebook or +# another machine can run it too, see services/edgar/README.md. Without +# EDGAR_URL the sec_* tools say the source is unavailable. +# EDGAR_URL="http://127.0.0.1:2100" +# +# Shared secret between the CRM and the service. Optional on loopback; set it +# whenever the service is reachable from elsewhere. +# EDGAR_SECRET="" +# +# The SEC requires every automated client to identify itself. Any real email. +# Read by the service, not by the CRM. +# EDGAR_IDENTITY="Jane Doe jane@example.com" +# # Hunter — a work email with the public pages it was seen on, and a # deliverability check. The free plan covers a few dozen lookups a month. # https://hunter.io/api-keys diff --git a/apps/agent/agent/lib/capabilities.ts b/apps/agent/agent/lib/capabilities.ts index 1c67716ce..a0e4ea97c 100644 --- a/apps/agent/agent/lib/capabilities.ts +++ b/apps/agent/agent/lib/capabilities.ts @@ -100,6 +100,12 @@ export function capabilitiesFrom( process.env.ZOOMINFO_PASSWORD?.trim(), ), }, + { + ...fromEnv("EDGAR_URL"), + label: "SEC EDGAR research", + gives: + "US public companies from SEC filings: profile, filings, 5%+ shareholders, insider transactions, the proxy statement with its executives and their pay, all free and with a filing URL to cite", + }, { ...fromEnv("BLOB_READ_WRITE_TOKEN"), label: "Picture storage", diff --git a/apps/agent/agent/lib/companies.ts b/apps/agent/agent/lib/companies.ts index 9aa37f8b5..90d458070 100644 --- a/apps/agent/agent/lib/companies.ts +++ b/apps/agent/agent/lib/companies.ts @@ -6,6 +6,21 @@ import { hostOf } from "./names"; import { scheduleTask } from "./tasks"; export const LEI_FIELD_LABEL = "LEI"; +export const CIK_FIELD_LABEL = "CIK"; +export const TICKER_FIELD_LABEL = "TICKER"; +export const SIC_FIELD_LABEL = "SIC"; + +const IDENTIFIER_BRIEFS = { + [LEI_FIELD_LABEL]: + "The 20-character Legal Entity Identifier from the GLEIF register.", + [CIK_FIELD_LABEL]: + "The SEC Central Index Key of a company that files with EDGAR.", + [TICKER_FIELD_LABEL]: "The stock ticker of a listed company.", + [SIC_FIELD_LABEL]: + "The four-digit Standard Industrial Classification code the SEC assigns.", +} as const; + +type IdentifierLabel = keyof typeof IDENTIFIER_BRIEFS; export type CompanySource = { label: string; url: string }; @@ -16,6 +31,10 @@ export type NewCompany = { country?: string | null; city?: string | null; lei?: string | null; + cik?: string | null; + ticker?: string | null; + sic?: string | null; + stateCode?: string | null; source: CompanySource; }; @@ -65,19 +84,20 @@ async function authorId(): Promise { return user?.id ?? null; } -async function recordLei(companyId: string, lei: string): Promise { +async function recordIdentifier( + companyId: string, + label: IdentifierLabel, + value: string, +): Promise { const fields = await listFields("COMPANY"); - const existing = fields.find( - (field) => field.label.toUpperCase() === LEI_FIELD_LABEL, - ); + const existing = fields.find((field) => field.label.toUpperCase() === label); const key = existing ? existing.key : await createField({ entity: "COMPANY", - label: LEI_FIELD_LABEL, + label: label === TICKER_FIELD_LABEL ? "Ticker" : label, type: "TEXT", - agentBrief: - "The 20-character Legal Entity Identifier from the GLEIF register.", + agentBrief: IDENTIFIER_BRIEFS[label], }).then((field) => ("created" in field ? null : field.key)); if (key) @@ -85,16 +105,27 @@ async function recordLei(companyId: string, lei: string): Promise { entity: "COMPANY", recordId: companyId, key, - value: lei, + value, }); } +function identifiersOf(input: NewCompany): [IdentifierLabel, string][] { + const pairs: [IdentifierLabel, string | null | undefined][] = [ + [LEI_FIELD_LABEL, input.lei?.trim().toUpperCase()], + [CIK_FIELD_LABEL, input.cik?.trim().replace(/^0+(?=\d)/, "")], + [TICKER_FIELD_LABEL, input.ticker?.trim().toUpperCase()], + [SIC_FIELD_LABEL, input.sic?.trim()], + ]; + return pairs.flatMap(([label, value]) => (value ? [[label, value]] : [])); +} + export async function createCompany( input: NewCompany, ): Promise { const name = input.name.trim(); const domain = domainFrom(input.website); - const countryCode = input.countryCode?.trim().toUpperCase() || null; + const countryCode = + input.countryCode?.trim().toUpperCase() || (input.cik ? "US" : null); const existing = await existingCompany(name, domain, countryCode); if (existing) { @@ -117,6 +148,7 @@ export async function createCompany( countryCode, country: input.country?.trim() || null, city: input.city?.trim() || null, + stateCode: input.stateCode?.trim().toUpperCase() || null, }, select: { id: true, name: true, domain: true }, }); @@ -150,7 +182,7 @@ export async function createCompany( subject: `Added from ${input.source.label}`, body: [ `${created.name} was added by the agent from ${input.source.label}.`, - input.lei ? `LEI ${input.lei.trim().toUpperCase()}.` : null, + ...identifiersOf(input).map(([label, value]) => `${label} ${value}.`), `Source: ${input.source.url}`, ] .filter(Boolean) @@ -168,7 +200,9 @@ export async function createCompany( }); } - if (input.lei) await recordLei(created.id, input.lei.trim().toUpperCase()); + for (const [label, value] of identifiersOf(input)) { + await recordIdentifier(created.id, label, value); + } await scheduleTask({ companyId: created.id, diff --git a/apps/agent/agent/lib/contacts.ts b/apps/agent/agent/lib/contacts.ts new file mode 100644 index 000000000..ccdb706c3 --- /dev/null +++ b/apps/agent/agent/lib/contacts.ts @@ -0,0 +1,163 @@ +import { ActivityType, db, type Prisma } from "@crm/db"; +import { PRIORITY } from "@crm/db/agent-tasks"; +import type { CompanySource } from "./companies"; + +export type NewContact = { + firstName: string; + lastName?: string | null; + title?: string | null; + email?: string | null; + companyId: string; + source: CompanySource; +}; + +export type CreatedContact = { + created: boolean; + id: string; + firstName: string; + lastName: string | null; + companyId: string | null; + reason?: string; +}; + +const select = { + id: true, + firstName: true, + lastName: true, + companyId: true, +}; + +async function existingContact( + input: NewContact, + email: string | null, +): Promise { + if (email) { + const byEmail = await db.contact.findFirst({ + where: { + email: { equals: email, mode: "insensitive" }, + archivedAt: null, + }, + select, + }); + if (byEmail) return { created: false, ...byEmail }; + } + + const where: Prisma.ContactWhereInput = { + firstName: { equals: input.firstName.trim(), mode: "insensitive" }, + companyId: input.companyId, + archivedAt: null, + }; + const lastName = input.lastName?.trim(); + if (lastName) where.lastName = { equals: lastName, mode: "insensitive" }; + + const byName = await db.contact.findFirst({ where, select }); + return byName ? { created: false, ...byName } : null; +} + +export async function createContact( + input: NewContact, +): Promise { + const company = await db.company.findFirst({ + where: { id: input.companyId, archivedAt: null }, + select: { id: true, name: true, ownerId: true }, + }); + if (!company) { + return { + created: false, + id: "", + firstName: input.firstName, + lastName: input.lastName ?? null, + companyId: null, + reason: "No such company. Add the company first.", + }; + } + + const email = input.email?.trim().toLowerCase() || null; + const existing = await existingContact(input, email); + if (existing) { + return { + ...existing, + reason: `${existing.firstName} ${existing.lastName ?? ""}` + .trim() + .concat(" is already in the CRM."), + }; + } + + const occurredAt = new Date(); + const created = await db.$transaction(async (tx) => { + const contact = await tx.contact.create({ + data: { + firstName: input.firstName.trim(), + lastName: input.lastName?.trim() || null, + title: input.title?.trim() || null, + email, + companyId: company.id, + ownerId: company.ownerId, + }, + select, + }); + + const payload: Prisma.InputJsonObject = { + type: "contact.created", + record: { kind: "contact", id: contact.id }, + occurredAt: occurredAt.toISOString(), + data: { + firstName: contact.firstName, + lastName: contact.lastName, + companyId: company.id, + }, + }; + await tx.agentTask.create({ + data: { + contactId: contact.id, + companyId: company.id, + kind: "agent-event", + reason: "contact.created", + payload, + priority: PRIORITY.event, + budget: 1, + dueAt: occurredAt, + }, + }); + + return contact; + }); + + const author = + company.ownerId ?? + ( + await db.user.findFirst({ + orderBy: { createdAt: "asc" }, + select: { id: true }, + }) + )?.id ?? + null; + if (author) { + await db.activity.create({ + data: { + type: ActivityType.ENRICHMENT, + subject: `Added from ${input.source.label}`, + body: [ + `${created.firstName} ${created.lastName ?? ""}`.trim(), + input.title?.trim() ? `(${input.title.trim()})` : null, + `was added by the agent to ${company.name} from ${input.source.label}.`, + `Source: ${input.source.url}`, + ] + .filter(Boolean) + .join(" "), + occurredAt, + contactId: created.id, + companyId: company.id, + createdById: author, + meta: { + source: input.source.label, + sourceUrl: input.source.url, + agent: "sourcing", + }, + }, + select: { id: true }, + }); + } + + return { created: true, ...created }; +} diff --git a/apps/agent/agent/lib/edgar-config.ts b/apps/agent/agent/lib/edgar-config.ts new file mode 100644 index 000000000..4979a1946 --- /dev/null +++ b/apps/agent/agent/lib/edgar-config.ts @@ -0,0 +1,32 @@ +const SECOND_MS = 1_000; + +export const EDGAR = { + env: { + url: "EDGAR_URL", + secret: "EDGAR_SECRET", + }, + timeoutMs: 45 * SECOND_MS, + search: { + defaultLimit: 10, + maxLimit: 25, + }, + filings: { + defaultLimit: 20, + maxLimit: 100, + }, + owners: { + minPercent: 5, + defaultLimit: 20, + maxLimit: 50, + }, + insiders: { + defaultLimit: 20, + maxLimit: 100, + }, + compensation: { + years: 3, + maxYears: 5, + maxTickers: 10, + }, + browseUrl: "https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK=", +} as const; diff --git a/apps/agent/agent/lib/edgar.ts b/apps/agent/agent/lib/edgar.ts new file mode 100644 index 000000000..3476323a6 --- /dev/null +++ b/apps/agent/agent/lib/edgar.ts @@ -0,0 +1,224 @@ +import { + edgarCompany, + edgarCompanySearch, + edgarCompensationComparison, + edgarFilingSearch, + edgarFilings, + edgarHealth, + edgarInsiders, + edgarOwners, + edgarProxy, +} from "@crm/validation/edgar"; +import type { z } from "zod"; +import { EDGAR } from "./edgar-config"; + +export type Outcome = { ok: true; data: T } | { ok: false; reason: string }; + +type Query = Record; + +export function edgarUrl(): string | null { + const url = process.env[EDGAR.env.url]?.trim(); + return url ? url.replace(/\/+$/, "") : null; +} + +export function edgarEnabled(): boolean { + return edgarUrl() !== null; +} + +function headers(): Record { + const secret = process.env[EDGAR.env.secret]?.trim(); + return secret + ? { accept: "application/json", authorization: `Bearer ${secret}` } + : { accept: "application/json" }; +} + +export function companyUrl(cik: string): string { + return `${EDGAR.browseUrl}${cik}`; +} + +async function request( + path: string, + query: Query, + shape: Shape, + notFound: z.infer | null, +): Promise>> { + const base = edgarUrl(); + if (!base) return { ok: false, reason: `No ${EDGAR.env.url}.` }; + + const url = new URL(`${base}${path}`); + for (const [key, value] of Object.entries(query)) { + if (value !== undefined) url.searchParams.set(key, String(value)); + } + + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), EDGAR.timeoutMs); + + try { + const response = await fetch(url, { + headers: headers(), + signal: controller.signal, + }); + + if (response.status === 404) { + if (notFound !== null) return { ok: true, data: notFound }; + return { ok: false, reason: await reasonOf(response) }; + } + if (!response.ok) { + return { ok: false, reason: `HTTP ${response.status}` }; + } + + const parsed = shape.safeParse(await response.json()); + return parsed.success + ? { ok: true, data: parsed.data } + : { + ok: false, + reason: `Unreadable EDGAR service response: ${parsed.error.message}`, + }; + } catch (error) { + const aborted = error instanceof Error && error.name === "AbortError"; + return { + ok: false, + reason: aborted + ? `The EDGAR service timed out after ${EDGAR.timeoutMs}ms.` + : error instanceof Error + ? error.message + : String(error), + }; + } finally { + clearTimeout(timer); + } +} + +async function reasonOf(response: Response): Promise { + const text = await response.text(); + try { + const parsed: { reason?: string } = JSON.parse(text); + return parsed.reason ?? "Not found."; + } catch { + return text || "Not found."; + } +} + +function normalizeCik(value: string): string { + return value.trim().replace(/^0+(?=\d)/, ""); +} + +export function health() { + return request("/health", {}, edgarHealth, null); +} + +export function searchCompanies(input: { query: string; limit?: number }) { + return request( + "/companies/search", + { q: input.query.trim(), limit: input.limit ?? EDGAR.search.defaultLimit }, + edgarCompanySearch, + { companies: [] }, + ); +} + +export function getCompany(input: { cik?: string; ticker?: string }) { + const key = input.cik + ? normalizeCik(input.cik) + : (input.ticker?.trim().toUpperCase() ?? ""); + return request( + `/companies/${encodeURIComponent(key)}`, + {}, + edgarCompany, + null, + ); +} + +export function listFilings(input: { + cik: string; + form?: string; + from?: string; + to?: string; + limit?: number; +}) { + return request( + `/companies/${encodeURIComponent(normalizeCik(input.cik))}/filings`, + { + form: input.form?.trim() || undefined, + from: input.from, + to: input.to, + limit: input.limit ?? EDGAR.filings.defaultLimit, + }, + edgarFilings, + { filings: [], truncated: false }, + ); +} + +export function searchFilings(input: { + query: string; + form?: string; + from?: string; + to?: string; + limit?: number; +}) { + return request( + "/filings/search", + { + q: input.query.trim(), + form: input.form?.trim() || undefined, + from: input.from, + to: input.to, + limit: input.limit ?? EDGAR.filings.defaultLimit, + }, + edgarFilingSearch, + { filings: [], total: 0 }, + ); +} + +export function listOwners(input: { + cik: string; + minPercent?: number; + form?: string; + limit?: number; +}) { + return request( + `/companies/${encodeURIComponent(normalizeCik(input.cik))}/owners`, + { + minPercent: input.minPercent ?? EDGAR.owners.minPercent, + form: input.form, + limit: input.limit ?? EDGAR.owners.defaultLimit, + }, + edgarOwners, + { owners: [], filingsRead: 0 }, + ); +} + +export function listInsiders(input: { cik: string; limit?: number }) { + return request( + `/companies/${encodeURIComponent(normalizeCik(input.cik))}/insiders`, + { limit: input.limit ?? EDGAR.insiders.defaultLimit }, + edgarInsiders, + { transactions: [] }, + ); +} + +export function getProxy(input: { cik: string; years?: number }) { + return request( + `/companies/${encodeURIComponent(normalizeCik(input.cik))}/proxy`, + { years: input.years ?? EDGAR.compensation.years }, + edgarProxy, + null, + ); +} + +export function compareCompensation(input: { + tickers: readonly string[]; + years?: number; +}) { + return request( + "/compensation/compare", + { + tickers: input.tickers + .map((ticker) => ticker.trim().toUpperCase()) + .filter(Boolean) + .join(","), + years: input.years ?? EDGAR.compensation.years, + }, + edgarCompensationComparison, + { rows: [] }, + ); +} diff --git a/apps/agent/agent/skills/sec-us-research/SKILL.md b/apps/agent/agent/skills/sec-us-research/SKILL.md new file mode 100644 index 000000000..c8ff5a3fd --- /dev/null +++ b/apps/agent/agent/skills/sec-us-research/SKILL.md @@ -0,0 +1,77 @@ +--- +description: Use when asked about a US public company, its SEC filings, who runs it and what they are paid, who its large shareholders are, or to find and import US listed targets — SEC EDGAR through the edgar service, with a filing URL on every line. +--- + +# SEC EDGAR research + +Every US public company files with the SEC, and every filing is public and +dated. The `sec_*` tools read them through the edgar service. They are free, +they need no key, and each answer carries the filing URL: that URL is the +source you cite, and a claim without one does not go in the CRM. + +## 1. Identify the company + +`sec_search_companies` with the name, the ticker or the CIK, then +`sec_get_company` on the match. The CIK is the key for everything after. A +name search can return a fund or a subsidiary with a similar name; check the +SIC description and the state before going on. If the tools answer +"unavailable", the service is not configured here: say so and stop. + +## 2. Read the company + +- `sec_get_company` gives the profile: legal name, tickers, SIC and industry, + state of incorporation, fiscal year end, business address, former names. +- `sec_list_filings` with a form for the documents that matter: `10-K` for + the annual report, `8-K` for events, `DEF 14A` for governance and pay. +- `sec_search_filings` across all companies when the question is "who + mentions X": a product, a customer, a competitor, a technology. + +## 3. Who owns it + +`sec_list_owners` lists holders of 5% or more from Schedule 13D and 13G. +13D is an active holder with intent; 13G is passive. Holdings are as of the +filing date. A holder below 5% never files, so absence is not zero. + +## 4. Who runs it, and what they are paid + +`sec_get_proxy` reads the latest DEF 14A: the named executives with titles +and the summary compensation table, the CEO's pay and pay actually paid with +the NEO average over the last years, pay versus performance, the holders the +proxy lists, the proposals, the CEO pay ratio. `sec_list_insiders` names +officers and directors with their titles from Forms 3/4/5 and shows recent +buys, sells and grants. `sec_compare_compensation` puts several tickers side +by side. + +Titles in the proxy are the company's own words; keep them. A name written +"Mr. Cook" in the pay-versus-performance table is the same person as "Tim +Cook" in the compensation table; use the full name. + +## 5. Deliver + +Write the answer as a table where it fits: company, CIK, ticker, SIC, state, +CEO, total pay, pay actually paid, TSR, 5%+ holders with percent. Under it, +the filing each figure came from, with its date. Blanks stay blanks. + +## 6. Import into the CRM + +When the rep wants the company in the CRM, `add_company` with the name, the +website when the profile has one, `countryCode` US, the state as +`stateCode`, the city, the `cik`, the first `ticker` and the `sic`. The source +is the EDGAR page `sec_get_company` returns as `sourceUrl`. The tool returns +the existing company when there is one. + +Executives go on with `add_contact` on that company: first name, last name, +the title as the proxy states it, and the proxy's filing URL as the source. +Then `record_fact` for the title with `web.cited-claim` and the same URL. A +5%+ holder is an institution, not a person: name it in the write-up, do not +add it as a contact. + +## Rules + +- **The filing URL is the source.** No filing, no line. +- **Never fetch linkedin.com** and never invent a profile URL; the people + pipeline finds profiles its own way after `add_contact`. +- **Numbers are the filing's numbers.** No conversion, no rounding beyond + what you show, the fiscal year end next to every figure. +- **A parse gap is a blank.** When the proxy carries no readable table, say + the figure is not machine-readable in that filing. diff --git a/apps/agent/agent/tools/add_company.ts b/apps/agent/agent/tools/add_company.ts index 62e8ec197..5f370db28 100644 --- a/apps/agent/agent/tools/add_company.ts +++ b/apps/agent/agent/tools/add_company.ts @@ -4,7 +4,7 @@ import { createCompany } from "../lib/companies"; export default defineTool({ description: - "Add a company to the CRM with the source it came from. Use it for a sourcing target or any company a rep asks for that search_crm cannot find. A company that already exists, by domain or by name in the same country, is returned rather than duplicated. The source is written to the company's timeline, a company.created event fires, and the brand and profile enrichment queue up on their own. Free.", + "Add a company to the CRM with the source it came from. Use it for a sourcing target or any company a rep asks for that search_crm cannot find. A company that already exists, by domain or by name in the same country, is returned rather than duplicated. The source is written to the company's timeline, a company.created event fires, and the brand and profile enrichment queue up on their own. A LEI, a CIK, a ticker and a SIC code are kept as custom fields. Free.", inputSchema: z.object({ name: z .string() @@ -35,6 +35,33 @@ export default defineTool({ .length(20) .optional() .describe("The Legal Entity Identifier, when it came from GLEIF."), + cik: z + .string() + .trim() + .regex(/^\d{1,10}$/) + .optional() + .describe( + "The SEC Central Index Key, when it came from EDGAR. '320193'.", + ), + ticker: z + .string() + .trim() + .min(1) + .max(6) + .optional() + .describe("The stock ticker of a listed company. 'AAPL'."), + sic: z + .string() + .trim() + .regex(/^\d{4}$/) + .optional() + .describe("The four-digit SIC code from the SEC profile. '3571'."), + stateCode: z + .string() + .trim() + .length(2) + .optional() + .describe("US state of the business address. 'CA'."), source: z .object({ label: z diff --git a/apps/agent/agent/tools/add_contact.ts b/apps/agent/agent/tools/add_contact.ts new file mode 100644 index 000000000..ed1b7a1cc --- /dev/null +++ b/apps/agent/agent/tools/add_contact.ts @@ -0,0 +1,35 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; +import { createContact } from "../lib/contacts"; + +export default defineTool({ + description: + "Add a person to the CRM as a contact of an existing company, with the source that named them. Returns the existing contact when the email or the name already exists on that company, so nothing is duplicated. The people pipeline then identifies and researches them on its own. Use it for an executive or a director named in a public document; use record_fact afterwards for each fact with its evidence.", + inputSchema: z.object({ + firstName: z.string().trim().min(1).describe("'Tim'."), + lastName: z.string().trim().min(1).optional().describe("'Cook'."), + title: z + .string() + .trim() + .min(1) + .optional() + .describe( + "Their title as the source states it. 'Chief Executive Officer'.", + ), + email: z.string().trim().email().optional(), + companyId: z + .string() + .trim() + .min(1) + .describe("The CRM id of their company."), + source: z + .object({ + label: z.string().trim().min(1).describe("'SEC DEF 14A'."), + url: z.string().trim().url().describe("The document that names them."), + }) + .describe("Every record the agent creates names its source."), + }), + async execute(input) { + return createContact(input); + }, +}); diff --git a/apps/agent/agent/tools/sec_compare_compensation.ts b/apps/agent/agent/tools/sec_compare_compensation.ts new file mode 100644 index 000000000..e107e0d6c --- /dev/null +++ b/apps/agent/agent/tools/sec_compare_compensation.ts @@ -0,0 +1,31 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; +import { unavailable } from "../lib/capabilities"; +import { compareCompensation, edgarEnabled } from "../lib/edgar"; +import { EDGAR } from "../lib/edgar-config"; + +export default defineTool({ + description: + "Compare CEO pay across several US public companies from their latest proxy statements: one row per company and fiscal year with the CEO's name, total pay, pay actually paid, total shareholder return and net income. Free, through the edgar service. A company with no readable proxy comes back with a reason instead of numbers.", + inputSchema: z.object({ + tickers: z + .array(z.string().trim().min(1).max(6)) + .min(1) + .max(EDGAR.compensation.maxTickers) + .describe("['AAPL', 'MSFT', 'NVDA']."), + years: z + .number() + .int() + .min(1) + .max(EDGAR.compensation.maxYears) + .default(EDGAR.compensation.years), + }), + async execute(input) { + if (!edgarEnabled()) return unavailable(EDGAR.env.url); + + const result = await compareCompensation(input); + if (!result.ok) return { found: 0, rows: [], reason: result.reason }; + + return { found: result.data.rows.length, rows: result.data.rows }; + }, +}); diff --git a/apps/agent/agent/tools/sec_get_company.ts b/apps/agent/agent/tools/sec_get_company.ts new file mode 100644 index 000000000..f63c0b650 --- /dev/null +++ b/apps/agent/agent/tools/sec_get_company.ts @@ -0,0 +1,41 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; +import { unavailable } from "../lib/capabilities"; +import { companyUrl, edgarEnabled, getCompany } from "../lib/edgar"; +import { EDGAR } from "../lib/edgar-config"; + +export default defineTool({ + description: + "Read a US public company's SEC profile by CIK or ticker: legal name, tickers and exchanges, SIC code and industry, state of incorporation, fiscal year end, filer category, business address, former names. Free, through the edgar service. The EDGAR page URL it returns is the source to cite when you add the company.", + inputSchema: z + .object({ + cik: z + .string() + .trim() + .regex(/^\d{1,10}$/) + .optional() + .describe("'320193'."), + ticker: z + .string() + .trim() + .min(1) + .max(6) + .optional() + .describe("'AAPL'. Used when no CIK is given."), + }) + .refine((input) => input.cik || input.ticker, { + message: "Give a CIK or a ticker.", + }), + async execute(input) { + if (!edgarEnabled()) return unavailable(EDGAR.env.url); + + const result = await getCompany(input); + if (!result.ok) return { found: false, reason: result.reason }; + + return { + found: true, + company: result.data, + sourceUrl: companyUrl(result.data.cik), + }; + }, +}); diff --git a/apps/agent/agent/tools/sec_get_proxy.ts b/apps/agent/agent/tools/sec_get_proxy.ts new file mode 100644 index 000000000..b76257db7 --- /dev/null +++ b/apps/agent/agent/tools/sec_get_proxy.ts @@ -0,0 +1,40 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; +import { unavailable } from "../lib/capabilities"; +import { edgarEnabled, getProxy } from "../lib/edgar"; +import { EDGAR } from "../lib/edgar-config"; + +export default defineTool({ + description: + "Read a US public company's latest proxy statement (DEF 14A): the named executives with their titles and pay from the summary compensation table, the CEO's pay and pay actually paid with the NEO average over the last years, pay versus performance (TSR, peer TSR, net income, the company's chosen measure), the 5%+ holders the proxy lists, the voting proposals, the CEO pay ratio and whether an insider trading policy is adopted. Free, through the edgar service. The filing URL is the source for every executive you add as a contact.", + inputSchema: z.object({ + cik: z + .string() + .trim() + .regex(/^\d{1,10}$/) + .describe("'320193'."), + years: z + .number() + .int() + .min(1) + .max(EDGAR.compensation.maxYears) + .default(EDGAR.compensation.years) + .describe("How many fiscal years of pay history to keep."), + }), + async execute(input) { + if (!edgarEnabled()) return unavailable(EDGAR.env.url); + + const result = await getProxy(input); + if (!result.ok) return { found: false, reason: result.reason }; + + return { + found: true, + proxy: result.data, + sourceUrl: result.data.url, + note: + result.data.executives.length === 0 + ? "The proxy carries no machine-readable compensation table. The CEO figures come from the pay-versus-performance disclosure." + : undefined, + }; + }, +}); diff --git a/apps/agent/agent/tools/sec_list_filings.ts b/apps/agent/agent/tools/sec_list_filings.ts new file mode 100644 index 000000000..6c2f77932 --- /dev/null +++ b/apps/agent/agent/tools/sec_list_filings.ts @@ -0,0 +1,50 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; +import { unavailable } from "../lib/capabilities"; +import { edgarEnabled, listFilings } from "../lib/edgar"; +import { EDGAR } from "../lib/edgar-config"; + +const day = z + .string() + .trim() + .regex(/^\d{4}-\d{2}-\d{2}$/); + +export default defineTool({ + description: + "List a US public company's SEC filings, newest first, with an optional form type and date range. Each filing has its accession number, form, filing date, report date, description and EDGAR URL. Free, through the edgar service. Form types worth knowing: 10-K annual report, 10-Q quarterly, 8-K current event, DEF 14A proxy statement, SC 13D and SCHEDULE 13G shareholder disclosures, 4 insider transaction.", + inputSchema: z.object({ + cik: z + .string() + .trim() + .regex(/^\d{1,10}$/) + .describe("'320193'."), + form: z + .string() + .trim() + .min(1) + .optional() + .describe("One form type. '10-K', '8-K', 'DEF 14A'."), + from: day.optional().describe("Earliest filing date. '2024-01-01'."), + to: day.optional().describe("Latest filing date. '2025-12-31'."), + limit: z + .number() + .int() + .min(1) + .max(EDGAR.filings.maxLimit) + .default(EDGAR.filings.defaultLimit), + }), + async execute(input) { + if (!edgarEnabled()) return unavailable(EDGAR.env.url); + + const result = await listFilings(input); + if (!result.ok) return { found: 0, filings: [], reason: result.reason }; + + return { + found: result.data.filings.length, + filings: result.data.filings, + note: result.data.truncated + ? `Showing the newest ${result.data.filings.length}. Narrow the form or the dates for older ones.` + : undefined, + }; + }, +}); diff --git a/apps/agent/agent/tools/sec_list_insiders.ts b/apps/agent/agent/tools/sec_list_insiders.ts new file mode 100644 index 000000000..e8a6a1581 --- /dev/null +++ b/apps/agent/agent/tools/sec_list_insiders.ts @@ -0,0 +1,39 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; +import { unavailable } from "../lib/capabilities"; +import { edgarEnabled, listInsiders } from "../lib/edgar"; +import { EDGAR } from "../lib/edgar-config"; + +export default defineTool({ + description: + "List a US public company's recent insider transactions from Forms 3, 4 and 5: the insider, their title, the form, the filing date, the transaction kind, shares and price. Officers and directors are named here with their titles, which makes it a source for who runs the company. Free, through the edgar service. Each row carries the filing URL to cite.", + inputSchema: z.object({ + cik: z + .string() + .trim() + .regex(/^\d{1,10}$/) + .describe("'320193'."), + limit: z + .number() + .int() + .min(1) + .max(EDGAR.insiders.maxLimit) + .default(EDGAR.insiders.defaultLimit), + }), + async execute(input) { + if (!edgarEnabled()) return unavailable(EDGAR.env.url); + + const result = await listInsiders(input); + if (!result.ok) + return { found: 0, transactions: [], reason: result.reason }; + + return { + found: result.data.transactions.length, + transactions: result.data.transactions, + note: + result.data.transactions.length === 0 + ? "No insider filing in the period read." + : undefined, + }; + }, +}); diff --git a/apps/agent/agent/tools/sec_list_owners.ts b/apps/agent/agent/tools/sec_list_owners.ts new file mode 100644 index 000000000..740da5b5e --- /dev/null +++ b/apps/agent/agent/tools/sec_list_owners.ts @@ -0,0 +1,49 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; +import { unavailable } from "../lib/capabilities"; +import { edgarEnabled, listOwners } from "../lib/edgar"; +import { EDGAR } from "../lib/edgar-config"; + +export default defineTool({ + description: + "List a US public company's beneficial owners of 5% or more from their Schedule 13D and 13G filings: the filer, the form, the filing date, shares held, percent of class, voting power and, on a 13D, the stated purpose. 13D is an active holder with intent to influence; 13G is a passive holder. Free, through the edgar service. Each row carries the filing URL to cite.", + inputSchema: z.object({ + cik: z + .string() + .trim() + .regex(/^\d{1,10}$/) + .describe("'320193'."), + minPercent: z + .number() + .min(0) + .max(100) + .default(EDGAR.owners.minPercent) + .describe("Lowest percent of class to keep."), + form: z + .enum(["13D", "13G", "all"]) + .default("all") + .describe("13D for activists, 13G for passive holders, all for both."), + limit: z + .number() + .int() + .min(1) + .max(EDGAR.owners.maxLimit) + .default(EDGAR.owners.defaultLimit), + }), + async execute(input) { + if (!edgarEnabled()) return unavailable(EDGAR.env.url); + + const result = await listOwners(input); + if (!result.ok) return { found: 0, owners: [], reason: result.reason }; + + return { + found: result.data.owners.length, + owners: result.data.owners, + filingsRead: result.data.filingsRead, + note: + result.data.owners.length === 0 + ? "No 13D or 13G above the threshold in the filings read. Holders below 5% never file one." + : "Holdings are as of each filing date; a holder who sold since may not have filed yet.", + }; + }, +}); diff --git a/apps/agent/agent/tools/sec_search_companies.ts b/apps/agent/agent/tools/sec_search_companies.ts new file mode 100644 index 000000000..f7da41f4f --- /dev/null +++ b/apps/agent/agent/tools/sec_search_companies.ts @@ -0,0 +1,40 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; +import { unavailable } from "../lib/capabilities"; +import { edgarEnabled, searchCompanies } from "../lib/edgar"; +import { EDGAR } from "../lib/edgar-config"; + +export default defineTool({ + description: + "Find US public companies in SEC EDGAR by name, ticker or CIK. Returns each match with its CIK, name, ticker and exchange. Free, through the edgar service. Use it to identify a company before sec_get_company, sec_list_filings, sec_list_owners or sec_get_proxy.", + inputSchema: z.object({ + query: z + .string() + .trim() + .min(1) + .describe( + "A company name, a ticker or a CIK. 'Apple', 'AAPL', '320193'.", + ), + limit: z + .number() + .int() + .min(1) + .max(EDGAR.search.maxLimit) + .default(EDGAR.search.defaultLimit), + }), + async execute(input) { + if (!edgarEnabled()) return unavailable(EDGAR.env.url); + + const result = await searchCompanies(input); + if (!result.ok) return { found: 0, companies: [], reason: result.reason }; + + return { + found: result.data.companies.length, + companies: result.data.companies, + note: + result.data.companies.length === 0 + ? "Nothing in EDGAR matches. Only companies that file with the SEC are listed; try the ticker or a shorter name." + : undefined, + }; + }, +}); diff --git a/apps/agent/agent/tools/sec_search_filings.ts b/apps/agent/agent/tools/sec_search_filings.ts new file mode 100644 index 000000000..66e2beb03 --- /dev/null +++ b/apps/agent/agent/tools/sec_search_filings.ts @@ -0,0 +1,53 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; +import { unavailable } from "../lib/capabilities"; +import { edgarEnabled, searchFilings } from "../lib/edgar"; +import { EDGAR } from "../lib/edgar-config"; + +const day = z + .string() + .trim() + .regex(/^\d{4}-\d{2}-\d{2}$/); + +export default defineTool({ + description: + "Full-text search across SEC filings of every company, with an optional form type and date range. Each hit names the filing company with its CIK. Free, through the edgar service. Use it to find which companies mention a product, a customer, a competitor or an event, or to find every DEF 14A in a period.", + inputSchema: z.object({ + query: z + .string() + .trim() + .min(2) + .describe( + "Words or a quoted phrase. '\"data center\" cooling', 'Ozempic supply'.", + ), + form: z + .string() + .trim() + .min(1) + .optional() + .describe("One form type. '10-K', '8-K', 'DEF 14A'."), + from: day.optional().describe("Earliest filing date. '2025-01-01'."), + to: day.optional().describe("Latest filing date. '2025-12-31'."), + limit: z + .number() + .int() + .min(1) + .max(EDGAR.filings.maxLimit) + .default(EDGAR.filings.defaultLimit), + }), + async execute(input) { + if (!edgarEnabled()) return unavailable(EDGAR.env.url); + + const result = await searchFilings(input); + if (!result.ok) return { found: 0, filings: [], reason: result.reason }; + + return { + found: result.data.total, + filings: result.data.filings, + note: + result.data.total > result.data.filings.length + ? `Showing ${result.data.filings.length} of ${result.data.total}. Add a form type or narrow the dates.` + : undefined, + }; + }, +}); diff --git a/apps/agent/test/capabilities.spec.ts b/apps/agent/test/capabilities.spec.ts index 95b7cc8e4..66862ae75 100644 --- a/apps/agent/test/capabilities.spec.ts +++ b/apps/agent/test/capabilities.spec.ts @@ -17,6 +17,7 @@ const KEYS = [ "DROPCONTACT_API_KEY", "ZOOMINFO_USERNAME", "ZOOMINFO_PASSWORD", + "EDGAR_URL", "BLOB_READ_WRITE_TOKEN", ] as const; diff --git a/apps/agent/test/companies.integration.spec.ts b/apps/agent/test/companies.integration.spec.ts index c44d09966..655a6f82b 100644 --- a/apps/agent/test/companies.integration.spec.ts +++ b/apps/agent/test/companies.integration.spec.ts @@ -1,6 +1,12 @@ import { afterEach, beforeEach, describe, expect, it } from "bun:test"; import { db } from "@crm/db"; -import { createCompany, LEI_FIELD_LABEL } from "../agent/lib/companies"; +import { + CIK_FIELD_LABEL, + createCompany, + LEI_FIELD_LABEL, + SIC_FIELD_LABEL, + TICKER_FIELD_LABEL, +} from "../agent/lib/companies"; const SUFFIX = "companies-spec"; const SOURCE = { @@ -21,7 +27,12 @@ async function clear() { await db.fieldValue.deleteMany({ where: { companyId: { in: ids } } }); await db.company.deleteMany({ where: { id: { in: ids } } }); await db.fieldDefinition.deleteMany({ - where: { entity: "COMPANY", label: LEI_FIELD_LABEL }, + where: { + entity: "COMPANY", + label: { + in: [LEI_FIELD_LABEL, CIK_FIELD_LABEL, "Ticker", SIC_FIELD_LABEL], + }, + }, }); } @@ -134,4 +145,41 @@ describe("createCompany", () => { expect(second.created).toBe(false); expect(second.id).toBe(first.id); }); + + it("keeps the SEC identifiers as fields and reads a CIK as a US company", async () => { + const result = await createCompany({ + name: `Apple ${SUFFIX}`, + website: "https://www.apple.com", + city: "Cupertino", + stateCode: "ca", + cik: "0000320193", + ticker: "aapl", + sic: "3571", + source: { + label: "SEC EDGAR", + url: "https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK=320193", + }, + }); + + expect(result.created).toBe(true); + const company = await db.company.findUnique({ where: { id: result.id } }); + expect(company?.countryCode).toBe("US"); + expect(company?.stateCode).toBe("CA"); + + const values = await db.fieldValue.findMany({ + where: { companyId: result.id }, + select: { text: true, field: { select: { label: true } } }, + }); + const byLabel = Object.fromEntries( + values.map((value) => [value.field.label.toUpperCase(), value.text]), + ); + expect(byLabel[CIK_FIELD_LABEL]).toBe("320193"); + expect(byLabel[TICKER_FIELD_LABEL]).toBe("AAPL"); + expect(byLabel[SIC_FIELD_LABEL]).toBe("3571"); + + const activity = await db.activity.findFirst({ + where: { companyId: result.id }, + }); + expect(activity?.body).toContain("CIK 320193."); + }); }); diff --git a/apps/agent/test/contacts.integration.spec.ts b/apps/agent/test/contacts.integration.spec.ts new file mode 100644 index 000000000..adc334063 --- /dev/null +++ b/apps/agent/test/contacts.integration.spec.ts @@ -0,0 +1,132 @@ +import { afterEach, beforeEach, describe, expect, it } from "bun:test"; +import { db } from "@crm/db"; +import { createContact } from "../agent/lib/contacts"; + +const SUFFIX = "contacts-spec"; +const USER_ID = "contacts-spec-user"; +const SOURCE = { + label: "SEC DEF 14A", + url: "https://www.sec.gov/Archives/edgar/data/320193/0001308179-26-000008-index.html", +}; + +let companyId = ""; + +async function clear() { + const companies = await db.company.findMany({ + where: { name: { contains: SUFFIX } }, + select: { id: true }, + }); + const ids = companies.map((company) => company.id); + const contacts = await db.contact.findMany({ + where: { + OR: [{ companyId: { in: ids } }, { lastName: { contains: SUFFIX } }], + }, + select: { id: true }, + }); + const contactIds = contacts.map((contact) => contact.id); + await db.activity.deleteMany({ + where: { + OR: [{ companyId: { in: ids } }, { contactId: { in: contactIds } }], + }, + }); + await db.agentTask.deleteMany({ + where: { + OR: [{ companyId: { in: ids } }, { contactId: { in: contactIds } }], + }, + }); + await db.contact.deleteMany({ where: { id: { in: contactIds } } }); + await db.company.deleteMany({ where: { id: { in: ids } } }); +} + +beforeEach(async () => { + await clear(); + await db.user.upsert({ + where: { id: USER_ID }, + create: { + id: USER_ID, + name: "Contacts Spec", + email: `${USER_ID}@example.test`, + }, + update: {}, + }); + const company = await db.company.create({ + data: { name: `Apple ${SUFFIX}`, ownerId: USER_ID }, + select: { id: true }, + }); + companyId = company.id; +}); + +afterEach(async () => { + await clear(); + await db.user.deleteMany({ where: { id: USER_ID } }); +}); + +describe("createContact", () => { + it("creates the contact, its event and its source note", async () => { + const result = await createContact({ + firstName: "Tim", + lastName: `Cook ${SUFFIX}`, + title: "Chief Executive Officer", + companyId, + source: SOURCE, + }); + + expect(result.created).toBe(true); + expect(result.companyId).toBe(companyId); + + const contact = await db.contact.findUnique({ where: { id: result.id } }); + expect(contact?.title).toBe("Chief Executive Officer"); + expect(contact?.ownerId).toBe(USER_ID); + + const task = await db.agentTask.findFirst({ + where: { contactId: result.id, kind: "agent-event" }, + }); + expect(task?.reason).toBe("contact.created"); + + const activity = await db.activity.findFirst({ + where: { contactId: result.id }, + }); + expect(activity?.subject).toBe("Added from SEC DEF 14A"); + expect(activity?.body).toContain(SOURCE.url); + }); + + it("returns the existing contact by email, then by name on the same company", async () => { + const first = await createContact({ + firstName: "Kate", + lastName: `Adams ${SUFFIX}`, + email: `Kate.Adams.${SUFFIX}@example.test`, + companyId, + source: SOURCE, + }); + const byEmail = await createContact({ + firstName: "Katherine", + lastName: `Adams ${SUFFIX}`, + email: `kate.adams.${SUFFIX}@example.test`, + companyId, + source: SOURCE, + }); + const byName = await createContact({ + firstName: "kate", + lastName: `adams ${SUFFIX}`, + companyId, + source: SOURCE, + }); + + expect(byEmail.created).toBe(false); + expect(byEmail.id).toBe(first.id); + expect(byName.created).toBe(false); + expect(byName.id).toBe(first.id); + expect(await db.contact.count({ where: { companyId } })).toBe(1); + }); + + it("refuses a company that does not exist", async () => { + const result = await createContact({ + firstName: "Nobody", + lastName: SUFFIX, + companyId: "missing-company", + source: SOURCE, + }); + expect(result.created).toBe(false); + expect(result.reason).toContain("No such company"); + }); +}); diff --git a/apps/agent/test/edgar.spec.ts b/apps/agent/test/edgar.spec.ts new file mode 100644 index 000000000..c648b3441 --- /dev/null +++ b/apps/agent/test/edgar.spec.ts @@ -0,0 +1,247 @@ +import { afterEach, beforeEach, describe, expect, it } from "bun:test"; +import { + companyUrl, + compareCompensation, + edgarEnabled, + getCompany, + getProxy, + health, + listFilings, + listOwners, + searchCompanies, +} from "../agent/lib/edgar"; +import { EDGAR } from "../agent/lib/edgar-config"; + +const realFetch = globalThis.fetch; +const savedUrl = process.env[EDGAR.env.url]; +const savedSecret = process.env[EDGAR.env.secret]; +const requests: { url: URL; headers: Record }[] = []; + +function replies(answer: (url: URL) => { status?: number; json: string }) { + globalThis.fetch = (async (input: URL | RequestInfo, init?: RequestInit) => { + const url = new URL(String(input instanceof Request ? input.url : input)); + const headers: Record = {}; + for (const [key, value] of Object.entries(init?.headers ?? {})) { + headers[key] = String(value); + } + requests.push({ url, headers }); + const { status, json } = answer(url); + return new Response(json, { + status: status ?? 200, + headers: { "content-type": "application/json" }, + }); + }) as typeof fetch; +} + +const COMPANY = { + cik: "320193", + name: "Apple Inc.", + tickers: ["AAPL"], + exchanges: ["Nasdaq"], + sic: "3571", + sicDescription: "Electronic Computers", + stateOfIncorporation: "CA", + fiscalYearEnd: "0926", + category: "Large accelerated filer", + businessAddress: { + street: "ONE APPLE PARK WAY", + city: "CUPERTINO", + state: "CA", + zip: "95014", + }, + website: null, + formerNames: ["APPLE COMPUTER INC"], + url: "https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK=320193", +}; + +beforeEach(() => { + requests.length = 0; + process.env[EDGAR.env.url] = "http://edgar.test:2100/"; + process.env[EDGAR.env.secret] = "s3cret"; +}); + +afterEach(() => { + globalThis.fetch = realFetch; + if (savedUrl === undefined) delete process.env[EDGAR.env.url]; + else process.env[EDGAR.env.url] = savedUrl; + if (savedSecret === undefined) delete process.env[EDGAR.env.secret]; + else process.env[EDGAR.env.secret] = savedSecret; +}); + +describe("the edgar client", () => { + it("is off without a URL, before any request", async () => { + delete process.env[EDGAR.env.url]; + expect(edgarEnabled()).toBe(false); + const result = await searchCompanies({ query: "apple" }); + expect(result.ok).toBe(false); + expect(requests).toHaveLength(0); + }); + + it("sends the bearer secret and trims the trailing slash", async () => { + replies(() => ({ json: JSON.stringify({ companies: [] }) })); + await searchCompanies({ query: "apple", limit: 3 }); + expect(requests[0]?.url.toString()).toBe( + "http://edgar.test:2100/companies/search?q=apple&limit=3", + ); + expect(requests[0]?.headers.authorization).toBe("Bearer s3cret"); + }); + + it("sends no authorization header without a secret", async () => { + delete process.env[EDGAR.env.secret]; + replies(() => ({ + json: JSON.stringify({ + ok: true, + version: "0.1.0", + edgartools: "5.56.0", + identitySet: true, + }), + })); + const result = await health(); + expect(result.ok).toBe(true); + expect(requests[0]?.headers.authorization).toBeUndefined(); + }); + + it("parses a company and strips leading zeros from the CIK it asks for", async () => { + replies(() => ({ json: JSON.stringify(COMPANY) })); + const result = await getCompany({ cik: "0000320193" }); + expect(requests[0]?.url.pathname).toBe("/companies/320193"); + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.data.sicDescription).toBe("Electronic Computers"); + expect(companyUrl(result.data.cik)).toBe(COMPANY.url); + }); + + it("reads a 404 on a record as the service's reason", async () => { + replies(() => ({ + status: 404, + json: JSON.stringify({ reason: "No SEC filer matches ZZZZ." }), + })); + const result = await getCompany({ ticker: "zzzz" }); + expect(result).toEqual({ ok: false, reason: "No SEC filer matches ZZZZ." }); + }); + + it("reads a 404 on a list as an empty list", async () => { + replies(() => ({ status: 404, json: "" })); + const result = await listFilings({ cik: "320193", form: "10-K" }); + expect(result).toEqual({ + ok: true, + data: { filings: [], truncated: false }, + }); + }); + + it("reports another failure as HTTP status", async () => { + replies(() => ({ + status: 502, + json: JSON.stringify({ reason: "SEC down" }), + })); + const result = await listOwners({ cik: "320193" }); + expect(result).toEqual({ ok: false, reason: "HTTP 502" }); + }); + + it("refuses a shape the service does not promise", async () => { + replies(() => ({ + json: JSON.stringify({ owners: [{ filer: "X" }], filingsRead: 1 }), + })); + const result = await listOwners({ cik: "320193" }); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.reason).toContain("Unreadable EDGAR service response"); + }); + + it("passes the proxy through with its executives", async () => { + const proxy = { + accession: "0001308179-26-000008", + filedAt: "2026-01-08", + url: "https://www.sec.gov/Archives/edgar/data/320193/0001308179-26-000008-index.html", + peo: { + name: "Mr. Cook", + totalComp: 74294811, + actuallyPaidComp: 108423733, + }, + neoAverage: { totalComp: 23812358, actuallyPaidComp: 34125743 }, + compensationByYear: [ + { + fiscalYearEnd: "2025-09-27", + peoTotalComp: 74294811, + peoActuallyPaidComp: 108423733, + neoAverageTotalComp: 23812358, + neoAverageActuallyPaidComp: 34125743, + }, + ], + payVsPerformance: [ + { + fiscalYearEnd: "2025-09-27", + peoActuallyPaidComp: 108423733, + neoAverageActuallyPaidComp: 34125743, + tsr: 233.88, + peerTsr: 279.51, + netIncome: 112010000000, + selectedMeasureValue: 416161000000, + }, + ], + executives: [ + { + name: "Tim Cook", + title: "CEO", + year: 2025, + salary: 3000000, + bonus: null, + stockAwards: 57535293, + optionAwards: null, + nonEquityIncentive: 12000000, + otherCompensation: 1759518, + total: 74294811, + }, + ], + holders: [ + { name: "The Vanguard Group", percentOfClass: 9.63, shares: null }, + ], + proposals: [ + { + number: 1, + description: "Election of Directors", + type: "director_election", + }, + ], + performanceMeasures: ["Net Sales", "Operating Income"], + selectedMeasureName: "Net Sales", + ceoPayRatio: { ceo: 74294811, medianEmployee: 139483, ratio: 533 }, + insiderTradingPolicyAdopted: true, + }; + replies(() => ({ json: JSON.stringify(proxy) })); + const result = await getProxy({ cik: "320193", years: 1 }); + expect(requests[0]?.url.search).toBe("?years=1"); + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.data.executives[0]?.name).toBe("Tim Cook"); + expect(result.data.ceoPayRatio?.ratio).toBe(533); + }); + + it("joins tickers upper-cased for the comparison", async () => { + replies(() => ({ json: JSON.stringify({ rows: [] }) })); + await compareCompensation({ tickers: ["aapl", " msft "], years: 2 }); + expect(requests[0]?.url.search).toBe("?tickers=AAPL%2CMSFT&years=2"); + }); + + it("reports a timeout as a reason", async () => { + globalThis.fetch = ((_: URL | RequestInfo, init?: RequestInit) => + new Promise((_, reject) => { + init?.signal?.addEventListener("abort", () => { + const error = new Error("aborted"); + error.name = "AbortError"; + reject(error); + }); + })) as typeof fetch; + const original = EDGAR.timeoutMs; + const result = await Promise.race([ + searchCompanies({ query: "apple" }), + new Promise<{ ok: false; reason: string }>((resolve) => + setTimeout( + () => resolve({ ok: false, reason: "test timed out first" }), + original + 500, + ), + ), + ]); + expect(result.ok).toBe(false); + }, 60_000); +}); diff --git a/apps/agent/turbo.json b/apps/agent/turbo.json index 584af362c..07b6a0d5d 100644 --- a/apps/agent/turbo.json +++ b/apps/agent/turbo.json @@ -26,7 +26,9 @@ "DROPCONTACT_API_KEY", "ZOOMINFO_USERNAME", "ZOOMINFO_PASSWORD", - "PERPLEXITY_API_KEY" + "PERPLEXITY_API_KEY", + "EDGAR_URL", + "EDGAR_SECRET" ] }, "dev:headless": { @@ -45,7 +47,9 @@ "DROPCONTACT_API_KEY", "ZOOMINFO_USERNAME", "ZOOMINFO_PASSWORD", - "PERPLEXITY_API_KEY" + "PERPLEXITY_API_KEY", + "EDGAR_URL", + "EDGAR_SECRET" ] }, /** diff --git a/apps/app/lib/agent-transcript.ts b/apps/app/lib/agent-transcript.ts index c7d8d00e2..c1bf82896 100644 --- a/apps/app/lib/agent-transcript.ts +++ b/apps/app/lib/agent-transcript.ts @@ -68,6 +68,15 @@ const VERBS: ToolVerbs = { gleif_search_entities: "Searched the GLEIF register for a company", gleif_get_entity: "Read a company's GLEIF record", gleif_list_subsidiaries: "Listed a group's subsidiaries from GLEIF", + add_contact: "Added a contact to the CRM", + sec_search_companies: "Searched SEC EDGAR for a company", + sec_get_company: "Read a company's SEC profile", + sec_list_filings: "Listed a company's SEC filings", + sec_search_filings: "Searched SEC filings", + sec_list_owners: "Listed a company's major shareholders", + sec_list_insiders: "Listed a company's insider transactions", + sec_get_proxy: "Read a company's proxy statement", + sec_compare_compensation: "Compared executive pay across companies", get_contact_work_history: "Read their work history", fetch_contact_photo: "Fetched their profile picture", find_contact_socials: "Searched for their other profiles", diff --git a/docker-compose.yml b/docker-compose.yml index 96fa7bf36..2ee9c60f7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,5 +18,23 @@ services: timeout: 5s retries: 20 + edgar: + build: services/edgar + container_name: crm-edgar + restart: unless-stopped + environment: + EDGAR_IDENTITY: ${EDGAR_IDENTITY:-} + EDGAR_SECRET: ${EDGAR_SECRET:-} + ports: + - "2100:2100" + volumes: + - crm-edgar-cache:/data + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://127.0.0.1:2100/health || exit 1"] + interval: 10s + timeout: 5s + retries: 12 + volumes: crm-postgres: + crm-edgar-cache: diff --git a/docs/agent.md b/docs/agent.md index 3b310fe41..46349733d 100644 --- a/docs/agent.md +++ b/docs/agent.md @@ -259,6 +259,33 @@ child place are the targets, and the people who run them come from web research under the same egress rules as everything else — never a LinkedIn fetch, never an invented URL, one source per line. +### SEC EDGAR comes from an external service + +`lib/edgar.ts` talks to `services/edgar`, a Python service on +[edgartools](https://edgartools.readthedocs.io) that the CRM does not host: Docker +Compose locally, a Colab notebook or another machine behind a tunnel elsewhere. +`EDGAR_URL` names it, `EDGAR_SECRET` goes in the bearer header, and without the URL +the capability is off and every `sec_*` tool answers `unavailable`. Every response +is parsed with Zod at the boundary — the shapes live in +`packages/validation/src/edgar.ts` because the app reads the same service — a 404 +is an empty answer, anything else non-2xx is a reason, never a throw. Limits and the +timeout live in `lib/edgar-config.ts`. + +The eight `sec_*` tools are free: search companies, one company's profile, its +filings, a full-text search across every filing, its 5%+ holders from Schedule +13D/13G, its insider transactions from Forms 3/4/5, its latest proxy statement +(DEF 14A: named executives with their pay, CEO pay and pay actually paid, pay +versus performance, the holders the proxy lists, the proposals, the CEO pay ratio) +and a CEO-pay comparison across tickers. Every row carries the filing URL, and +that URL is the source the agent cites. + +`add_company` keeps a CIK, a ticker and a SIC code as custom fields the way it keeps +a LEI, and a CIK implies `countryCode` US. **`add_contact` is the one agent-side +contact write path**: an executive or a director named in a public document goes on +as a contact of an existing company, with the source on the timeline and a +`contact.created` event so the people pipeline runs; the email or the name on the +same company dedupes. The `sec-us-research` skill is the method. + ## Budget and scheduling - `lib/focus.ts` — per-session budget in `defineState`; running out is a normal ending. diff --git a/docs/environment.md b/docs/environment.md index 77ecca6ff..948b76070 100644 --- a/docs/environment.md +++ b/docs/environment.md @@ -128,6 +128,19 @@ single place that knows what is set. | `BLOB_READ_WRITE_TOKEN` | Mirrors logos and photos into Blob | | `AI_GATEWAY_API_KEY` | The model. Not needed on Vercel (OIDC) | | `AGENT_BRIDGE_SECRET` | The rep-facing Agent panel — see `agent.md` | +| `EDGAR_URL` + `EDGAR_SECRET` | SEC EDGAR research through `services/edgar`: profile, filings, 5%+ holders, insiders, proxy statement and executive pay | + +### External services + +`services/edgar` is the first **external service**: a process the CRM does not host, +reached over HTTP with a bearer secret. The contract is generic — `EDGAR_URL` is the +base, `EDGAR_SECRET` goes in `authorization: Bearer`, `GET /health` says whether it is +up, every response is JSON that the CRM parses with Zod +(`packages/validation/src/edgar.ts`) before use, and a missing URL turns the +capability off without an error. Only the routes are SEC-specific. The same shape +works for a service on another machine or in a Google Colab notebook behind a +tunnel; `services/edgar/README.md` shows both. `EDGAR_IDENTITY` is read by the +service alone: the SEC requires every automated client to name a contact email. `BLOB_READ_WRITE_TOKEN` is also in `env.validation.ts` and `apps/api/turbo.json` because the API and the seed write pictures too. The Next.js app is deliberately diff --git a/docs/setup.md b/docs/setup.md index 03ffb3610..d7cb58de4 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -8,11 +8,18 @@ reads once. ```sh cp .env.example .env # fill DATABASE_URL, BETTER_AUTH_SECRET, ALLOWED_SIGN_IN -docker compose up -d # Postgres, matching .env.example +docker compose up -d # Postgres and the SEC EDGAR service, matching .env.example bun run db:migrate && bun run db:seed bun run dev # app :3000, api :3001, agent :2000 ``` +`docker compose up -d` also builds `services/edgar`, the Python service behind the +agent's `sec_*` tools, on port 2100. It needs `EDGAR_IDENTITY` in `.env` (the SEC +asks every automated client for a contact email) and the CRM needs +`EDGAR_URL=http://127.0.0.1:2100`. Leave both unset and the agent simply reports +the source as unavailable. `services/edgar/README.md` covers running it on another +machine or in Google Colab. + Prisma from the repo root: `db:generate`, `db:migrate`, `db:push`, `db:reset`, `db:seed`, `db:studio`, `db:deploy`. diff --git a/packages/telemetry/src/allowlist.ts b/packages/telemetry/src/allowlist.ts index cb901d6d5..93f64aa53 100644 --- a/packages/telemetry/src/allowlist.ts +++ b/packages/telemetry/src/allowlist.ts @@ -115,6 +115,7 @@ export function permitted( export const AGENT_TOOLS = [ "add_company", + "add_contact", "agent", "archive_field", "enrich_company", @@ -141,6 +142,14 @@ export const AGENT_TOOLS = [ "resolve_linkedin_profile", "schedule_recheck", "search_crm", + "sec_compare_compensation", + "sec_get_company", + "sec_get_proxy", + "sec_list_filings", + "sec_list_insiders", + "sec_list_owners", + "sec_search_companies", + "sec_search_filings", "set_chat_title", "set_contact_socials", "set_field_value", diff --git a/packages/validation/package.json b/packages/validation/package.json index d6fb2c36a..843e97626 100644 --- a/packages/validation/package.json +++ b/packages/validation/package.json @@ -9,6 +9,7 @@ "./agent-events": "./src/agent-events.ts", "./agent-manifest": "./src/agent-manifest.ts", "./builder-question": "./src/builder-question.ts", + "./edgar": "./src/edgar.ts", "./enrichment-queue": "./src/enrichment-queue.ts", "./eve-stream": "./src/eve-stream.ts", "./eve-tool": "./src/eve-tool.ts", diff --git a/packages/validation/src/edgar.ts b/packages/validation/src/edgar.ts new file mode 100644 index 000000000..6467078af --- /dev/null +++ b/packages/validation/src/edgar.ts @@ -0,0 +1,202 @@ +import { z } from "zod"; + +const cik = z + .string() + .trim() + .regex(/^\d{1,10}$/); +const day = z + .string() + .trim() + .regex(/^\d{4}-\d{2}-\d{2}$/); +const money = z.number().nullable(); + +export const edgarHealth = z.object({ + ok: z.literal(true), + version: z.string(), + edgartools: z.string(), + identitySet: z.boolean(), +}); + +export const edgarCompanyMatch = z.object({ + cik, + name: z.string().trim().min(1), + ticker: z.string().trim().nullable(), + exchange: z.string().trim().nullable(), +}); + +export const edgarCompanySearch = z.object({ + companies: z.array(edgarCompanyMatch), +}); + +export const edgarAddress = z.object({ + street: z.string().nullable(), + city: z.string().nullable(), + state: z.string().nullable(), + zip: z.string().nullable(), +}); + +export const edgarCompany = z.object({ + cik, + name: z.string().trim().min(1), + tickers: z.array(z.string()), + exchanges: z.array(z.string()), + sic: z.string().nullable(), + sicDescription: z.string().nullable(), + stateOfIncorporation: z.string().nullable(), + fiscalYearEnd: z.string().nullable(), + category: z.string().nullable(), + businessAddress: edgarAddress.nullable(), + website: z.string().nullable(), + formerNames: z.array(z.string()), + url: z.string().url(), +}); + +export const edgarFiling = z.object({ + accession: z.string().trim().min(1), + form: z.string().trim().min(1), + filedAt: day, + reportDate: day.nullable(), + description: z.string().nullable(), + url: z.string().url(), +}); + +export const edgarFilings = z.object({ + filings: z.array(edgarFiling), + truncated: z.boolean(), +}); + +export const edgarFilingHit = edgarFiling.extend({ + company: z.object({ cik, name: z.string().trim().min(1) }), +}); + +export const edgarFilingSearch = z.object({ + filings: z.array(edgarFilingHit), + total: z.number().int().nonnegative(), +}); + +export const edgarOwner = z.object({ + filer: z.string().trim().min(1), + form: z.string().trim().min(1), + filedAt: day, + shares: money, + percent: money, + soleVoting: money, + sharedVoting: money, + purpose: z.string().nullable(), + url: z.string().url(), +}); + +export const edgarOwners = z.object({ + owners: z.array(edgarOwner), + filingsRead: z.number().int().nonnegative(), +}); + +export const edgarInsiderTransaction = z.object({ + insider: z.string().trim().min(1), + title: z.string().nullable(), + form: z.string().trim().min(1), + filedAt: day, + kind: z.string().nullable(), + shares: money, + price: money, + url: z.string().url(), +}); + +export const edgarInsiders = z.object({ + transactions: z.array(edgarInsiderTransaction), +}); + +export const edgarCompensationYear = z.object({ + fiscalYearEnd: day.nullable(), + peoTotalComp: money, + peoActuallyPaidComp: money, + neoAverageTotalComp: money, + neoAverageActuallyPaidComp: money, +}); + +export const edgarPerformanceYear = z.object({ + fiscalYearEnd: day.nullable(), + peoActuallyPaidComp: money, + neoAverageActuallyPaidComp: money, + tsr: money, + peerTsr: money, + netIncome: money, + selectedMeasureValue: money, +}); + +export const edgarExecutivePay = z.object({ + name: z.string().trim().min(1), + title: z.string().nullable(), + year: z.number().int().nullable(), + salary: money, + bonus: money, + stockAwards: money, + optionAwards: money, + nonEquityIncentive: money, + otherCompensation: money, + total: money, +}); + +export const edgarProxyHolder = z.object({ + name: z.string().trim().min(1), + percentOfClass: money, + shares: money, +}); + +export const edgarProposal = z.object({ + number: z.number().int().nullable(), + description: z.string(), + type: z.string().nullable(), +}); + +export const edgarProxy = z.object({ + accession: z.string().trim().min(1), + filedAt: day, + url: z.string().url(), + peo: z.object({ + name: z.string().nullable(), + totalComp: money, + actuallyPaidComp: money, + }), + neoAverage: z.object({ totalComp: money, actuallyPaidComp: money }), + compensationByYear: z.array(edgarCompensationYear), + payVsPerformance: z.array(edgarPerformanceYear), + executives: z.array(edgarExecutivePay), + holders: z.array(edgarProxyHolder), + proposals: z.array(edgarProposal), + performanceMeasures: z.array(z.string()), + selectedMeasureName: z.string().nullable(), + ceoPayRatio: z + .object({ ceo: money, medianEmployee: money, ratio: money }) + .nullable(), + insiderTradingPolicyAdopted: z.boolean().nullable(), +}); + +export const edgarCompensationComparison = z.object({ + rows: z.array( + z.object({ + ticker: z.string(), + cik: cik.nullable(), + name: z.string().nullable(), + fiscalYearEnd: day.nullable(), + peoName: z.string().nullable(), + peoTotalComp: money, + peoActuallyPaidComp: money, + tsr: money, + netIncome: money, + reason: z.string().nullable(), + }), + ), +}); + +export type EdgarHealth = z.infer; +export type EdgarCompanyMatch = z.infer; +export type EdgarCompany = z.infer; +export type EdgarFiling = z.infer; +export type EdgarFilingHit = z.infer; +export type EdgarOwner = z.infer; +export type EdgarInsiderTransaction = z.infer; +export type EdgarProxy = z.infer; +export type EdgarCompensationComparison = z.infer< + typeof edgarCompensationComparison +>; diff --git a/services/edgar/.dockerignore b/services/edgar/.dockerignore new file mode 100644 index 000000000..1dec89c76 --- /dev/null +++ b/services/edgar/.dockerignore @@ -0,0 +1,6 @@ +.venv +__pycache__ +*.pyc +tests +colab.ipynb +README.md diff --git a/services/edgar/Dockerfile b/services/edgar/Dockerfile new file mode 100644 index 000000000..da4e92ab2 --- /dev/null +++ b/services/edgar/Dockerfile @@ -0,0 +1,26 @@ +FROM python:3.12-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + EDGAR_PORT=2100 \ + EDGAR_DATA_DIR=/data \ + EDGAR_LOCAL_DATA_DIR=/data + +RUN apt-get update \ + && apt-get install -y --no-install-recommends curl \ + && rm -rf /var/lib/apt/lists/* \ + && useradd --create-home --uid 1000 edgar \ + && mkdir -p /data && chown edgar:edgar /data + +WORKDIR /app +COPY requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt +COPY edgar_service ./edgar_service + +USER edgar +VOLUME ["/data"] +EXPOSE 2100 + +HEALTHCHECK --interval=30s --timeout=5s --retries=3 CMD curl -fsS http://127.0.0.1:2100/health || exit 1 + +CMD ["sh", "-c", "uvicorn edgar_service.app:app --host 0.0.0.0 --port ${EDGAR_PORT}"] diff --git a/services/edgar/README.md b/services/edgar/README.md new file mode 100644 index 000000000..c0c320f2f --- /dev/null +++ b/services/edgar/README.md @@ -0,0 +1,87 @@ +# edgar — SEC EDGAR research for the CRM + +A small HTTP service on [edgartools](https://edgartools.readthedocs.io) that the +agent's `sec_*` tools and the app's SEC page call. It reads SEC EDGAR (free, no +API key) and answers JSON: company profile, filings, full-text search, 5%+ +holders from Schedule 13D/13G, insider transactions from Forms 3/4/5, the latest +proxy statement (DEF 14A) with its executives and their pay, and a CEO-pay +comparison across tickers. + +The SEC requires every automated client to identify itself, so `EDGAR_IDENTITY` +(a name and a real email) is mandatory. + +## The contract, which any external service can reuse + +| Piece | Here | +| --- | --- | +| Base URL | `EDGAR_URL` in the CRM's `.env`, e.g. `http://127.0.0.1:2100` | +| Auth | `authorization: Bearer `; optional on loopback | +| Liveness | `GET /health` → `{ ok, version, edgartools, identitySet }` | +| Answers | JSON, HTTP 200; `404 { reason }` for a missing record; `502 { reason }` when the SEC misbehaves; `401 { reason }` for a bad secret | +| CRM side | every response parsed with Zod in `packages/validation/src/edgar.ts`; a missing `EDGAR_URL` turns the capability off | + +Only the routes are SEC-specific. A service for another source keeps the same +shape. + +## Routes + +| Route | Answers | +| --- | --- | +| `GET /companies/search?q=&limit=` | companies matching a name, ticker or CIK | +| `GET /companies/{cik or ticker}` | profile: tickers, exchanges, SIC, state, fiscal year end, address, former names | +| `GET /companies/{key}/filings?form=&from=&to=&limit=` | filings, newest first | +| `GET /filings/search?q=&form=&from=&to=&limit=` | full-text search across all filers | +| `GET /companies/{key}/owners?minPercent=&form=13D\|13G\|all&limit=` | 5%+ holders, one row per holder from their newest filing | +| `GET /companies/{key}/insiders?limit=` | Forms 3/4/5, one row per filing | +| `GET /companies/{key}/proxy?years=` | latest DEF 14A: executives, CEO pay, pay vs performance, holders, proposals, pay ratio | +| `GET /compensation/compare?tickers=A,B&years=` | CEO pay rows per ticker and fiscal year | + +## Run it with Docker Compose (default) + +```sh +# in the repo's .env +EDGAR_IDENTITY="Jane Doe jane@example.com" +EDGAR_URL="http://127.0.0.1:2100" + +docker compose up -d edgar +curl http://127.0.0.1:2100/health +``` + +The edgartools cache lives in the `crm-edgar-cache` volume, so repeat lookups +are fast and the SEC rate limit (10 requests a second) is respected by +edgartools itself. + +## Run it on another machine + +```sh +cd services/edgar +python -m venv .venv && . .venv/bin/activate +pip install -r requirements.txt +EDGAR_IDENTITY="Jane Doe jane@example.com" EDGAR_SECRET="$(openssl rand -hex 24)" \ + uvicorn edgar_service.app:app --host 0.0.0.0 --port 2100 +``` + +Expose it with a tunnel when the CRM runs elsewhere: + +```sh +cloudflared tunnel --url http://127.0.0.1:2100 +``` + +Then set `EDGAR_URL` to the tunnel URL and `EDGAR_SECRET` to the same secret in +the CRM's environment (Vercel → crm-agent, or the local `.env`). A quick tunnel +URL changes on every start. + +## Run it in Google Colab + +Open `colab.ipynb` in Colab and run the cells: it installs the service, sets the +identity and a generated secret, starts uvicorn and a `cloudflared` tunnel, and +prints the `EDGAR_URL` and `EDGAR_SECRET` to paste into the CRM. The service +lives as long as the notebook does. + +## Tests + +```sh +pip install -e ".[dev]" +pytest # mocked edgartools objects +EDGAR_LIVE=1 python tests/smoke_live.py # real calls on CIK 320193 +``` diff --git a/services/edgar/colab.ipynb b/services/edgar/colab.ipynb new file mode 100644 index 000000000..33a89d9f5 --- /dev/null +++ b/services/edgar/colab.ipynb @@ -0,0 +1,54 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": "# CRM \u00b7 SEC EDGAR service on Colab\n\nRuns `services/edgar` here and exposes it through a Cloudflare quick tunnel. Fill the two values in the next cell, run every cell, then paste the printed `EDGAR_URL` and `EDGAR_SECRET` into the CRM's environment. The service lives as long as this notebook runs." + }, + { + "cell_type": "code", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": "IDENTITY = \"Jane Doe jane@example.com\" # the SEC asks every automated client for a name and a real email\nREPO = \"https://github.com/teknewmcc26/crm\" # the CRM repository that holds services/edgar\n" + }, + { + "cell_type": "code", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": "import secrets, subprocess, sys\n\nSECRET = secrets.token_hex(24)\nsubprocess.run([\"git\", \"clone\", \"--depth\", \"1\", REPO, \"crm\"], check=False)\nsubprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"-r\", \"crm/services/edgar/requirements.txt\"], check=True)\nsubprocess.run([\"bash\", \"-c\", \"curl -sSL https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64 -o /usr/local/bin/cloudflared && chmod +x /usr/local/bin/cloudflared\"], check=True)\nprint(\"installed\")\n" + }, + { + "cell_type": "code", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": "import os, subprocess, time\n\nenv = dict(os.environ, EDGAR_IDENTITY=IDENTITY, EDGAR_SECRET=SECRET, EDGAR_PORT=\"2100\", EDGAR_DATA_DIR=\"/content/edgar-cache\")\nserver = subprocess.Popen([sys.executable, \"-m\", \"uvicorn\", \"edgar_service.app:app\", \"--host\", \"0.0.0.0\", \"--port\", \"2100\"], cwd=\"crm/services/edgar\", env=env)\ntime.sleep(5)\nprint(subprocess.run([\"curl\", \"-s\", \"http://127.0.0.1:2100/health\"], capture_output=True, text=True).stdout)\n" + }, + { + "cell_type": "code", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": "import re, subprocess, time\n\ntunnel = subprocess.Popen([\"cloudflared\", \"tunnel\", \"--url\", \"http://127.0.0.1:2100\", \"--no-autoupdate\"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)\nurl = None\nfor _ in range(60):\n line = tunnel.stdout.readline()\n found = re.search(r\"https://[a-z0-9-]+\\.trycloudflare\\.com\", line)\n if found:\n url = found.group(0)\n break\n time.sleep(0.5)\nprint(\"EDGAR_URL=\" + (url or \"\"))\nprint(\"EDGAR_SECRET=\" + SECRET)\n" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "Set both values in the CRM (Vercel \u2192 crm-agent \u2192 Environment Variables, or the local `.env`) and redeploy or restart the agent. `sec_search_companies` then answers from here." + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/services/edgar/edgar_service/__init__.py b/services/edgar/edgar_service/__init__.py new file mode 100644 index 000000000..1cf6267ae --- /dev/null +++ b/services/edgar/edgar_service/__init__.py @@ -0,0 +1 @@ +VERSION = "0.1.0" diff --git a/services/edgar/edgar_service/__pycache__/__init__.cpython-311.pyc b/services/edgar/edgar_service/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..61d0fd2f12028ee873629ddf79217795d634476d GIT binary patch literal 180 zcmZ3^%ge<81X*=6vxI>3V-N=h7@>^MASKfoQW&BbQW%37G?}Vc4fG844E!`1Z?T8D z1_gWi`>ka73{w5eTt6c}H&ws1IJHPWxhPj3NS9?Mrxxp{rlcnpLCAQhpniOOW?p7V te7s&k3MfU2=4T7^fZf9aB^EGXpeRuNV_{r>r8Z3%nWPTTf4m*B&E!Vz{PoMC6e6?P@uVK*J);+}*M7NE_Gj<`4B3;PoO zu%CWA<5h|3a5Zha;x&oda4pM7cF`?4M32OZLQ8=cW`PsEkGW^?{hMN6xX#7^rcd&T zf+$FJJN{=pjKgb|aJ^KYZYX?}){^rX3z+bl1y*#6Rgx1{6&4&~b=gXd6)E}eHx;NH zNwZ`;Pras$dh;KlUQ|H^>c#Oq_1ZG($0|~9J(6Z?o>E;IrQ<-Utsre_E!H2RL>R*3Nv?( zJ+lKPc+CPlwn`_&<5DNc+zys+d(2zshPx_q)?Ut;_y$lzN}L!dlfAnlrH&&g2~-|I zJh9Vhm0mPQx{j3eWI?hrNjb_}H}EH{!+sV|7F#{VRzVM5vA0YQJr%hRmUB;e?>lUv zJm>#+GV#=UuPf7=Wis_vl&QahOs6374II`jm3B}(y%VxpiuNCEr(i`p*{Pk*NPQy8 zRj`Oh94h1f&5GQgt-$@!y!2P(g{8a{cyNg4%6K?ck%ymD;9>Ykza98*co-?}(9tp; z1}pL~R)L43W5@rLGn+UL7CPYUd>Ln_MPY^s{&XLO3$ljqqP&)jUQ;9zQBnjt>LSGNo$jZg(Yld&~gAb-B9!8}VC6-DW{EbzKtQpRm z(t}k=R-Utl_xjBHQ#U1fC6$z=eLUBKHpBVh#Lby&?@t+aS&G7PeggP0BF)1mkre20 z$&#YPl8dsz&KnNsTP4YOZL zzNdW}*QRrAD%Y0f+?x{zf3Ek+4WL}98*GVM5jS6~(SC z@2VF++Pv~YXi87$UFY^bR)s5?a77ocsN5CO4Ad;vK&?@0T!BWpHAGglDaDKd^>tXE zJN7c2aZ|CU;J4%rAV~rMp4&_v-~kQoq7G#Ai-Pp-WJ;M|NUbJC@+Ne*!utw!louZ4 zxwPB+_m_U)zq|hZbv1NV6(%)dQWqvwZu0+2uKoYC$VG5Nkv;$xNnEQdV0I~zAcHUz zKpZ?~%9(UPFID~-XdJD4(Z0d{iuuMCV|TbRMe|bx&km~S^}3mfOM!SQwY<6#SO5={ z#IXR?KEYc~IhUT*r1Wq_iYk&wLV(c?uiOj3qHy*6ZT03XLU=-9Rk))GcXZ*7%H5&z ztUCvnq)1#@io(od!99gu^Xel2aC>5v;t<1>`qTYlWUz)!3r^AhtT>|WuQ<9xKL~P$ zEwxOJ!6Jr-hD0Tmipz!&6Qv}?sR!|xd(qgUH}-3d1A61Yi>70GQ;*gZ)SH4iH}7@i7!&XgQ(ZNK z(NqG0atwmEEJa9k2~HxfNY6Ye8HH~6_9SA-`M8u^RF?M9cyOHA7A3<5F_(Q{@bOq8 zrdW1yQQxNlo9Y|=?3m!(D=b^BF2ISZBuyqZF`le2AIhBd*HfSxwddsjXjHuj*sgddujr{qtwD!ob z1GJpO=m6B6Q2Uk6;0?wanr0YM59kGXAY7v$YjChmK(~3YQu~st@J+#a9H<2V2c+Qt zVth;1gjQW>Rk>D^*XF~#4lcyvG=<<6NGbuQ<0J7{MBc}h4Q?St5(bYW^M*}HMbS)! zf`OOHgnPfTOdITmZNt97Sugj8Wy^CBL;9Q9)knSewcAl0<6k<`7gBgW_;h(yIxoeM3EpgAH z3p|6FzbVc><^d&w zXt)E-ZG3*)z76i%sSBMd*J-+MW0@Q)B=vA@ADK4zH3(yW3W0dKNI@FPW~ns3f@LTg z5rLNlt0<#eGuz;y0bK~FT)-5ifr=s}$h;C`#Zav@cqZiS#Ok1S4 zY&GZW%>ET4y1ua=#h2(-gu)sOb07C3EwJB?Iww8lx5GNXfAuMVubD+q<=yj-*bO#K zi5v!2;eOlrdU$CtRC%$|`qK8rPhsnf03KutJmj9S6@y=0S%C{RWDWzT4OS`LHa~jK z$78aB-uBjdFIENbF;9CU4V;gsqLH}#7BmW&D?nbxQ+<}R*Xxc$rd&*wE?yY6c*;!L=HMZ+?9AtA?VqL9dagmdp3kbc^Ka{7G%pff9v}j-)>av z9n*WqphNRc>fXuCiL6kaV_5z!`@-LvvG4ww`gT}7dq?%pY5qCgKc~9pUN}8Z&i#D- z@%mSjUtWKDU9a!^y|Yhs_7x{A|E~9+`u@33zy34z?dAM)Hm>>;nm?iY6RInb_11oM z>FGOZKE0~dpVI43sltHD4Nz4FLq=6J6^}!P4{<>rh}?@38s0ql-(2FcdjvO@;amXi zM^<8lhcAM=fU1+Yu!Wapqb83rK=Qw`$n-*IFeN!P7g<%72>c&~F&yS^8PpW!?*MlA zQNpN^3P0k^FGZ4K9B#Tji0+z7DOZXYKf`sa^!jADi$DF)Yl>lCloZm9tYJQGI2TtV zM3h80Wx?e61oR}1*kehBrlK^rGhL!I+aZ_>nA6Oo5K^^xDx%O6(eNTU%duj(v5TG! zG&WP8pu3Lho92=~y3Qw3;%Z!ai+m2$ar4U0K%+T3%d%M}s9K*a)2CXWm&_&g^`4{V z9#feko-7kktxuLYu38_s%-9-oHXG~NTFfy}rBCg0`iXP2eE|N$4rn@CPcdN=IQ^T8 zYJJac(+_9WnOXJTebuq7IhJ+D@}?b0^8dm%{+?~zdZe-4I@^uYuCm)sBzTq0_{@5* z->gd^E_AXu16C@MlAms9Y?sb<x| zo|21x8#i0Kd9trZZcSr5bhZO|o1=naopIyjId+$loMX)|6C!6PiZdu@f6?@5(~}mB zt<%}MoXx>HwzwPv)d9ds^@Y(~E#s|wGQ9PcCY;cP6RM|k(@AxYbX528%_A zWY6Xp09m&;XNUTw$G63O)|caPzyaCrR{dvVIVXO(7`OL{_?ai?#xDS3*VSg#&F!T_Uc&6rj*rq_(++zwP30BQiJUsgBdcHG2>D9)7p} z{ks14hid<<)<3KF&!U@kQLs(@w<{f`U+Ub?4u)t8S2_>m&{w-k+^}QUiAEjK*^z8; zAk&a}uzO=~WN+lVvG2$9@%Po>v=*G!gVRW)gMuxYR_P&Hskd$)4AB;@1UeqUl{!jR z3S>UPm4!)nclRx_g4jO2JX lJD6htWPQ~+JJc^hx3trN8g&+iDGsgY}TbjG{u@3nyt2#N`d4;6-BlavBVt5!{M%9qkA$pQI*v)!54Yu3AFW}VQ; zsVoFika7qIR5&07sM0_<W=HS6dGF2po_(mQ zQNUN_-j9`(1i)Y7&|C5eaWu(^`v3t1El?0RV8Lpj5U5H82?8V`*^;YDL4lwQkn$@m z1W^!6D1_jGjKW{b1y0R^Oyp1)oX!N8`oY}X?fjPpuDPaTv(OCwqK>J12)R51S(swi zHSH3;%0mLj(!|8rab#@#-RxWZ8)Fg6F?6f!P?v>B_a+Mx?AD2G2n)TXa6%Y3`kiy{ z1NV$^Op?~TnRJ+Zn(z*U=ff20<*x4^?4{D z>SMw`lud4P`_%4NklPLR^fp30>MTY<1kc}Eks8v*8IOJK^BHYK`jS3{LYq9(?bB>Y zsv)5;0^bB)I#t^6=i&5tLY%km=Poy7GU&e8*Q~^QbVF|ZD{eaQpPEFTRufUZOmvz7 z^E)1;=&DW~d-+=a{hQZkinI9}Gx<5qq?+xx8XuBTjXIiRTX!|facXyT<8zHqhJ^{$ zbb>YA5EF(`mXBPFy1HAZMZ-au1rc_2(_*S#cgqejmvu3R%b`1%UnAO~=*IHPlRDBI zqE$^w`7mlHLW&JOYuSuSME6**duBnyGO_J4*`LesC%10hp1gkJV6$5>D8(AvGwuw@wJck~(Czyab{NZK?2)reC K@&7|Vuzvvj9B@?t literal 0 HcmV?d00001 diff --git a/services/edgar/edgar_service/__pycache__/config.cpython-311.pyc b/services/edgar/edgar_service/__pycache__/config.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4a0cea528adc81d08c654d7f8015378ef4a156cf GIT binary patch literal 1833 zcmbVM&1)M+6rcUj>TBioT2keR5Zcm1b!cS=oDh-}Ol^WNmW!<%s>`Bk*3PEV%AH-O zbq;0)fkG)Xloq!)pIjIF&_6{Af-D1uLQlHMIH#QY-mbNdT1s(dXMXd0Z|2R+liqv3 zWilxO?eX5Pk7bFFzfc&r*abPKK=uhGRA`Yp&SFbwi*->TA~8iOoydm*A;&PnCXxq) z#-0(X9EzZa5w=5^j>iJ2OEhjun<7m-lj|}SHb^Ns#MNp^;7QtW8_ib3acUqbo2>KH zw40GF5O{6S&i7#2Cnl*2l+?wC#FR`p?H*_YCI&19Iw{i0kuDZ7C1UZ2B_fuLSSn&^ zWK@NBXLw@6bY08da(Kd`rtMnpE>|40$xN5a+a2ce1bSppi}6&u@r~h_4c2_b)5tfi zR$!{5(D~XnS)>Z~1}cZExNO-DT)-TjL}?dJal%7ftNA79t0jde48v};O~c?R!)SNt zP7Aqe7+>u)S{Eyl<1(%pM#Hu{Zo{=YwgY=4%XWV!=lH|EUMN56w9WF4W3qCSwaYL+ zv6`k+HtAM_g$Qm0%5t+~Z(3U)Z|^b%Yy-gXak>EB8{eN(y0_}Bc2|8>^H#i-?uwtv zconbGt@zQ>bktH zT^qlNEAWTxu*pcuz$!-5`3XeuK5?TsCd7?m7jh)x2jkcSqsvAGDo)1xfqc1o$tJVl zN;gPvbwm2{?kn9U)7YdxlUaVHvy``aIgQf-L9mmVyx z8q88VA(pry=JW6- z0>e zv$k0Qcg`XN(OCq>6V62yVIP5*gW((jc;qylJ=L>6758uac%%33+#m1v^@V}H@KV3~ zLciPB?+^6*r<(r#(;xP}+xx{n`l7GR548E0+U*zG?Y?$rpxrsmO#eI+9&n%@ZHD&! z%&kG@*8iRCXY+op;1{nW$&(llaXVdI2((lTOUVcmpv#ESQ( z3F~CeNX}&LNbaO<#5QRku}|iWcjTPdL6PSp9PY(%_d! zQ^VdB<6X(7ceQ*eaw_9>*NyzHpBP8>@MXce>w3N%;d+D{5UxOYFTw=~S0cQR_XX>4 zZ$X;fxNqdEf)?C21?%ukUy>}C!~1HyZ{}+dzc$!{IQuup%7|~~_aOd(?ebGwj9lvw z>tLc5Ta{Xm_pN-xwzb-e_lNj>c;AR0a!_;Tn^PfbT`u2(c!yazjj1z^N5LMw7`usR+J9z%PQJ-xX$x%^?oeTK!|uEu8m2;#J{z9|bH z4_Y>T(W7|Z&L2aIo_VS-+J^Wi_~VtPWN)HP_9xH}fYDr<3o%7^+K; znWwBpC(@qebwgak$$y~nIoP06jC>#*#ML?(3ic3j(sc`_K)+ysqlFCj$#jG+#yFtC>Rh%Z}_i|PmE8E zh5mu&G8H-$42S7KgITeTj^FZ!gRg`Y10T4fn5SkZuLT9gC``|V6eD|Kn!7PQ5uES5 z5f0CU+FDxX=H{A1!O`Zi>Dw)%W8+QN#;01YQGE0>KNb)=1Eb;b>8TT_TB4eJdd>`- zn70r0obKzre6G)TptX12Jk-OknNRx@Px~A6iUE-b^fwq3U8|xyr05RMTYCFW`3@ZF zn$JDBfB%6dcFtG7T;II6v*E?iz8CB3JKJ99Y8q*NvFW9KFRG6kI#GIqQ89;Z1cYEn zF$X5mn_;Cyz4u=WP6e-zkB$c>ChqvB=caQ#;(eat#Mw4zF7^tw z$ryh~3og%u!a^`GIVkujc?|)U(H;slB!Q0tbp+T;dahWLJ*Jq2V0c!T8cp^V-g2Or z9MOw!RsrsF5iWq=%~TAbHcT-R$ITRFiDo=azU4r5Gt>TD4_btudgv%$Q}RwI4Cf`o zoaT*|>JiNyV!sCcpyKK4?eDtiKX<;T>zu#0>r$7$_sqoxi%^cN6f4bRKR+%g`st8j z2~OP}7pA9_Tr{5#UJuMpgcWOYstUwRH0XrAxP+*&K7l$?$r+uVx;{QOD+GmB+*3D) zUI19+zRE9NJQKG#WlQk}r!$?=ty^>N=6-1{e_|~cD-NyQkdO3B*3+`}v}iq@j#s~K z^(;Rxd-q7zI@ww$CeICn&Qwns)w3_9846|;$Ue#I2;Ap#xNz!oa2cP5*J-)U7GZ+6I$ABSNtxUMZdnfzexmv0!rQso2v3q!b8;1;xZN%o@jv zD_u;o41qYDCj3H!0E^s)-e5Ym?ktj>l~KFoY?7T#qP6Mw>o&J+V;y>odQ}DM&cb(R zS0-b_lJltSJStifCw2a@-~ayiAv>~s-CfB2d2NT`Z*?68&1azYu+Q)!0@Bn)i#`x+Pz<6t6o9#iFB<B!7$cH&XNH2;9n-A@WUC`xl5nVK6E~ZmxV+y;%DMZI<=3OLf z&T0qClrJ+K%E}lx+m)rwVL?B&2rR%vB{c=j{RKlNvIUaX{RL+I1A{hg#PGVF=N1gF z8s>E2q)3HKki*3V18)cyr0WpLN*kqNyXnyO%56^YJZ(L~-gIWXk=54}G5iJhz_@*# zllUs4|KoyiQ96A@&zpIR7Q+edr>V*7x`-a5);c)v9h~+J_4W9`$pr=9WFS0xBN+01 zr)!?ip#glI_`~@^iZL-geQS2ccRhfZJYERCd3~Etp=f>LN+y@iOq>c�&5Xp?H!M^V1+1Lax%k?Hx@cYO_(PLa+98*AEWWl)l|^e?+~!&w3=w?~opo`+L(HGMNO_EQNry(=KdF)|l~bG;gNY@Z({wmza4 zteC3Cw0t*(7Do(tYRG=dn^ujOnC3yn5*{DD1#U#i3BEFVBQP}<41H&Wz~K{!`+?YD zfPkeKiLY18YFx#djM`vff?z-~PK;xQ(WDXx@d=j+xCmSUz(k{uC=3(0NMHycYkHae z3@Zr4Ez$%F5%R<2vbpZIFZQlGT<;!TITS64^-AsovipGOKDu^MC02|5>-nB{udZB* z9$zy`-ovu@u;@Mh&?V)!%lYk#XMXD}TCuF!BxjB6tU=V}tM^`%@~hY=3| z!jNV-WzisBcZ<msIzIXz{xvC$rV zG=mpNm#A%H#Hg*Ej=Otksu682Os7CAD;F#geaIaqp*C@VHd|=&YRipSUWWt$Uc&OK zWeN&ay3Xm6y%SDXmqYzB#K2^(9H;&6+T4K@bbpfZg@6c5ga1G{h|OClSWMyR8UNIL z8PVeeO$5tHs+a^_Zu2!bgnmkK8i43X*u!^zdqj z5G_M&Wi!I$=3Pwcx^F5~fwvBQ%6fNVb?vU9_Pl)wMh%-JEv=uuiGgg8Srqbh(e zIQh^EfKzgH$*!(Pv!eYR)s)g~4H-vZpj;99*2>xQmreKW@7UjV+;d2_O4(K!HAlTs zvuvvqjdh`Y_#B_P>$;9}pB*pgb{al+=m@&IcK00Sesegtx5Dt7PH%6C;qOXJ2yadO zUuV=GBE?e*A5#UGkQqo46H=8VGeq@>4yI@YtU<{z5qS|8Wls7Q2M5WH~vI} zJR`ca&PF>8HAqNzlja{gw6U)1dv~17ASL=4(D|&~Qe1;Jx2#wxCB}}iQlvl|E6qcg zf|E1hJEy+W`7~dO`kItL-;haWfYUKv?Ir61#HUXol3d~L>wqY8s5d{>4KhI;dQ*M# ze#JNw5W*o!pP1I)<+P7M&H2iV&@o@;5AZyYL8!TXVtO<%5$b48zeJKyJ;48tzeP^; zzKnAvEDpbf>oTw2XF^80nQ|4{!C+W1j*=c4;sw;(ieY|yrXg3!zcw2hpF)Ajf>q>2 z+>T0ygANBv;d8zY~+<8f? zyNn2u_hlI{?`1jf<;7E9xr&#YS9&$9)p~(;-?(`%>t4k^K%XwlW%Cqmo`Rd3f_!n7vukA)A z8%fU9JaBROgRrhm3zI>CS!hDSCFG+)$uyIQZ~lfD)BD^Q=Fm&#Btm{$f7ddUFpbpU zJyeO)xF|X;D$Z>uVUPGL;Nk>#n1PB@FW*Gkgo06Y52)wL?#BfPEqA4!sOm`)3jaT- zXfTsz2(Kg49c%cHGivBPFS2u`>+y34EeF(FK(&TLh^rVv#N37YJ`VoiHir+Us0Z7sF1 zv}3+>^AJw58my;IBT|B!XqXZkrXF4T;%Y#=cKwSR6Vla5`RXK|{i-Gw{`8ekUU_Ji zS~}&HPHFE+dGATQ$U0SSBTrS2kY*&U4H5nXw=j5*kM?E^Ee{)7=dYj946HnLU=3qj z^-b}@8-sRCZUb_3z;;Fc;;O=p#bMidP zmsUDrhorpya^8M1@5oyB;we(pm{)UEy9;p>SMI$aPmVr}=YPOn}0^@!AX zQf@qnTbyx*SU$AGjIr^D9?91s`#MBl@1rW<7s)*!y9eTIR)1I*t68%>yd+h1%T?WC z)#*pqFta7^dD(lO>^=P}z0um3NAfhwo@U9}B0F2wODi7CMZ;oK$0MUu*(XNo*Z>d`YUkDA!(;d_%HtXuYKT!B~_R8&5n8OBKCxMXyxSCzn7?mo}4b5i^sy+EyFN^2~^tNopfxUfr2d$iO~X zDGfWPgg!MTb@9ekGgE{pm^DHWNIdK(9~Y%)Mi`$A2zUJavzusnoK*CzT=Z<*>6M*T8+jHQdn!nbJ>fmngEf!IRRTP9KnzEXc$z$j31%blzac=D z;)qTt+)l2CGBC>sIOX0h?$43mFOsEjq@MiaYN+7!(27?^jM0pdlZphD8pfj@20kcX z)zAaOFSE>M3&yEhXcIEp0Bu4sl#&^vm^VdCTd4L-%t&Z@)s$A~MO0lpJ}esq4^Wry zbZxk$>Z@8fbx4G?)-h<|jEapss1mP%lgB5*FkUh38Y$iy%<1kIKE?y#696_)$$?7M zqR}LP$caPMp*g{=kUv&3TVV|m{uzFuPXS2yG{Ju(-g;DYJ-58)OIOVkS52%&bk#_% z7TMJzx>_Dy`SQg1CnwHJC!Ui}JhxbDFvBzseSFTSk#VFa-5JICmtP!ijw^O&;zuPj1%{!)F!lt^IOCn+b}M^N7OWB+KHe-)(IY% zn=`aa$SZMZ#~`!GVEp|YW9JvC@v~r#n4x*f7(Eg57Mdp$JYFF-uEMCQ(d3w;ne9jx z%t1+QoX4FOFKh1>EF?}wAOJ>;nM&bJqDh9DIbzY~5i#CkEm3O*tCDaFrJjQPK;uQp z2@A7R#QO3A*#I;&i0bXe@0&(|gg{b|%~5JggP7tw5NM0Dbj@lo*ckmPtyc09M(AKy~TQ&t}96P@4_4vN-;>o(`Ig_N|= z7C-miu#{gX=hsQLdbj~C_I|~BV~u}!V4wf zvtcq2zE?q#*D&0G{VNl-(y(R|i`z7@G}8!9Jv$R@LdO^!scFM;QObAIkuAGJ7nGBZ z1hSu+F50056RH;$6+#4?4)}PX_Gx$WDQKIZ?=e1I&P+--(Zr?c!K|T#=GlZ`Vmih! zGn$fAkYErJw7>Vv(1mkd!~VgpXZwWflqh9lRZL^T^y~~2iT?`E6@w6%V==T!M7tj! zB;~yDH;DPy_=WBOU|i)o{%meJv~ngImYj{Uvr)=zl5?BH+}1VMV$Zt0_)B}m6MMx< zNU~SU_UgsHb$8iH&uYKqu94j}kGx{b`Pl5IKmO#$4-b6&>Mvi7z52LEY@QM?O?~mg zsK}4W{4ME)3HgNyJfYJzO{Z~Ml`d1C0aYQ^hYgZn1H!7WQtADm#^y{XDCoNlGc}pR zCWHS(7HMj~_0Z5_;{}7()CoQ!vrfoQJ+!G3*#0SECb50Q2%ZJ;f00T9ULgacW|{;e zrYNC=O$Y1{ngO`eT{bp{$&;{b;>dIkikNuA_e?s6#u>=0EqB4fI3XgkiGy)M)^J(! z9uAVmh;<82$jUe&>#No@Cp5+_lxA1J}JU<>a~zJ(Gu z_h!m?tc|;1kK{y&_1<(uY+0qHdP|!Fl@=zxDsiB!l%v(oF*SXj3x}$mWtxn(XQlie zRV{Cg*ulHzL~K#g@ohTzT;8PB%BZmcZ4WHrmUMzhuC_(IeKm{j!+P^YBnP}h9z$#n zQpCV=D@=yqG#rKrN0p$gVE0vF<|X8g7$hW-jX4~cm@j8uVY3qxP2{VeZDke+Nh)S| zbd68*Nhf`)c8PHu?4S( zLvS!9i@C}us5}BRPy)WPrVEq1afH!mC5C7$mU^Xnc|EScaQCpwzmy>xHn?HBL8_@#H> z6YoCB+a!CNmJAzun`r<>=W>{?-x_#x;EiYRJ{zyvBf9o4*WBOx&fd2h?=>zr#(LvL zyW-yBxT|QxW_IU(%>n$5z>?)#2M6b$-j$-CRsOUxRwR`ikV_6oo`bUIpyX_oovmx9 z)-JD|lAXtwjO+Q{cgI%7qH}N)Dm*L~9+vWt$oWUa{GPw9e#rlI!{0P4nU+jRZ{GnX zaU>y=I6Bs5#i9ZE$&}Tn3g5PywyD<&q{< zy(D@df-mJD!{yMuImuBeJ1Un9ajWgE+&6Q-D&MnSylZ7>_4)T4s}8ZaF;=x+2I{+d z=e>^A4za8yc5%I8H`H0xu~#6NS9Qo$9a2T7T+z8+S^Z&utVcZ1^XQUPb5^c7D^(82 zl>_mjA3v%Vi++sran6^gho77t7DrwZ{Q>FpHTm>4^^F*WUWuaMTraDO8C0i+4W~8F zvB3d+ZOF|*mji61K_ORM7QHOm_R(qF7a}6?FS~s`Mci+Sa(kXJ{ARbi=aAtyhfD}7 zw#o4+KPK`hWJ`_kpR(}%SMYV;BX+?b2VP1TZy0}NnTqC2k0Z@oR5ix==!UN|i#55xiD)2yh(V#$L8a!sR#WXDwv0wqQv+`ylcv~|anGAw9 z?ah>+s72Ae+_LT^aUnO7!&~PucNcP?O3B@tozUjEV4KQ^LL-AXf{Yu;E9>swl;2UR zRP>LH-(#Ep-G`?PE_Apy<(>2eDKXpXoN!- zNyk7s6s15_Fhh$#I5jjqMSgK)-%@tb<&b}77KX@B&_8|MABNjqNL|y&>}5)R0u9s4 z(TPAPq_SE0fw3_mI0i+HKe5EHAcTdJ{@c@}t?4;1<<0L2&z!p%`)L98j{nc@k{1hJ-pRztF8(;Sx8~OzFPNM*dA3gqX_FpWlE&SQQib=Ltd{`E{EbnfW z?5(1`_3!tIS4QNsFF;Z!JyA3my9CP^afdLZ`Lmx8CBnFW5ptE>Z+xfm!Q6XyR`1A_ZIbJ_>^i=|RhrtrCa`3VTkXrbH*%JYSY4LmjTe>Q zpMPim?Z~~z(&?qszh8IKlD`~UK9vA*yJzV%xfIgkso%OvR_eu?qu^)QA_UO|10H-I z^NUujSX6LG%HJ>N?*}7!;qn*5FN!aX$}ioJhR5aMaq$A3@7BCKw=xTL*dtyl8jyyV z3rt*r>uZBO&+$7BV1vN7rCh#yx%G`#?!K~e_=(deI(_k~TId;zj?4Hw`|}1*cZdG-j@%xb={LDLK#fB@%|Q~KzKH;S56h=nkrB~n zD*v`%&|tI4(xV%;(4!l9!x%S?6@*||jBuR?!_Uq5dM3qsr8F8@&hD1Y+M6v_wOAuotOBuSuL4=f(P)J-**h~2_E*c+ zY6%^f%-oRf?whcpZS5eRyF=dJr~OEgU$BR!9oP_yhG8+?Olj1#bp;z@*|s)dm3MCG zO!>BHEt{EOZONHhB@N@2vzak|>XhHZ_@c*Bx-#f&vKSN{u^~a&HjOm?8}bsi8pWQ$ zC4?CNq1fS3gVnOCGnrxqJ2B2y)H0n+enxzM_|8laTt&iAPOQdlm=6WU132+`G)P>E zpAr5BE0znW8AEe29>#h=Mrhy13{;G_pr29GnOHiNEyz#XU6i~tgF|O}Q``Zwz%ZWR zf1?b%WDQlhrW@lk{?P0s+3r=AfOIfSx1y4NR%)6DV0-~F0u45m2S`(XBKnNnGeXa* z3oVYCwUV1?E|waA@T*-Q{0=EX#6D+G_{1v0MB78c|Bct!28RpD02=zv2=|ARYnL8^ zaG#VNC$}Tq`4v$;Ru*81@>jWb5NnlAugprWTG>?#s%L_(1YpU$fdvcRxVz+j+dFM< zx8G|AxhpLFvT*;C!u?X=LAmhY5(w7!5^68G1k}YkP_NRHa_LF2^wgtn$=fe``;`b2$3)+9j!|O)g3>4j{?SLdFY#4M^%e%Bm6$G_kV} zjZ(!4x#EOW(jk|0kUHeb>I+hFgIwGo7Lx)53X$7#uPEIYP#4Us^EuZqnl zAJs_Z{c?G~RD4=4J}nkU{=WB%=bjgb|46(R7H`i>&;3|_?#I%(hC|0yq z|FBou-6iktl1jVf((d(=N;ni*!~@-rdL`c(*>^@NIV+c(1y>^%?cFHIA$ke$wZTRz z5r7Q>U$am#mk0BRX!D6iAEShymDG3Z^`Gl=yPYPs%)z8Fjgh9GQNl+U1lg1@Q&q7A zt%CvTDOM{qW#S#ICj>K@JfQP^YV#FudXTlS3X9WI_!f9t7tNp_$r7OZ&$1V2u^ewn zt7jJUpsf0*QdSt4%^@XpvL)8V3&u>f0`%aBVW0bHs3+OnbHjHk~(Mx$}IUZ+eO?cxJ1Zg+6qOYVV^@FyBDy_-BOaSMK(3G7Sp9wlKoko=C6oK$sH?-7!B)3S?_=bwNC84CMZ z+enAR?I$ev4Wa?dkxcU8&9xd#a^|(M4QgZR*0T>r#GBk-+>Jvc8 zF(5kzMC-tMUcqu52`DQ+db{Obi=2C$-t#<(;7hIpawJ1XC*%Do3jAf71*vgZN{Js}#4nE_m|B9BeI&OZAEWQ<-1Odha`mXoV~ zgoBE)DANKS!N*MFQSCQSRZn*-XqA;Pg}kuDs^IkbbHqUno$k8W=O5@B9%5P%bOqIi z^-rdswBqn%11(|%)g7k<;#zvC_3g+HbK2=6lTQr;uw>XA%hgM5OKt1!l9lR}(bam% zT`jw-MSJ4hFmU$5uWZhxL#h`*21Vk$$dR@Ge*tC89(CH5Y1isUvPRmA$aTR0n!wa? zY9GQMYLkP>0+jWzj%U!WtQ6nleafW58w(bV8;E6V`7qmsB&$7;&>&IOZznZItWQ-& zmZze&42_?`cD2l+^I17$NTN`3XY{Vx#{_q#`I@EO+vSjDY5oCq$dY@0Kn_`o-ye`e z7N_t7afC9Ee}_3e4$~pbou?VM`Py8ycbQs%Y!67jU^T0L@QwIKyen&=o{@KF_Z(?! zdb5^=!S^N8UCnSarD4(52=vvX!)aey70V@3>V{YYm1S@_PIT|;e7nf;iRvhW9| zEZR9GU%aZ>JSnT(lATlUP~*~_Q}Si2dhG1VX|y;Q7!UNMJUa(y4MBO*yY}^FeOv4Z zyHkFZe`tOYGdyr|c^??P-J3>RRd#M$jkfF>?l3m0v-8uAjfiOx)N*Qg+a5RZH9P09 zL(1A8oN~|3Dfv2#u1q7P_6yf%rw;oPQw$gFw-&T}2N?NHX~`(fH>^6q2k#wJ@?fF*@8i(G@(*ru9`uZ>`2?@4+pM6MaG1+Z&9LRBv}EQfd(d^*iuZ_rl)2@u&ITo zM{oH9a{+-5DS6XuVPf*uGlgt|Kbj?veymU=-~QV{#Wfwi0hW$dza`5OK1Tr!x#_|L zH}QiW0$u>c#)|1uOR=k^B<@@&q>q(>wQ7kX3Rn?6*(E5R^p>S*U7cb}FqyQ~kf2aa z463=(k+343zuv=)mBi|5m}XM6F;V4zL?Gcr{<7AIT*EHYN6_u0+|EQWSt;RvvQtW+ zWWx@r!GP|R8e1wD;NI@X@Gvu$d{A{qV;a{y2-MKQVnHdXThTd8WBCD^jD2p^C)5e5 zm`}P9VHqHkkyriUOPFAuDM%%k^WGqy{>Mm!B_5omfEMy3ft|}M_%??t+J(hGy|MbW zUTN1e@~&s3!Zx|EZM}>%X2#eg7TEc^WnZ^c)+3koh-GIU3CZPeuR-81IU$#vkUSl- zr(?rnB)u5{L#hxej)??He$wWgXtHXd7XBa9O~G)O_A}=}Hh7gnx`z@LyU$_++?l|* zy1Q_4^yILpGzb;A)Tyy9AQF4xnxeF~$1?LPO+HJ;&d5_Q)X`+92iST9zKl1C5%`IA z1VTZEcbgBO7d%h69oQuBGa`MgwjL9V-y z@%*gRJRmm@NOkArx^wHzht`Jv)zyD-RXi~)9UhSnk4Vig$jvXnvyv3s1PrMvQRUPb zSr~jJI66y|Lqjq&vw+mW3=jWjG%_&@{*V8E7NA$y@l|343q1agoSt&c}eo@5Ts zsu^6Noy7!FKMMx{{*Zb4rxc(3cW9p4%E`u)1Rk2F1>4Tk?dIb)^KQMUd}Z`!V?P}e z_ned}y5x#3si<2n>RvC~9j*ST?)^G(Z<|zgT&_AUm9@)d?d#>$(Vma`-|vr|lWLxk zYo3wH+vM`L_4?*m=+n7R=ES2{rIzRAmglAVVYz-d?ZUfZNVS04qE2fUv7fg?WS(Wd z>)_oqQ<`*fY)aE&g2a^m{tZL4+bq0}r1RTz9e++eeKgV2oUcc}0R?apMgzF((sw5w zMxn7!zf{Aj5)qlC4kJRX_Oo*Y zJ5>gm)IsX^Wh9+B1TywHpI(rLTlkQL2#~{$>Lf^tf8o#QwgVvPD!}Xo%uGfWG6CM6 z9P3iy9EF%toe6Ny((NF?*1iF7d4M@k7(#>riWI>>);b0h*G8{P#Z7W?lVtQRo?1Gy zZg*o_(%0b2W{uVSg_m{id$S2Rl{EpkcAdU5jc z_4lS%r={X%xwsj2^{D%!qW6np#nSGB^6rCDX{%h?3cGsL@R8+x*w!WAe%ZHQU8}KP zSsNYwXzcy5*p#&AguLg3RM{a{cBI&y*lSWaU##cT{!jYFL#L(2Gjih@srIZ~dv=>bH@r4tb^yLMixoo_kbbVQARYknB|w{!n`PV66+I!ENr5$V_q^061b7@HMu-;u`V<*|8n zO@?HR$kvF+&JBH1VEWe16_&3Ii}h{rVDNUy-Y(JG|L7t%VJI+vO~Tag2rQoY*2NW; zZrTK5^Og_l9uB`{8nq*8s=QHiey@H@s)dy>TNTTd!g zs&MhbWvS?jTyzEFpd6_H;K%{`Ud$Ekj=M|a-ii%dBHm$u;Oiyci#;odgc+lKyj zP=7n9zwL!s=`=1joOygmD!d>UUibp;pth^B?dpa>U++@C*ogSzA4zoqxh^2q1-{BF z*cjCvq~+WyNN(S85hzV#nkEd%Y;@!%Mmzy+XUyzi``LP0pswVRz0E!Bag;J~L=4&5 z(goAhfOcPvaE3mgZ54bJdnM?jMB+Cc*ykEcb~67o`@v+fwY+Y#?SpN)Ykt*ycMOY1 za7}xO@cN&?7m?3cPSl&F*~R9*rc4h@SV4XcyM$#siv({<+ib_PG>KRgv~3$THaz1^ z*h$qx%g$)56~^r>2=EEbs0Heuu@!QrZw}&F)3wE2PR89fHWxn2vb3FThR?Ebk#cVH zm$~~9{1y*J^bkAjgV0f4MWfRz=2f{LBEdBA1wtc+peipMqY&-qLI1la01tc(cT1y) zakoAM?sMa`9iEMB>^zszKHWdn-?gJ$ohpaaP|4Zi#CN&vwCfo0KgnJ3gkXk_E%ox> zpe`ZOHe<($!V&dc8^QwoPPu3w_C`rQV_cy?Y=Cf<026^uQ;2aATeJ|&$OWnzu@4zL zFEh?2SG^u23bD?8k3x)&+og7skNA>?G z;9mfZAjXFj>rDVG5oIe1TqPS^jmhyf0dh=q<#^Y<@CuT=wX(Ms4*K9JuxvppZj_4~ zm(DDm$=E^JN$UraAZ}N^=__jDWtH(=zIbgzys|o8Qy<^gLQi(rVyR@b_Pv{{H-YEL z=^X&{o}AtRK<@!|ad}=^q`(#`z|m);64}$$#}&Px!{>N-Mm+Vr+&+w;R5&6Rjx5>Y z?p!z+o(`St) z1RJ`97W&qDA;|t4R4_b_*+du{tD7fjPppaYYwG_gGDGDI89^X@f^eJwA!_yiZV*ah z|A&YDmne0$@PEWJ#jO6L8i5o=0#AUv-c`*SPa({767u_1hc@Q(MvJ!C|D;k~j>vU|@$kY)3Uw1; z)90U1=mi322rvh^aSAOE7$guQutwlh0<_v1o?H{tfiPRN!#sQ8+dMmi3Nnd5deo#cCv9==ZMzf$WEc)Z2Qh2BTsVvUSW7w2*o6Mv~-oGTHvXPm1LwP&0w z6t(A9oPCk~#kpbe>CQOUB5KdCxLu<5jC1YcA98+`H@KMlmA!Q3v}mi1C$3=moDlN? z7H#;aA_cx>=dyD#FP>Mpm>VxBUv$KsB}>BHd9n1QHmf5!0-zU$H_*s;iG)VwY^-GU|@4 z^neM|9m@s|qt3g`(^H@@EpQm{su)eDS6X1V>)gxOIv$r5^}@m#pdv?SUPh6)tn^0R z(a~t}>L9z#2E`s!A2D%>8Q5jb2p{w6hoOh{zkXTX-^-q5gXTQsKMG`t{P}2W%n&`c zIu*kfhpp^cHt@L+9jo7QS@EolMq5{ItX8nwY_QjgD5wW6QDbx{Rvmr*10Yk}W`jZt zT91n35)HA-TI)m4KW}5fM?H^gK0C{T;?Na#%?2LoBGiexs3%(U9%3UH3$0!Hc%B6x z86NSUfp^AJmIqy5@QZ&mDh6+ee{@5BVVu3o2E_*5jE?$-uIbKE@3(+-EWFnH(EV`qVezjA!x<)iw^EU34tn*QiWbX2W`$J^yymY({YzPwX~oW3lCTpv=pFt+-xlpBHs7*lhrK549!0co4=R2`?cr_#<6$MRXaAlvHKIN1>X| z$|5${3vw!*O=bXfpU-U>p7)cPP1*1gS=}VL6t?$52-Jn#@I9S!QS}tiABNSg`2tdb z-fj;8e`Jq|OF~P?!OgUJ5prw4yg-{{kwP_ zoFgjNmIAI~^EppKZub~_G1-ik#vP|9J0(C#;$}>D|pl3KHaz-Gn zq;raz%%x<5c`u*O5D#o;cvdp`q-wa+In`jYNp;Qeq?KGUXSfxW7%Y)DGD%qBfl+$H z2)s?5^dcd7@`Z~~UopHJNutP-LR~HzeKwa^%V*`prXrI>iewYezn@OYNGy9%Q$YGE|#VY>+P(%%Opt$oOP@VfSri~`_(L`~l!G}v~lMR*Q z4+M=J)7i0oggwWahlb1Y-GO@pC66^Z{$R6mRAVP}c7ihe1~-MVYW&M#O-SfMqQ<}T z^@XqA_-0d^y{OM#gkec7sXxrse*N~h=e|Ez^S!h0a(g1p(8$M1W$y0wz3n|gtPA4q z_%6|e)4FhaPgtl63*Vgihf5O{bzyN&xLg-5Yr>nl@TL}eOAozO@|L^@O>UTOAL#(| z0ws0d4b+2!gMWvGdkd8nEgaXw@ftT%nk~)l&exb3B@8NfJbZp+aT-0D4lFLXpPa=2 zdr|{U3ubC~9gt5P0ov;rJKz*WD{$zFRx3E%2iyij!ZHm(4gnJp0JN-2Xx=kCH`ekQ znH&Xtn9hv>D5B(hd3K+aNjth7L z(_4rxh`p%Dl0jf6H2IJa1w#M~W<8(I5o%tDtBp)rZO0Ou%E-x_9ZZ(G8CCv3B{ATp z!J#Apv;w0m6oxhSWu1N5^s>({eD=ofrWTviV{<0K&AFbV9>$vk|U6|AaQ5Qrl zIHd=tikBMPQ0bRtsm2^8$6%KpYfHIUtY z5s2<92CL?;OX-wKYjrA5Bt>Klc2!oBD%5W#lU7v2e|2laD%ynF8M6BnbYvPpkGUC= zzoC#BAk$CD0pJn++bOsBwmxe7Jn6e5rC-i%jT| z3ETDu&4J;Mw`-i~BSX~q7BE+q{ID8I;SiKUh33gmg)``H;q!j?KYbWrk5Q;X>Gw1T z;HRry6;$2ddV}`)2}yDUC?s_Hm~ve)@F~&V6++&O`6v>x`4Cd*hEXeJ`%>xOT{qMA z91vfHPdNzyiX;~-ZQfeAv#`e>OAI8o=s8h2`Usm7c#Q?NK> zl@00_Rxq=Y^>2gqdh@DV8|mCC@zQy-%Fr5N)oogzt<%ORM#GpS0Koqu;4Y?_2|eBy z$lS_2`7__SY+l;T$ghzFn4(uwUIhpi!nlFBqWRZAI6d^j+7L- zL!_)!uHM}?Q71U+$3s*WbQPv@wK`r^s;~U{xB4g?5NJ*?it$W&5u6#%RIJX#p9f6! F{uf5O^rZj* literal 0 HcmV?d00001 diff --git a/services/edgar/edgar_service/app.py b/services/edgar/edgar_service/app.py new file mode 100644 index 000000000..bc57190ff --- /dev/null +++ b/services/edgar/edgar_service/app.py @@ -0,0 +1,117 @@ +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager + +from fastapi import Depends, FastAPI, HTTPException, Query, Request +from fastapi.responses import JSONResponse + +from . import VERSION, sec +from .auth import require_secret +from .config import settings + + +@asynccontextmanager +async def lifespan(_: FastAPI) -> AsyncIterator[None]: + sec.configure(settings()) + yield + + +app = FastAPI(title="CRM EDGAR service", version=VERSION, docs_url=None, redoc_url=None, lifespan=lifespan) + + +@app.exception_handler(HTTPException) +async def http_error(_: Request, error: HTTPException) -> JSONResponse: + detail = error.detail if isinstance(error.detail, dict) else {"reason": str(error.detail)} + return JSONResponse(status_code=error.status_code, content=detail) + + +@app.exception_handler(sec.NotFound) +async def not_found(_: Request, error: sec.NotFound) -> JSONResponse: + return JSONResponse(status_code=404, content={"reason": str(error)}) + + +@app.exception_handler(sec.Upstream) +async def upstream(_: Request, error: sec.Upstream) -> JSONResponse: + return JSONResponse(status_code=502, content={"reason": str(error)}) + + +@app.exception_handler(Exception) +async def unexpected(_: Request, error: Exception) -> JSONResponse: + return JSONResponse(status_code=502, content={"reason": f"SEC lookup failed: {error}"}) + + +@app.get("/health") +def health() -> dict: + return { + "ok": True, + "version": VERSION, + "edgartools": sec.edgartools_version(), + "identitySet": sec.identity_set(), + } + + +guarded = [Depends(require_secret)] + + +@app.get("/companies/search", dependencies=guarded) +def companies_search(q: str = Query(min_length=1), limit: int = Query(10, ge=1, le=settings().max_search)) -> dict: + return {"companies": sec.search_companies(q, limit)} + + +@app.get("/companies/{key}", dependencies=guarded) +def company(key: str) -> dict: + return sec.company(key) + + +@app.get("/companies/{key}/filings", dependencies=guarded) +def company_filings( + key: str, + form: str | None = None, + from_: str | None = Query(None, alias="from"), + to: str | None = None, + limit: int = Query(20, ge=1, le=settings().max_filings), +) -> dict: + return sec.filings(key, form, from_, to, limit) + + +@app.get("/filings/search", dependencies=guarded) +def filings_search( + q: str = Query(min_length=2), + form: str | None = None, + from_: str | None = Query(None, alias="from"), + to: str | None = None, + limit: int = Query(20, ge=1, le=settings().max_filings), +) -> dict: + return sec.search_filings(q, form, from_, to, limit) + + +@app.get("/companies/{key}/owners", dependencies=guarded) +def company_owners( + key: str, + minPercent: float = Query(5, ge=0, le=100), + form: str = Query("all", pattern="^(13D|13G|all)$"), + limit: int = Query(20, ge=1, le=settings().max_owners), +) -> dict: + return sec.owners(key, minPercent, form, limit) + + +@app.get("/companies/{key}/insiders", dependencies=guarded) +def company_insiders(key: str, limit: int = Query(20, ge=1, le=settings().max_insiders)) -> dict: + return sec.insiders(key, limit) + + +@app.get("/companies/{key}/proxy", dependencies=guarded) +def company_proxy(key: str, years: int = Query(3, ge=1, le=settings().max_years)) -> dict: + return sec.proxy(key, years) + + +@app.get("/compensation/compare", dependencies=guarded) +def compensation_compare( + tickers: str = Query(min_length=1), + years: int = Query(3, ge=1, le=settings().max_years), +) -> dict: + names = [t.strip().upper() for t in tickers.split(",") if t.strip()] + if not names: + raise HTTPException(status_code=422, detail={"reason": "Give at least one ticker."}) + if len(names) > settings().max_tickers: + raise HTTPException(status_code=422, detail={"reason": f"At most {settings().max_tickers} tickers."}) + return {"rows": sec.compare(names, years)} diff --git a/services/edgar/edgar_service/auth.py b/services/edgar/edgar_service/auth.py new file mode 100644 index 000000000..33be65e1c --- /dev/null +++ b/services/edgar/edgar_service/auth.py @@ -0,0 +1,24 @@ +import hmac + +from fastapi import HTTPException, Request + +from .config import settings + +LOOPBACK = {"127.0.0.1", "::1", "localhost"} + + +def require_secret(request: Request) -> None: + secret = settings().secret + if not secret: + host = request.client.host if request.client else "" + if host in LOOPBACK: + return + raise HTTPException( + status_code=401, + detail={"reason": "EDGAR_SECRET is not set, so only loopback callers are accepted."}, + ) + + header = request.headers.get("authorization", "") + given = header[7:].strip() if header.lower().startswith("bearer ") else "" + if not given or not hmac.compare_digest(given, secret): + raise HTTPException(status_code=401, detail={"reason": "Bad or missing bearer secret."}) diff --git a/services/edgar/edgar_service/config.py b/services/edgar/edgar_service/config.py new file mode 100644 index 000000000..79a864b04 --- /dev/null +++ b/services/edgar/edgar_service/config.py @@ -0,0 +1,33 @@ +import os +from dataclasses import dataclass + + +@dataclass(frozen=True) +class Settings: + identity: str + secret: str + port: int + data_dir: str + max_search: int + max_filings: int + max_owner_filings: int + max_owners: int + max_insiders: int + max_years: int + max_tickers: int + + +def settings() -> Settings: + return Settings( + identity=os.environ.get("EDGAR_IDENTITY", "").strip(), + secret=os.environ.get("EDGAR_SECRET", "").strip(), + port=int(os.environ.get("EDGAR_PORT", "2100")), + data_dir=os.environ.get("EDGAR_DATA_DIR", "").strip(), + max_search=25, + max_filings=100, + max_owner_filings=60, + max_owners=50, + max_insiders=100, + max_years=5, + max_tickers=10, + ) diff --git a/services/edgar/edgar_service/sec.py b/services/edgar/edgar_service/sec.py new file mode 100644 index 000000000..d9aa56cab --- /dev/null +++ b/services/edgar/edgar_service/sec.py @@ -0,0 +1,460 @@ +import os +import re +from datetime import date +from importlib import metadata +from typing import Any + +import edgar +from edgar import Company, CompanyNotFoundError, find_company, get_identity, set_identity +from edgar import search_filings as efts_search + +from .config import Settings +from .values import cik_text, day, number, rows, text, whole + +BROWSE_URL = "https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK=" +OWNER_FORMS = { + "13D": ["SCHEDULE 13D", "SC 13D"], + "13G": ["SCHEDULE 13G", "SC 13G"], +} +INSIDER_FORMS = ["3", "4", "5"] +PROXY_FORM = "DEF 14A" +EFTS_FIRST_DAY = "2001-01-01" +DISPLAY_NAME = re.compile(r"^(.*?)\s+\((?:[A-Z.\-]+\)\s+\()?CIK") +HOLDER_SHARE_KEYS = ("shares", "amount", "shares_beneficially_owned", "amount_beneficially_owned") + +_settings: Settings | None = None + + +class NotFound(Exception): + pass + + +class Upstream(Exception): + pass + + +def configure(settings: Settings) -> None: + global _settings + _settings = settings + if settings.data_dir: + os.environ.setdefault("EDGAR_LOCAL_DATA_DIR", settings.data_dir) + if settings.identity: + set_identity(settings.identity) + + +def edgartools_version() -> str: + try: + return metadata.version("edgartools") + except metadata.PackageNotFoundError: + return getattr(edgar, "__version__", "unknown") + + +def identity_set() -> bool: + try: + return bool(get_identity()) + except Exception: + return False + + +def _limits() -> Settings: + if _settings is None: + raise Upstream("The service is not configured.") + return _settings + + +def _company(key: str) -> Any: + cleaned = key.strip() + try: + company = Company(int(cleaned)) if cleaned.isdigit() else Company(cleaned.upper()) + except CompanyNotFoundError as error: + raise NotFound(f"No SEC filer matches {cleaned}.") from error + except Exception as error: + if "not found" in str(error).lower(): + raise NotFound(f"No SEC filer matches {cleaned}.") from error + raise Upstream(f"SEC lookup failed for {cleaned}: {error}") from error + if company is None or getattr(company, "not_found", False): + raise NotFound(f"No SEC filer matches {cleaned}.") + return company + + +def _match(company: Any) -> dict[str, Any]: + data = company.data + tickers = list(getattr(data, "tickers", None) or []) + exchanges = list(getattr(data, "exchanges", None) or []) + return { + "cik": cik_text(company.cik), + "name": text(company.name) or f"CIK {cik_text(company.cik)}", + "ticker": tickers[0] if tickers else None, + "exchange": exchanges[0] if exchanges else None, + } + + +def search_companies(query: str, limit: int) -> list[dict[str, Any]]: + found: dict[str, dict[str, Any]] = {} + cleaned = query.strip() + + if cleaned.isdigit() or (cleaned.isalpha() and len(cleaned) <= 6): + try: + match = _match(_company(cleaned)) + found[match["cik"]] = match + except (NotFound, Upstream): + pass + + try: + results = find_company(cleaned, top_n=limit) + for index in range(len(results)): + match = _match(results[index]) + found.setdefault(match["cik"], match) + except Exception as error: + if not found: + raise Upstream(f"SEC company search failed: {error}") from error + + return list(found.values())[:limit] + + +def _address(company: Any) -> dict[str, Any] | None: + try: + address = company.business_address() + except Exception: + return None + if address is None or getattr(address, "empty", False): + return None + street = " ".join(part for part in [text(address.street1), text(address.street2)] if part) + return { + "street": street or None, + "city": text(address.city), + "state": text(address.state_or_country), + "zip": text(address.zipcode), + } + + +def company(key: str) -> dict[str, Any]: + entity = _company(key) + data = entity.data + former = getattr(data, "former_names", None) or [] + return { + "cik": cik_text(entity.cik), + "name": text(entity.name) or f"CIK {cik_text(entity.cik)}", + "tickers": [t for t in (getattr(data, "tickers", None) or []) if t], + "exchanges": [e for e in (getattr(data, "exchanges", None) or []) if e], + "sic": text(getattr(data, "sic", None)), + "sicDescription": text(getattr(data, "sic_description", None)), + "stateOfIncorporation": text(getattr(data, "state_of_incorporation", None)), + "fiscalYearEnd": text(getattr(data, "fiscal_year_end", None)), + "category": text(getattr(data, "category", None)), + "businessAddress": _address(entity), + "website": text(getattr(data, "website", None)), + "formerNames": [name for name in (text(item.get("name")) for item in former if isinstance(item, dict)) if name], + "url": f"{BROWSE_URL}{cik_text(entity.cik)}", + } + + +def _filing_row(filing: Any) -> dict[str, Any]: + return { + "accession": text(filing.accession_no) or "", + "form": text(filing.form) or "", + "filedAt": day(filing.filing_date) or "", + "reportDate": day(getattr(filing, "report_date", None)), + "description": text(getattr(filing, "primary_doc_description", None)), + "url": filing.homepage_url, + } + + +def _entity_filings(entity: Any, form: Any, start: str | None, end: str | None) -> Any: + try: + filings = entity.get_filings(form=form) if form else entity.get_filings() + if filings is not None and (start or end): + filings = filings.filter(filing_date=f"{start or ''}:{end or ''}") + except Exception as error: + raise Upstream(f"SEC filings lookup failed: {error}") from error + return filings + + +def filings(key: str, form: str | None, start: str | None, end: str | None, limit: int) -> dict[str, Any]: + entity = _company(key) + listing = _entity_filings(entity, form.strip() if form else None, start, end) + if listing is None or getattr(listing, "empty", False): + return {"filings": [], "truncated": False} + page = list(listing.head(limit + 1)) + return { + "filings": [_filing_row(filing) for filing in page[:limit]], + "truncated": len(page) > limit, + } + + +def _hit_company(hit: Any) -> dict[str, Any]: + raw = text(getattr(hit, "company", None)) or "" + match = DISPLAY_NAME.match(raw) + return {"cik": cik_text(hit.cik), "name": (match.group(1) if match else raw) or f"CIK {cik_text(hit.cik)}"} + + +def search_filings(query: str, form: str | None, start: str | None, end: str | None, limit: int) -> dict[str, Any]: + if start or end: + start = start or EFTS_FIRST_DAY + end = end or date.today().isoformat() + try: + results = efts_search( + query.strip(), + forms=form.strip() if form else None, + start_date=start, + end_date=end, + limit=limit, + ) + hits = list(iter(results.sort_by("filed")))[:limit] + total = whole(getattr(results, "total", None)) or len(hits) + except Exception as error: + raise Upstream(f"SEC full-text search failed: {error}") from error + + out = [] + for hit in hits: + accession = text(hit.accession_number) or "" + cik = cik_text(hit.cik) + out.append( + { + "accession": accession, + "form": text(hit.form) or "", + "filedAt": day(hit.filed) or "", + "reportDate": day(getattr(hit, "period", None)), + "description": text(getattr(hit, "file_description", None)), + "url": f"https://www.sec.gov/Archives/edgar/data/{cik}/{accession.replace('-', '')}/{accession}-index.html", + "company": _hit_company(hit), + } + ) + return {"filings": out, "total": total} + + +def owners(key: str, min_percent: float, form: str, limit: int) -> dict[str, Any]: + entity = _company(key) + forms = OWNER_FORMS["13D"] + OWNER_FORMS["13G"] if form == "all" else OWNER_FORMS[form] + listing = _entity_filings(entity, forms, None, None) + if listing is None or getattr(listing, "empty", False): + return {"owners": [], "filingsRead": 0} + + seen: set[str] = set() + out: list[dict[str, Any]] = [] + read = 0 + for filing in listing.head(_limits().max_owner_filings): + read += 1 + try: + schedule = filing.obj() + except Exception: + continue + persons = getattr(schedule, "reporting_persons", None) or [] + items = getattr(schedule, "items", None) + purpose = text(getattr(items, "item4_purpose_of_transaction", None)) if "13D" in str(filing.form) else None + for person in persons: + name = text(getattr(person, "name", None)) + if not name or name.lower() in seen: + continue + seen.add(name.lower()) + percent = number(getattr(person, "percent_of_class", None)) + if percent is None or percent < min_percent: + continue + out.append( + { + "filer": name, + "form": text(filing.form) or "", + "filedAt": day(filing.filing_date) or "", + "shares": number(getattr(person, "aggregate_amount", None)), + "percent": percent, + "soleVoting": number(getattr(person, "sole_voting_power", None)), + "sharedVoting": number(getattr(person, "shared_voting_power", None)), + "purpose": purpose, + "url": filing.homepage_url, + } + ) + if len(out) >= limit: + return {"owners": out, "filingsRead": read} + return {"owners": out, "filingsRead": read} + + +def insiders(key: str, limit: int) -> dict[str, Any]: + entity = _company(key) + listing = _entity_filings(entity, INSIDER_FORMS, None, None) + if listing is None or getattr(listing, "empty", False): + return {"transactions": []} + + out = [] + for filing in listing.head(limit): + try: + form = filing.obj() + summary = form.get_ownership_summary() + except Exception: + continue + activities = list(getattr(summary, "transactions", None) or []) + primary = activities[0] if activities else None + insider = text(getattr(form, "insider_name", None)) + if not insider: + continue + out.append( + { + "insider": insider, + "title": text(getattr(form, "position", None)), + "form": text(filing.form) or "", + "filedAt": day(filing.filing_date) or "", + "kind": text(getattr(primary, "transaction_type", None)) or text(getattr(summary, "primary_activity", None)), + "shares": number(getattr(primary, "shares", None)), + "price": number(getattr(primary, "price_per_share", None)), + "url": filing.homepage_url, + } + ) + return {"transactions": out} + + +def _latest_proxy(entity: Any) -> tuple[Any, Any]: + listing = _entity_filings(entity, PROXY_FORM, None, None) + if listing is None or getattr(listing, "empty", False): + raise NotFound(f"No {PROXY_FORM} on file for {text(entity.name)}.") + filing = list(listing.head(1))[0] + try: + statement = filing.obj() + except Exception as error: + raise Upstream(f"The proxy statement could not be parsed: {error}") from error + if statement is None or not hasattr(statement, "peo_name"): + raise Upstream("The proxy statement could not be parsed.") + return filing, statement + + +def _by_year(frame: Any, years: int) -> list[dict[str, Any]]: + items = rows(frame) + items.sort(key=lambda row: str(row.get("fiscal_year_end") or ""), reverse=True) + return items[:years] + + +def _holder_shares(row: dict[str, Any]) -> float | None: + for key in HOLDER_SHARE_KEYS: + if key in row: + return number(row[key]) + return None + + +def proxy(key: str, years: int) -> dict[str, Any]: + entity = _company(key) + filing, statement = _latest_proxy(entity) + ratio = getattr(statement, "ceo_pay_ratio", None) + executives = [] + for row in rows(getattr(statement, "summary_compensation_table", None)): + name = text(row.get("name")) + if not name: + continue + executives.append( + { + "name": name, + "title": text(row.get("title")), + "year": whole(row.get("year")), + "salary": number(row.get("salary")), + "bonus": number(row.get("bonus")), + "stockAwards": number(row.get("stock_awards")), + "optionAwards": number(row.get("option_awards")), + "nonEquityIncentive": number(row.get("non_equity_incentive")), + "otherCompensation": number(row.get("other_compensation")), + "total": number(row.get("total")), + } + ) + holders = [] + for row in rows(getattr(statement, "beneficial_ownership", None)): + name = text(row.get("holder_name") or row.get("name")) + if not name: + continue + holders.append({"name": name, "percentOfClass": number(row.get("percent_of_class")), "shares": _holder_shares(row)}) + proposals = [ + { + "number": whole(getattr(item, "number", None)), + "description": text(getattr(item, "description", None)) or "", + "type": text(getattr(item, "proposal_type", None)), + } + for item in (getattr(statement, "voting_proposals", None) or []) + ] + return { + "accession": text(filing.accession_no) or "", + "filedAt": day(filing.filing_date) or "", + "url": filing.homepage_url, + "peo": { + "name": text(getattr(statement, "peo_name", None)), + "totalComp": number(getattr(statement, "peo_total_comp", None)), + "actuallyPaidComp": number(getattr(statement, "peo_actually_paid_comp", None)), + }, + "neoAverage": { + "totalComp": number(getattr(statement, "neo_avg_total_comp", None)), + "actuallyPaidComp": number(getattr(statement, "neo_avg_actually_paid_comp", None)), + }, + "compensationByYear": [ + { + "fiscalYearEnd": day(row.get("fiscal_year_end")), + "peoTotalComp": number(row.get("peo_total_comp")), + "peoActuallyPaidComp": number(row.get("peo_actually_paid_comp")), + "neoAverageTotalComp": number(row.get("neo_avg_total_comp")), + "neoAverageActuallyPaidComp": number(row.get("neo_avg_actually_paid_comp")), + } + for row in _by_year(getattr(statement, "executive_compensation", None), years) + ], + "payVsPerformance": [ + { + "fiscalYearEnd": day(row.get("fiscal_year_end")), + "peoActuallyPaidComp": number(row.get("peo_actually_paid_comp")), + "neoAverageActuallyPaidComp": number(row.get("neo_avg_actually_paid_comp")), + "tsr": number(row.get("total_shareholder_return")), + "peerTsr": number(row.get("peer_group_tsr")), + "netIncome": number(row.get("net_income")), + "selectedMeasureValue": number(row.get("company_selected_measure_value")), + } + for row in _by_year(getattr(statement, "pay_vs_performance", None), years) + ], + "executives": executives, + "holders": holders, + "proposals": proposals, + "performanceMeasures": [m for m in (text(item) for item in (getattr(statement, "performance_measures", None) or [])) if m], + "selectedMeasureName": text(getattr(statement, "company_selected_measure", None)), + "ceoPayRatio": None + if ratio is None + else { + "ceo": number(getattr(ratio, "ceo_compensation", None)), + "medianEmployee": number(getattr(ratio, "median_employee_compensation", None)), + "ratio": number(getattr(ratio, "ratio", None)), + }, + "insiderTradingPolicyAdopted": getattr(statement, "insider_trading_policy_adopted", None) + if isinstance(getattr(statement, "insider_trading_policy_adopted", None), bool) + else None, + } + + +def compare(tickers: list[str], years: int) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + for ticker in tickers: + try: + entity = _company(ticker) + statement = proxy(ticker, years) + except (NotFound, Upstream) as error: + out.append( + { + "ticker": ticker, + "cik": None, + "name": None, + "fiscalYearEnd": None, + "peoName": None, + "peoTotalComp": None, + "peoActuallyPaidComp": None, + "tsr": None, + "netIncome": None, + "reason": str(error), + } + ) + continue + performance = {row["fiscalYearEnd"]: row for row in statement["payVsPerformance"]} + for row in statement["compensationByYear"]: + year = performance.get(row["fiscalYearEnd"], {}) + out.append( + { + "ticker": ticker, + "cik": cik_text(entity.cik), + "name": text(entity.name), + "fiscalYearEnd": row["fiscalYearEnd"], + "peoName": statement["peo"]["name"], + "peoTotalComp": row["peoTotalComp"], + "peoActuallyPaidComp": row["peoActuallyPaidComp"], + "tsr": year.get("tsr"), + "netIncome": year.get("netIncome"), + "reason": None, + } + ) + return out diff --git a/services/edgar/edgar_service/values.py b/services/edgar/edgar_service/values.py new file mode 100644 index 000000000..f61ad5ebe --- /dev/null +++ b/services/edgar/edgar_service/values.py @@ -0,0 +1,62 @@ +import math +from datetime import date, datetime +from decimal import Decimal +from typing import Any + + +def number(value: Any) -> float | None: + if value is None or isinstance(value, bool): + return None + if isinstance(value, Decimal): + value = float(value) + if isinstance(value, (int, float)): + return None if isinstance(value, float) and math.isnan(value) else float(value) + try: + parsed = float(str(value).replace(",", "").replace("$", "").strip()) + except ValueError: + return None + return None if math.isnan(parsed) else parsed + + +def whole(value: Any) -> int | None: + parsed = number(value) + return None if parsed is None else int(parsed) + + +def text(value: Any) -> str | None: + if value is None: + return None + if isinstance(value, float) and math.isnan(value): + return None + cleaned = " ".join(str(value).split()) + return cleaned or None + + +def day(value: Any) -> str | None: + if value is None: + return None + if isinstance(value, datetime): + return value.date().isoformat() + if isinstance(value, date): + return value.isoformat() + cleaned = str(value).strip()[:10] + try: + return date.fromisoformat(cleaned).isoformat() + except ValueError: + return None + + +def rows(frame: Any) -> list[dict[str, Any]]: + if frame is None: + return [] + to_dict = getattr(frame, "to_dict", None) + if to_dict is None: + return [] + try: + return list(to_dict("records")) + except (TypeError, ValueError): + return [] + + +def cik_text(value: Any) -> str: + return str(value).strip().lstrip("0") or "0" diff --git a/services/edgar/pyproject.toml b/services/edgar/pyproject.toml new file mode 100644 index 000000000..302d1825c --- /dev/null +++ b/services/edgar/pyproject.toml @@ -0,0 +1,24 @@ +[project] +name = "edgar-service" +version = "0.1.0" +description = "SEC EDGAR research for the CRM, on edgartools" +requires-python = ">=3.11" +dependencies = [ + "edgartools==5.56.0", + "fastapi==0.141.1", + "uvicorn[standard]==0.41.0", + "pydantic>=2.9", +] + +[project.optional-dependencies] +dev = ["pytest>=8.3", "httpx>=0.28"] + +[build-system] +requires = ["setuptools>=70"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +include = ["edgar_service*"] + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/services/edgar/requirements.txt b/services/edgar/requirements.txt new file mode 100644 index 000000000..d8f7ac2d2 --- /dev/null +++ b/services/edgar/requirements.txt @@ -0,0 +1,4 @@ +edgartools==5.56.0 +fastapi==0.141.1 +uvicorn[standard]==0.41.0 +pydantic>=2.9 diff --git a/services/edgar/tests/__init__.py b/services/edgar/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/services/edgar/tests/__pycache__/__init__.cpython-311.pyc b/services/edgar/tests/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d6e5dc80434edfbb440eb6008b5591d547c27707 GIT binary patch literal 156 zcmZ3^%ge<81SjfeW`XF(AOZ#$p^VRLK*n^26oz01O-8?!3`I;p{%4TnFMa)t{M=Oi z(&E%2{p6xteIQ+ynVedzpPG`MSfpQ)T3k}BA0MBYmst`YuUAm{i^C>2KczG$)vkyY Ys2^lQF+Y&_z|6?V_<;dN6fpzE0AIQ!6951J literal 0 HcmV?d00001 diff --git a/services/edgar/tests/__pycache__/conftest.cpython-311-pytest-9.1.1.pyc b/services/edgar/tests/__pycache__/conftest.cpython-311-pytest-9.1.1.pyc new file mode 100644 index 0000000000000000000000000000000000000000..492e8e8459a1582631afbd19dfb17bc80ed0258b GIT binary patch literal 1717 zcma(RO-~y~bar>_^%@&eA|O;sOI0OV`L*+aZyo`r|M~zsvr*+V@kw{7vqwn7j-yQH{lsY zLqR0IDZe0W9rm>?_8him<0tU#XRxCITKZI};&GafR$%>1m zTv|sc4*2`9&yzs?3@H%&E;{^&Ko_+YTYb}c5s<5R>Yf6r(d?M3d3XU0hgMEbutHYf%zVQ8`Gz#)u^jvr^$H$A&66LA}ICV09fs zU<{$_u+K%Bp{+y-)#bv^h>DV*!$z-jxeebVxhf+xXHhQ)=ayp;mLvApCe4Y8v7F_X z*9DfX)L$oNT+@4Ko3lGNELs2TjO$yb%Vq(II0ImqiElR0^Fjkrku5$b4FT>Y7eIXHYA>zeSwIMYl zu|AQ%Qj{?!-uoB~83vC>R5}V&VUqm?;CGk}bczRF;^7lK{OH@m+A$t);_(oVxA1#m z?`q)>!rmn{M^5m_fp+lK(+5u<9OLOGo(}PJOW2G!;mteY-33_3IN!wi5a)k2u0L9M zJ`@_0$HrvSm^?#D?ClpHwfaZHM0=fSK$3I`UI~dm1?bYQ#4au_uH;Du8-&=8ZpG%O zw4*6JKcgRgtN`R`vl?vp)Y&mbt#;C%5R<~+C3+Q^0iJkRb=<%yGoG5O)JvDQ(Nd)@ zJJ)1TaG{%}WScqx;l?G2>m6DWlx3kz%2l87cy#51>n01#iWB|8WZTPna~;~np?;Ys zp$%-ZZdToZ>+4P}sDgi@q8!hEURZjgb>S6xzFl?6EWHgf;-Rwd0YGjQr6nHD71Llv2)N38W SFfwi&yc_D71{PC3+P?vnh^MLm literal 0 HcmV?d00001 diff --git a/services/edgar/tests/__pycache__/test_app.cpython-311-pytest-9.1.1.pyc b/services/edgar/tests/__pycache__/test_app.cpython-311-pytest-9.1.1.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f86423c29ecef3a1a325e83dbc4ac7598b697ae2 GIT binary patch literal 16648 zcmeHOU2GdycAg=Je;SFD(W};l|d#e241-=o$Am+psE%I?bITmU0M`0 zuEjtTS_f#iCLfnl>H{ipA|+cvL(kzyIsEpCH*>OG4f=`TKZyTFCg?TElolW?zcl$K zQr&h^)?`zjZb<1ozLk~}#%tv18noV?P9m?{_KS^)nF`dgmnQ;qVh1p<;8@b*(p z(PCPMrh<0fP-+oVF(dZ)M9rx2D^CsF;}gaBbdmP&%=jqYe)P1wHP#WYmKMLEh;h>r zTDKXtM=`dBQH*(W@=ESe#7IVK5lzCw+A~p&^qXE6Q}ETq+v+V$cdgm&g#2y^rHr z!_ZlU=dmBE2;6_78XD*u>>H?t_xA1G*Ef(-tUzhTik#P(kuMdk4qcndvP!8`Ff28% z>BUOEa`CucLG@mzx{Nfd2ga^mn;K6WrD8|zl~k*`sKcVdI?-g-1w!>1Vcp<7H9aV5 zdN8G0!706BsjjA59p5WotmsB1lVw&Yp<3{ErIa=EX3(OSoQPRzVa-pG{7 zdXe=bQ+odzVZ6~VN%s>QKCXULy`EZ1ywpg%bm`6e>bi!yy{_z97zU}U+ZTrKs=J8g z_dR7-qsiNWqyl!S86#925m6#!{(uvhavZ;RRbs_`Pnl^nc@e@@z%Dgo zgo+~~N`z3m?ZnFhT1si5WNVDOB~-=9K@ZE6`EU+=Wh@&SxCT5B_NA#Y0oR>IuOML1ZA*k&%e1~V7+ z=H!)ZGbh`CB}kwJ|6z>wvUx&4B`?>4V!oKc&(xcE?cC;IlBN>@O4pnTW6s3Puo<)G zOo!QF{MOSa?m5$eInznn&*qGHHUX_$^KWf)#s^D1ecj5-2Ya-x8xcOUqFP)_m{EIX zscV>7s&_7VrFJt5v#X;P<6x?LqPn@CoBv`Srq9r`EO&1Fy);Z-LC^mLFC<3QPRZ37 zf-eV}@CA$wU-mVFz^smFfJOjlY!n1svoa_f984MPWhyh=!a8gd0toQzBXYn}$`=O* zs|goC8yp7iq2OVwD?3*?S7P}p2OZVU-$SExmVBR;ic^cS6@dlU0N2o_KcmVF0v`&< zNSxh-6&l(Gr+HenJ85U0rkv3u?VE}ErXs!;vLQ5+4O3gGDFbXTSZ^~4vr^pNR0rEa z+FOY{Ph=aB7l>>pk|eT&$csdFf}}cm&$3+<@CuRLAZ`G)UtlI1jt#h@4tn6!kVhKv zB(vk}Oa?ug`wRlG4hjRq`lt?DIk;k_Is9gdx~^p^=kyF#<4mux*P)@}-=H@gK$hBb z^-x_oQr|cTI)7;X5V$3EsG$x~C^2JTzSzm7u`hPsRgVzM?|aITMw7P#Rj7bnYQ_i^ zM?{nencwGxEemKV9TG}*Ij9)V))v(vuQ=uwzt|}xQ7!O-ai|DEPIhL^D!WrT%Iz4t z^UA#Q%6Ik)e9mmeVmQ%2ip`f=dO`T1f?Uk45hLNK0(jn?Js@c*b61a4$JNLo-+w9pM@irl{82`fB2 zG%$E@xY}`;08jD{in%^Ge-9sf%L-3q4K4d#O0j~u{EVe=hex=Q&&}w}is~0~=d#5q zeNjpUEcrbvRLIZfD+WE1wncj!9W7*M&uZDR>K<;}?5e_^&9S19fo~*tPB;2Sg_mUv zj8G@vr$NT~lJw<)`k8!v?t_Pt^oM~T1n~2QeBAZ(x_lyVN#bvr>W;QBZWpgxdYJ7; zri^9;$8?OIKkkBI^mH(KgtTIbvqhBmjFN#L&qWr!?gd>|2 zao+`TV;B#h84@wVKeX2^>|Yh6tUQqH49X&JobjJP%>XG%;$M_aca=?_2J74UF?2&G zXyicu3m&7l1OLq_LcrTzqxzQgKUe0fONqA{iMO5(*U`u6kGR+9`fCmKJGbP9`rTUx z8tO=08M_k(sjDM*!gmSH+4ntVtkLA{KvDs_#AEDub;K>+2@6Ri0Pkey4na~UccJf2 z4bgqHb@+S_>Y8_@>HG8+L)`^{{; z&myPFvCPw47Jtpwe zgtbns3!WUmcji}~R?xeeUqD~vor$mLo%#PdGJapl${5Y>D@oX!O;N2|>v8!?o;-eQ z!54h|T>g>X36CS!K@_x5za~}irf*&?x0@!L($BFm{i$y%e<-Jd6V(k9rR4D=uO}z- z1vn*a6p(x@_oV{dI9kz(QsroAuBgFR5rl~|s1SRV9GT4-DxL8fNfW;|X97g1SidsDZMubH*s#{iUF=f@qHlQ|aBN3_`BW%ll2-5UaoTLDa z2;oxUxLCo)vsWpWoEdBnNE`FLa}AxwrjVZIlCcNa=aTd=ActOiC^;l7g}R$|{rbDN zB1_6ZLm8+m1C9-U^_M5eh97Dq4m}+_i0I}ij#vV5Y&AmR%9en3ms(U zPUY73@vESLUC+<}9;xZpir@EY{M|+aS|{v$a80Xz*Gj*JoftEGUaHZqO}kZnoa2H7 zX|8cq@3W^KywO_w`)sq;ZLjK8tw-xMReM$6xrSAJrxz!AC3}Y=#w~7`ujzjFuU5J=aBzZEte{@h*f9hfR4wkf79Q*HSyr0Si`CuSs?6DWWT=EWrjte9J22F; z!9x2*%81(InBdW+c)Af!*Q06M?gAK4jjKCdc2{^Q`t7K3s$Y6=V9)otEq?IZC);9$ z8q6k9Fi=GxUtDyp&pwqM0JO<3YU{=W8P1=6LsP%j^SH_iIueE6hUi^A+ zW%PJz+cLL4qpbBhPV{x!ddG?WZEZv-*!Ir>A(t_~;Cgz_RDUjg8mQrL9GptwRkx-(%}$rws!Q8ki<@0$+}_$qn2Aqh<5f@p zxko#J(cVJ(%SL1qf6*aQ1@Cnm&51gt>cfFDZPkO%WSNA@av9%lk@;;!Z-1LBmD;8wZ3T> z_rl@pk?Zsy5we@|5qUr^(CQ1Tk4THUPo3awMz8-F8K6leFbH(R{OU5f6#!vpLvWk`>==S0)6N7~D$Fkj*| z7dx%r?};aT*V?8P`vgUiW7GHt^bbAEYS*PV9xB05??Y(?((y#sD~;%1%v$zB`qp{~w9ok1u?5LA*Hre#0%cxcRfqkd`(LH$W4^jl?hw zEI%KYt8YB~`2`TVm7F->b@;21x_WF$J=Rc@fG^rldUVjQKCk=^Hp(B z%vH>fsBKQez$2mi79psUT`k@h(08YD>#xEuU*S53z@E8<@pgOW7N$<{UEWoe`vBO} z>lOy=aV`Q3+%230<8=%BKf%SeYhdDWZ?04<>N)xhEy?sreD$T_JKxS9%;5{YS)G5g z#?B%y*KR8-LbeZY0T5(LIFZbr;NMMC7IcD({JUG~Tl<&d1C97VJvtyDw@z91Ec5IN zL|sPY>U*l}FG*#R2qRJAxQT+wsp2Gan8DFFyIl1J>q@_(rr(oZrU-)5rApS-yhrJ)IycVu$fh53DZ! z%_NTN^G_(nIfgh*GougB-6V0077P4yNq(H4AD82s?cCANu`Zu_d}Ssc7Fx*S80Bn9 zn=9yJ>;^=72N?eZ@-QgN@_l;FR{pC?+kE$ZX?xv!KMV)uf($I(qMZTxJ%qSuha!(Z llpGR|$iEK-7hNBbalFqV>*H`#Ka^bb#fY3H1wY}9{9jTqFMt35 literal 0 HcmV?d00001 diff --git a/services/edgar/tests/__pycache__/test_sec.cpython-311-pytest-9.1.1.pyc b/services/edgar/tests/__pycache__/test_sec.cpython-311-pytest-9.1.1.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d4fbc0e9852141b31f96de901e91118e347ec122 GIT binary patch literal 25178 zcmeHvYit}>mR?nN^@A+FNl_2#(UPb~OX6G9%a%mRl16@rmi4g5E!piBt4KE0?53-l zlGveUxC8B)p2!#~v0{2xv&$d@FFj5c-p$Ug|199yAc;2!f@-0O^wh$Lksufd`$vlG zU?Sv4zH@JNbyZQ6G_jLx##3E=>fCeBxpm*S?mg$;Kd!E>6mWg^{r5-CEEk0TLXOfE zu#o#Bq9D8{D1suUgqX;lzES^xPZUO_SU@BnUn)2niiJkQu`qM{Qx&6?u}X%EDLGaJ z9w}8lS`({bv_PtMv@TZ1@L;Nbv?10o+8ApbZHhICf*SG(-w~A1H9-k~;1h(8@aNYD zczjfH2(e`rX9YMbAIaHlamwJVdL-v^i?bS>HIL+6!D3kpzB)HwY$fxlhfjmYXO+@; zO^CH9O@OPFWq@mxX27+|a=>+pcv^_AxI-!IkNV6CC6P@Gq!PMr1_qOAN{NbQ=!iOy z98IKv`gL^xkMLwU!plYU+Rj*_R?Cb*Kw;cBefxIWEtT=&c-u?ah|xk#L$qY4kfZ`JTaOXPiM_~J(E)7mok*D_*mvLQq!mpCp1-Y z5u=r6I3A~9<8iYx9v{sp&U8lR%bAd zl4u{B(5ff|0y6&k2*8x%_eEv()U#91{?4<-=9XFIM#_*j;Lb}MiW|2WQq#=9?2#KI zh6H>|UfNP@-D*h7X7t%JH*$sqd~06XTHL(Nke1Ij&Su|VWk|rcjmS_8eY zkw6oHwFH(CXeO|d0E<)$VQWf&c=as+uDt2!F{H@c#(C+VB2~b9@(cpuVUYO46Gjnt zCaheSeW=Z9@SpoWKnL~_{``uC)Sz9vB_ue2CNwatQq?i*HFYeb@oL9vt&SE}tEhvC z@l-ZGh@@w<2{S+$qxVO{o-8n>R1$TKC11kZc%nBF=E;o+(FhP-e+s~r8_Qb^iSE4A zQfyo?(>uS+Xza*0b`%;rrjAbc8d3-DJcB@Z4pC?i&!I8+JUoY3Ua_KQ7;EtiW9=Nd zCn>yj2hj`om7vu#gji(>A}7nVFr%R>j)j#Va<$S79!sXseobjGqm7#KAaljhr>SOD zVgT((M>UG4GiDVvv$4dG8XwnEW>~ihqu-QyX>gh4V;VX>Z6dB@2I7jU4`|6Ts)z%2 z$s(DA`|-a|jrl!cQlLi2uhYVZVt-WBw!?`+m2@>Vs1tugqY7&JGZ#j3l|0Vvz@s4Q zG(1iT#oET}SB>y$meJ7)PpLcVd<5*;4gzfix(IkmelyVstixY_3&2(Kb{|pSggyax zUTP{fEJIg-J1;GB;Wf>MM0Z|lF0S1$_54iZO!fvkhEl(>tsQk}_VnD&_hW_xyge_q z-<8C`n!Cbcz-tl3%vFm@xqCkXpK>iyM3l5&@d09_01PO+Ernh{6F{*K*rPV&2xrC8 z$^w}bCxtjLeocy1D5!YpMfjdLEodF^!w?wawFU#AIy#n}_|T_y!b9r?&?(S}Rd^T% zlIdhN9?#WT?{6x}R^9w9q7 zz(hEWDEET^6d~)itZLaWjk{hNM8;)REmu=|EqAkw!|={uJT0@A^)K$r_kcyC7Ofp5 z;HkqsL?b}8Tt5UbW&iH_e1Y|M1sjCLK!mEB1%hIriJV@*YafN_s(qzObiWlo&e}(H z&#rkvC2&oQ1z8Qm_r}q5HkqAB{`@~nLay@A*jP%9JewY9cNMZ-BWXB7p1NwJT?4g! z@YQw$I5StH+{xAQ*Vy%JKXFw9*!66>YPyPNuDP`B=BXG|6*K~S@%L1WVrAXbGevp%)Dh}})=?#}0P4ghay}IB7*Uw6 zib0=ktdX7v;?Bf!7G_Ix81N{f;!{v*6bxxfPzeEsm5LA0u0Eo+#V&W2)9i!>vi@B2 z8GPzR=zFhEy^1>p{5!X1SGwU|59xbM)L@6hNG^=s5@&5D6cdN4UO zuBjTeKX1XW8Vv`@P~)jvCIoIGWmdBu1&W}JUL z-w-b}#PiY{1?df=bh8|DWmW}(W0{q!q0CamtDaitPWV+ZOWf8qDSCW;itnm_(l_b9 z$Xe%BX;PXI^$pI<9`RmL^ZuzRB>U4|TC(vO@p2u6!%^net=>`pjD5{QKT#x>8Usc~8oh z;0n9*X9+G(AEngYl30IJsnjbCla*HQBR@g!BfH<{c3Hho2)#}i|B93>kUduR9FoqM{v+B<)V){vV0iyMbN6d(BZ?~jVP7dLGk zo7f%IEB}FoMD|Ar>x}AK>DgQI0%Cb}SzbL+Jz9AccVs`8>4K0(6Az8%mC>N-M>qt! z2=ox>C9vC+K! zguWm3J^Aab?8NIHRQIdd2wkTWDOG<^K~9Ehw9t`j?C$K|)z;b7*16|QXXie4Kd3Ad zHLR5&vKJCcw3-zPD;~45Z)}2AQ{oBDtd9>QQmHtW{k4J2=ol1!W)-~S7sj!gnM~_u zFqHv^Zmx*y!UT zER{$zO&}`*7JnAKXq73`3waiHI~_Znj$Ka2UZ-OZk1>B!SC`Ax<7Dh|I$S=z+=mC@ z4zE)u^&%?-3vlJd7M6pT5~*=jM=Fsq3fB|s$dxoSpCCZ-qH7ZnKaRitQ+%vrrE=9~ zL)vw7%j}MP!{GL(+H=AQ#FDao6gJdzFqB)QmZfHQFy62e}_ zT}X!0Ik(de58}Z2>uty>5VfRBiE~QX%!`p?!wN%MkIdZs_JQjMfX6#t11K{ribl&jg;c#KjgRq=ZPJSCN)}hUGyt8wrHZMmC zas-aK=K%~kGWYy#nbhI@UXVHpCEOws3s9)bxesTto$%t6cTjCzEDnOrga|Mu>pgPB z>70As4uQWA(0Rj`2V}>R^E;)S{5D>Maz^TXr^l#m&C8n#@+P?ER|6RGruo&kWmp7Q z_kz@0DB%{7Sb*kBKAgpN!i!Vhfz5>_yog{kQ4AQD^&%O42MKZK{AxQ0{zgFO4PPFT zC0&Y@Q_9J2dhGd=$HzU%vd&XRk2A09G#S?gYq))Y!OCo&FcO}^y8Icz7G zIEB(~$u1(;Ow;3>iE~WvbGl}ss4u@9xE(2ua0(>3Fo>MrDdiDyI=N2A)(v6+Cj$qj z(>afo__soDh2N@htHIn_6j5-lOj^n?$vK0}+N*a^mN6CC2MGTodsTACqfz((-_;-4 zI-4JcUExEurc^5OBf_szs@>tc-e05CDs{BJ=YGqOXWhu{`@;%X92%7-Wtq~vWE_?& zE0mS)I1Gu-8NRa0z4GEV=JUKeG%3v4xq-5=uPlABS(nl>9hweLR}A`<)z`xAccX+z zxoTY_c4;VUl(ot_CF0hdT@{t}*PymmHk21WjtK5D#|mbRvV3TfwoXqu*Gflh?ToUK zX+T;ZrU7wD(C0q8yz6*NaC!Adn{N3m{n2J+i$j0pSeJ8!War=$>5nYEkvngcP&uEJ ztz{f}w92c^{8r{qLpe`8`Qy@`Etxk_R(H1j#?+nd<@`V$srz=;5oQ3fqd!-3`kCWL zkDPtsXr!y>I~|8|)~vp}4GSm!L|a#`X*io5)Ax12Ms|SNMs$E`hF?ykhsG0{GW7fJ z)798_hla3Rl`}XNSZ`ZrDX`9m24*c?2y6T9kk;R}>0#1&W+sd1#bDHA6>JsbnUbA$E>m|?QAy#!M_ zm~*R_$=D=-Sv^crDLRpO6!ba7NnZ^x zB^YuSzh}Fq4^0nUf6gYbZ!`MwX8coQrh3 z_#)jdGh~@sYh={q48&UVLm(wPTKyEv2(gaF?1nZ|un__@QZlP!Go-7qb5{ts&;F?M?7y4 zmid#EK#YkoBEBS^BO*QEsyWVco*@@K;Ho=I4#277MXB~bO3QCc%YW>jjTzCGJ$GJur69dx zNU!|Y>bmQ(+3?)@xwG?!^VJ=N>W-;nU$BZk7o59Nh;*Wo=jE;fNYaUqFMhh$=sf|9 zs($Z@+cLx9luj&2CkhN9Uw$(l;Cbd{F(5dnv)A(FBJe7UGn`W(YlqLpIrK1dXHl-7 z!P;fZ?4g3Zc6Op5M-Azq(T3HMg9h-ngSTad!zmqHkPa3YLcaWFJiznJ%VLnDE@zwN z%SGT-7H2r8K-LbQi*x8<=FW#jyK5c`Tj+J7cv1;i_kz?_DB%{7u}H{U^0E9W&SmF3 z)}+c}!0q5T!YL57L+AWXDUS$6xs9#GY??b{NSkOa=Fn|go3QQ$X;YzuTSQ_3nrA+i zKdqqQ5|b`RO^36%&# zcI-@p>AT`{PX>Qj!F*r{uSg!BF!``apATlr*c{>UuVns}F!%|oBc|I!Y1Lj{4l+PA*f|06POHO8u9SSZu4XSu~<&rUehO+gdOuq&*#(e=J;*Q9puc~a?U909C)OR!vJ*sZ}SIBa?bp5EczgS$gmad1xAIBYFQgu%WD12 zXj!N`n_P7S3rUKgY`x|G7PqEoIY(TzWm}n!+Oqvy+nRQ`@}W&>$I|Gxwdd_9=Y`T) z#!)}IzO{MM?aGIqZ(py8m9yF zJuq#5^u^aJDO72uKGx3FmfcR!I9B)vtF^#xN7+?&M9t0 zB|A1mo7E@-7I-{6fh9kx)s8#~+pqp9Jd*h9w7K4qE%go4!D4gUv|L=VeY%={T>MYK z=E@Z8CS%wAOQ*|=aW4aWK_>5dJM;a(Ef8z2$)W(6(jcN zINQo+X&Y|^6lfKp^CI1$d@JD97RF?cY!!H!meHD~guFV=Sd~YjSz%g3WtGx`|7!f1 zcJ7wENV3`^8Bx|N8*cd)@zwZ*cRhN$Au(2~Y*bo@e6c!ZQ>-2wqI)75Yfv`F8kJR8 zE8X&dt+&S?*X~YZHBcSJxf>6{C$#p+Gnvdq?IcPsCwF!3+1cH*yQinU6RYGsd%C)N zdiN0Cv$LzaclSgfkUt z2a`HX_2Lt1LW`?u#jF`ai_YQzjudOTW;405%8he6unIkvNGhD(kS6-2At&#u5^XX6 z1JXr6eSoO#)?PxCeu;l!{ul4F*t{|Gvr7!Q@>fTv4h^0E(O-YH_36<*j$C=O4{4bC z?Ir?JqP6%!B(|^+@ z^e6ZK-Kak)wEU%zT>Fbvg7y={Au6+tw`Kqvn{i5oM44&^(S6`N1T%0UlOEU2Dm@Er zNIY>FIttya&alG~O3tPu#qj= z1(44~o4k9A!&DiVFfA}>|B?VJfu9k^iorLgm=#BGBF-@lFEN|dE9$^FCBq%xhOxv% z{E|+(4rV}!#G0%0@lkek1eHl?K#6B*WHOsBsA+XDIgmuX%Z zBfwHo!VYz*QZ}n;0He6DfyrD8>(wOmH*|!G!cVnHB?n+=s-U1$1-tmEchzXzVR{Zr z!DCj~?N<93erfXmDf(2bc$a>~#x>JnoM|w-d~S7qWqV;|`*cll{nokV^K0_!cNf<0 zHo|L*Yg%VtnLD3f(_L88ZG=}9H|>~vW&V7A)4sx{eMWdgaedod!%tR!(c1axbH>Ox zR-F5>ppC~B-(f!<_0L$a_*jm+0e^MPJ%3)*>tbMEq`0*xWsL>L=%m&5`-uPg+0Q zVw``&&~gy##~L{vDPNWz$Neu*kc$5zJudmbM~|BCmtl1ug}j+U-i(kpAmqr7S?$)O zvHvAw=%Ue%QB`PM(hpnbDSg81t-pFVKG2W7kC6r8Q(3X6l#A9`dj$x?EsFp zaMveR(5V8C0K7athg>dnYX6u--X(C8z&`!Z7_>=CV`y ztK9%K$4}mR+-FZ&O-w@8m#!f{%kR0<)6dM*7OEmPh1CwT!8uSIo}fr+e@RiPv_~DC z85!{NF#ZZ&+FuhmPhgffLwrQkwZ9?KzXK@cEJj^+0NRU#&#<%Ig6zveH!F}9_SufT zC+x$W&Ok659plak{sp14QB?c)EIhmN?^PE2h1s^iF=BN717Hh~k!$+Kx85(d7 z@$u#iz9{;^$4(;BB82@ufgpex;5B?nb#ayU2gFNFNV`Sg#{_HzvG#}L{38PPn8sSO z_Mbp=ZD4>}*wPEPJfdgO?m>>eC-ez?B|+PZdTpzF?@Imv4wG37?h)bpPOAd_kz)V+<5N7 zf|4>uGkImKpp50K-z-$WId#mV%ROpDtvfFrD@ex-=@@BoU!A!!SD&xmSg77e8r=Gs zP1pAu(#TxwlM&A2WKsQ`m7FtuZe@TL4Wu_gU(**YN=9v)6CS=Llu@f^@D> z!Yv}P0L?QW%O4>*oS(hUU0eXTC0Rl^1)_H7oZl(s5g{$;4s7X|6ASW&xm^W$n<4F) zN0#g{^+3CdqLV$DB%{7Sb*l4kL54pP(Nbi&Us{ZSq!)x97i|>ymsiE-znu0 zaYuP~LEesI_U5y6?w-6IPnpi&uwxl+DZ$igL6d!;XE}*ml4;^|Enc&{#i2cVw7|@QMuImKhGGG`t`U7Z^gm z{AN7B^UTX)kYTb;oR%|U`En6>mBks(DPXq4=i(fCn7NbUzMYH_-Ym%L=QcBogg&FK z&yf0cZ3s}%Q zOclHeVH7j*$M}vNWZ!t`W)!b}1od z2aS_k7N2+JlCWdywV_ukoTnX)z%+$l?i^*OVmV_#mVfMW?l*9+o#NyOr-U7Yr6q96 z0L)FEFg{JL(Czqmb5V2i*|G5Q;oX(45X+pN^<%8{@)Kd$`mbX@fh{2ju8?fglV4{) z!K+XqTou0ud)3nxc3HeET=BmOqrYfP|05=Z5tNI&b$AVST$4eOO$HrDlWK1xfZRUE z7xLv~7Tce(Wx8b9o5Vif3=aLn>B$eV;id0$_TA0qgNOe#_UExF?_4lfk6qDhp)%5+ z$sVJCa+Up=h&4rqO|CDQj#|6a$*?t)!LnJSYsdN?ZVbC8TEizO_OwL9`VBW)HN&MC@}pNh6BU{PX~P7P z1X2K4)MZnauZX~AE0+i(b5=G{aoV)*)iKE^ZL#KRq^(by_1gb{D7{JlBlLETS&HW? z{-6lHIwXAkAWYYXAEST*}9EYz+s)^z4;x(YR2M!4%&UxkFK z+Url#QHk5`5nx!cw#5iXf2_~-zMs3513Eo9n=@*o!1%oojuuKdr!bCiG|xP^_w*!p zfq@fjr%m~>Zqqo6gW!CV&1Fl@_1ZKp=^)rlvpJgyG~hf;8*&Qdc@Ufj33E!>%u69O zdA#dK*5RqsirBwccBo$Xd41i{r~E%(f28*4F8{yY6@VieF~g-(2Q;ct=E^ufr=Xpm z4FHv7yQW^&a1^vzVUx^|MPtU+ri@MZ`~-Jwy{C~2f~2pl57mH{RR znU&?MAt8q9TQrUc}sC>E)*Km2D1TOWECg%-noe<3s)rTYtE zjp4qFLey~IMWNeR`o0?!d@`Drxc#oM7~CTo)~Cho_x#|ywvuz!F&R Date: Thu, 3 Sep 2026 14:51:19 +0000 Subject: [PATCH 16/19] =?UTF-8?q?feat(app):=20a=20Research=20=E2=86=92=20S?= =?UTF-8?q?EC=20page=20on=20the=20edgar=20service?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The page searches SEC EDGAR by name, ticker or CIK, shows a company's profile, its filings by form, its 5%+ holders, its latest proxy with the named executives and their pay, pay versus performance, holders and proposals, and its insider transactions. Reads go through an authenticated /edgar route that proxies only the routes the page uses, with the secret kept server-side. Import hands the company to the agent as a prefilled chat, so add_company and add_contact do the writing. --- .../(app)/[slug]/research/sec/edgar-client.ts | 80 +++ .../app/(app)/[slug]/research/sec/page.tsx | 46 ++ .../(app)/[slug]/research/sec/sec-company.tsx | 620 ++++++++++++++++++ .../[slug]/research/sec/sec-research.tsx | 159 +++++ apps/app/app/edgar/[...path]/route.ts | 54 ++ apps/app/components/app-icon-rail.tsx | 2 + apps/app/lib/edgar.ts | 34 + apps/app/proxy.ts | 2 +- apps/app/test/edgar.spec.ts | 68 ++ apps/app/test/onboarding-gate.spec.ts | 10 + apps/app/turbo.json | 4 + docs/environment.md | 3 +- 12 files changed, 1080 insertions(+), 2 deletions(-) create mode 100644 apps/app/app/(app)/[slug]/research/sec/edgar-client.ts create mode 100644 apps/app/app/(app)/[slug]/research/sec/page.tsx create mode 100644 apps/app/app/(app)/[slug]/research/sec/sec-company.tsx create mode 100644 apps/app/app/(app)/[slug]/research/sec/sec-research.tsx create mode 100644 apps/app/app/edgar/[...path]/route.ts create mode 100644 apps/app/lib/edgar.ts create mode 100644 apps/app/test/edgar.spec.ts diff --git a/apps/app/app/(app)/[slug]/research/sec/edgar-client.ts b/apps/app/app/(app)/[slug]/research/sec/edgar-client.ts new file mode 100644 index 000000000..da41bc446 --- /dev/null +++ b/apps/app/app/(app)/[slug]/research/sec/edgar-client.ts @@ -0,0 +1,80 @@ +import { z } from "zod"; + +const withReason = z.object({ reason: z.string() }).partial(); +const jsonValue = z.json(); + +type JsonValue = z.infer; + +export type Answer = + | { status: "ok"; data: T } + | { status: "missing"; reason: string } + | { status: "failed"; reason: string }; + +export async function readEdgar( + path: string, + query: Record, + shape: Shape, +): Promise>> { + const url = new URL(`/edgar/${path}`, window.location.origin); + for (const [key, value] of Object.entries(query)) { + if (value !== undefined && value !== "") { + url.searchParams.set(key, String(value)); + } + } + + const response = await fetch(url, { + headers: { accept: "application/json" }, + }); + const body = await bodyOf(response); + const reason = withReason.safeParse(body); + const said = reason.success ? reason.data.reason : undefined; + + if (response.status === 404) { + return { status: "missing", reason: said ?? "Nothing on file." }; + } + if (!response.ok) { + return { + status: "failed", + reason: said ?? `The SEC service answered HTTP ${response.status}.`, + }; + } + + const parsed = shape.safeParse(body); + return parsed.success + ? { status: "ok", data: parsed.data } + : { + status: "failed", + reason: "The SEC service answered in a shape this page does not read.", + }; +} + +async function bodyOf(response: Response): Promise { + try { + return jsonValue.parse(await response.json()); + } catch { + return null; + } +} + +const usd = new Intl.NumberFormat("en-US", { + style: "currency", + currency: "USD", + maximumFractionDigits: 0, +}); + +const compact = new Intl.NumberFormat("en-US", { + notation: "compact", + maximumFractionDigits: 1, +}); + +export function money(value: number | null): string { + return value === null ? "—" : usd.format(value); +} + +export function count(value: number | null): string { + return value === null ? "—" : compact.format(value); +} + +export function percent(value: number | null): string { + return value === null ? "—" : `${value.toFixed(2)}%`; +} diff --git a/apps/app/app/(app)/[slug]/research/sec/page.tsx b/apps/app/app/(app)/[slug]/research/sec/page.tsx new file mode 100644 index 000000000..82abcf56c --- /dev/null +++ b/apps/app/app/(app)/[slug]/research/sec/page.tsx @@ -0,0 +1,46 @@ +import type { Metadata } from "next"; +import { Suspense } from "react"; +import { + PageShell, + PageShellContent, + PageShellDescription, + PageShellHeader, + PageShellHeading, + PageShellLoading, + PageShellTitle, +} from "@/components/page-shell"; +import { edgarConfigured } from "@/lib/edgar"; +import { requireSession } from "@/lib/session"; +import { SecResearch } from "./sec-research"; + +export const metadata: Metadata = { + title: "SEC research", +}; + +export default function SecResearchPage() { + return ( + + + + SEC research + + US public companies from their EDGAR filings: profile, filings, + major shareholders, executives and their pay. + + + + + + }> + + + + + ); +} + +async function Research() { + await requireSession(); + + return ; +} diff --git a/apps/app/app/(app)/[slug]/research/sec/sec-company.tsx b/apps/app/app/(app)/[slug]/research/sec/sec-company.tsx new file mode 100644 index 000000000..30fa39edc --- /dev/null +++ b/apps/app/app/(app)/[slug]/research/sec/sec-company.tsx @@ -0,0 +1,620 @@ +"use client"; + +import { Badge } from "@crm/ui/components/badge"; +import { Button } from "@crm/ui/components/button"; +import { Link } from "@crm/ui/components/link"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@crm/ui/components/select"; +import { SimpleTable, SimpleTableRow } from "@crm/ui/components/simple-table"; +import { Spinner } from "@crm/ui/components/spinner"; +import { StatCard } from "@crm/ui/components/stat-card"; +import { TableCell } from "@crm/ui/components/table"; +import { + Tabs, + TabsContent, + TabsList, + TabsTrigger, +} from "@crm/ui/components/tabs"; +import { + type EdgarCompany, + type EdgarProxy, + edgarCompany, + edgarFilings, + edgarInsiders, + edgarOwners, + edgarProxy, +} from "@crm/validation/edgar"; +import { useMutation, useQuery } from "@tanstack/react-query"; +import { useRouter } from "next/navigation"; +import { useState } from "react"; +import { toast } from "sonner"; +import { useTRPC } from "@/lib/trpc/client"; +import { useWorkspaceUrl } from "@/lib/use-workspace-url"; +import { type Answer, count, money, percent, readEdgar } from "./edgar-client"; + +const FORMS = [ + "all", + "10-K", + "10-Q", + "8-K", + "DEF 14A", + "SCHEDULE 13G", + "SC 13D", + "4", +] as const; +const FILINGS_LIMIT = 25; +const OWNERS_LIMIT = 20; +const INSIDERS_LIMIT = 20; +const PROXY_YEARS = 5; + +function importPrompt(company: EdgarCompany): string { + const ticker = company.tickers[0] ? `, ticker ${company.tickers[0]}` : ""; + return `Import ${company.name} (CIK ${company.cik}${ticker}) from SEC EDGAR into the CRM: the company with its CIK, ticker, SIC and state, the executives named in its latest DEF 14A as contacts with the filing as their source, and a note on its 5%+ shareholders.`; +} + +function Status({ + state, + children, +}: { + state: { isPending: boolean; isError: boolean; data: Answer | undefined }; + children: (data: T) => React.ReactNode; +}) { + if (state.isPending) { + return ( +

+ Reading EDGAR… +

+ ); + } + if (state.isError || !state.data) { + return ( +

The request failed. Try again.

+ ); + } + if (state.data.status === "missing") { + return

{state.data.reason}

; + } + if (state.data.status === "failed") { + return

{state.data.reason}

; + } + return <>{children(state.data.data)}; +} + +export function SecCompany({ + cik, + onBack, +}: { + cik: string; + onBack: () => void; +}) { + const profile = useQuery({ + queryKey: ["edgar", "company", cik], + queryFn: () => readEdgar(`companies/${cik}`, {}, edgarCompany), + staleTime: 60 * 60_000, + }); + + return ( +
+ + + {(company) => ( + <> + + + + Filings + Holders + Proxy & pay + Insiders + + + + + + + + + + + + + + + + )} + +
+ ); +} + +function CompanyHeader({ company }: { company: EdgarCompany }) { + const router = useRouter(); + const workspaceUrl = useWorkspaceUrl(); + const trpc = useTRPC(); + const handOff = useMutation( + trpc.conversations.createBuilder.mutationOptions({ + onSuccess: ({ id }) => router.push(workspaceUrl(`/chat/${id}`)), + onError: (error) => toast.error(error.message), + }), + ); + + const address = company.businessAddress; + const place = [address?.city, address?.state].filter(Boolean).join(", "); + + return ( +
+
+
+

{company.name}

+ {company.tickers.map((ticker) => ( + + {ticker} + + ))} +
+

+ {[ + company.sicDescription + ? `${company.sicDescription} (SIC ${company.sic})` + : null, + place || null, + company.stateOfIncorporation + ? `Incorporated in ${company.stateOfIncorporation}` + : null, + company.fiscalYearEnd + ? `Fiscal year ends ${company.fiscalYearEnd.slice(0, 2)}/${company.fiscalYearEnd.slice(2)}` + : null, + company.category, + ] + .filter(Boolean) + .join(" · ")} +

+

+ + EDGAR record for CIK {company.cik} + +

+
+ +
+ ); +} + +function Filings({ cik }: { cik: string }) { + const [form, setForm] = useState<(typeof FORMS)[number]>("all"); + const filings = useQuery({ + queryKey: ["edgar", "filings", cik, form], + queryFn: () => + readEdgar( + `companies/${cik}/filings`, + { form: form === "all" ? undefined : form, limit: FILINGS_LIMIT }, + edgarFilings, + ), + staleTime: 10 * 60_000, + }); + + return ( +
+ + + {(data) => + data.filings.length === 0 ? ( +

+ No filing of that form. +

+ ) : ( + + {data.filings.map((filing) => ( + + {filing.form} + + {filing.filedAt} + + + {filing.reportDate ?? "—"} + + + + {filing.description ?? filing.accession} + + + + ))} + + ) + } + +
+ ); +} + +function Holders({ cik }: { cik: string }) { + const owners = useQuery({ + queryKey: ["edgar", "owners", cik], + queryFn: () => + readEdgar( + `companies/${cik}/owners`, + { limit: OWNERS_LIMIT }, + edgarOwners, + ), + staleTime: 10 * 60_000, + }); + + return ( + + {(data) => + data.owners.length === 0 ? ( +

+ No Schedule 13D or 13G at 5% or more in the filings read. Holders + below 5% never file one. +

+ ) : ( + + {data.owners.map((owner) => ( + + + + {owner.filer} + + {owner.purpose ? ( + + {owner.purpose} + + ) : null} + + {owner.form} + {owner.filedAt} + + {count(owner.shares)} + + + {percent(owner.percent)} + + + ))} + + ) + } + + ); +} + +function Insiders({ cik }: { cik: string }) { + const insiders = useQuery({ + queryKey: ["edgar", "insiders", cik], + queryFn: () => + readEdgar( + `companies/${cik}/insiders`, + { limit: INSIDERS_LIMIT }, + edgarInsiders, + ), + staleTime: 10 * 60_000, + }); + + return ( + + {(data) => + data.transactions.length === 0 ? ( +

+ No insider filing in the period read. +

+ ) : ( + + {data.transactions.map((row) => ( + + + + {row.insider} + + + {row.title ?? "—"} + {row.form} + {row.filedAt} + {row.kind?.replaceAll("_", " ") ?? "—"} + + {count(row.shares)} + + + ))} + + ) + } + + ); +} + +function ProxyTab({ cik }: { cik: string }) { + const proxy = useQuery({ + queryKey: ["edgar", "proxy", cik], + queryFn: () => + readEdgar(`companies/${cik}/proxy`, { years: PROXY_YEARS }, edgarProxy), + staleTime: 60 * 60_000, + }); + + return {(data) => }; +} + +function ProxyView({ proxy }: { proxy: EdgarProxy }) { + return ( +
+

+ Latest proxy statement, filed {proxy.filedAt}:{" "} + + DEF 14A {proxy.accession} + +

+ +
+ + + + +
+ +
+

Named executives

+ {proxy.executives.length === 0 ? ( +

+ This proxy carries no machine-readable compensation table. +

+ ) : ( + + {proxy.executives.map((executive) => ( + + {executive.name} + {executive.title ?? "—"} + + {executive.year ?? "—"} + + + {money(executive.salary)} + + + {money(executive.stockAwards)} + + + {money(executive.total)} + + + ))} + + )} +
+ +
+

Pay versus performance

+ {proxy.payVsPerformance.length === 0 ? ( +

+ No pay-versus-performance table in this proxy. +

+ ) : ( + + {proxy.payVsPerformance.map((row) => ( + + + {row.fiscalYearEnd ?? "—"} + + + {money(row.peoActuallyPaidComp)} + + + {money(row.neoAverageActuallyPaidComp)} + + + {row.tsr ?? "—"} + + + {row.peerTsr ?? "—"} + + + {money(row.netIncome)} + + + {money(row.selectedMeasureValue)} + + + ))} + + )} +
+ +
+
+

Holders the proxy lists

+ {proxy.holders.length === 0 ? ( +

None listed.

+ ) : ( + + {proxy.holders.map((holder) => ( + + {holder.name} + + {count(holder.shares)} + + + {percent(holder.percentOfClass)} + + + ))} + + )} +
+ +
+

Proposals and governance

+
    + {proxy.proposals.map((proposal) => ( +
  • + {proposal.number ? `${proposal.number}. ` : ""} + {proposal.description} +
    • + ))} +
  • + Insider trading policy adopted:{" "} + {proxy.insiderTradingPolicyAdopted === null + ? "not stated" + : proxy.insiderTradingPolicyAdopted + ? "yes" + : "no"} +
    • + {proxy.performanceMeasures.length > 0 ? ( +
  • + Performance measures: {proxy.performanceMeasures.join(", ")} +
    • + ) : null} +
+
+
+
+ ); +} diff --git a/apps/app/app/(app)/[slug]/research/sec/sec-research.tsx b/apps/app/app/(app)/[slug]/research/sec/sec-research.tsx new file mode 100644 index 000000000..9819d3d72 --- /dev/null +++ b/apps/app/app/(app)/[slug]/research/sec/sec-research.tsx @@ -0,0 +1,159 @@ +"use client"; + +import { Button } from "@crm/ui/components/button"; +import { + Empty, + EmptyDescription, + EmptyHeader, + EmptyTitle, +} from "@crm/ui/components/empty"; +import { Input } from "@crm/ui/components/input"; +import { SimpleTable, SimpleTableRow } from "@crm/ui/components/simple-table"; +import { Spinner } from "@crm/ui/components/spinner"; +import { TableCell } from "@crm/ui/components/table"; +import { edgarCompanySearch } from "@crm/validation/edgar"; +import { useQuery } from "@tanstack/react-query"; +import { useQueryState } from "nuqs"; +import { type FormEvent, useState } from "react"; +import { readEdgar } from "./edgar-client"; +import { SecCompany } from "./sec-company"; + +const SEARCH_LIMIT = 10; + +export function SecResearch({ configured }: { configured: boolean }) { + const [q, setQ] = useQueryState("q", { defaultValue: "" }); + const [cik, setCik] = useQueryState("cik", { defaultValue: "" }); + const [draft, setDraft] = useState(q); + + const search = useQuery({ + queryKey: ["edgar", "search", q], + queryFn: () => + readEdgar( + "companies/search", + { q, limit: SEARCH_LIMIT }, + edgarCompanySearch, + ), + enabled: configured && q.trim().length > 0, + staleTime: 5 * 60_000, + }); + + if (!configured) { + return ( + + + The SEC EDGAR service is not configured + + Set EDGAR_URL and EDGAR_SECRET to a running services/edgar and + restart the app. docs/setup.md and services/edgar/README.md cover + Docker Compose, another machine and Google Colab. + + + + ); + } + + const submit = (event: FormEvent) => { + event.preventDefault(); + void setCik(""); + void setQ(draft.trim()); + }; + + return ( +
+
+ setDraft(event.target.value)} + placeholder="Company name, ticker or CIK — Apple, AAPL, 320193" + aria-label="Search SEC EDGAR" + /> + +
+ + {cik ? ( + void setCik("")} /> + ) : ( + void setCik(picked)} + /> + )} +
+ ); +} + +function SearchResults({ + query, + state, + onPick, +}: { + query: string; + state: ReturnType< + typeof useQuery< + Awaited>> + > + >; + onPick: (cik: string) => void; +}) { + if (!query.trim()) { + return ( +

+ Search a US public company to read its filings, its 5%+ shareholders and + its proxy statement, then hand it to the agent to import. +

+ ); + } + if (state.isPending) { + return ( +

+ Searching EDGAR… +

+ ); + } + if (state.isError || !state.data) { + return ( +

The search failed. Try again.

+ ); + } + if (state.data.status !== "ok") { + return

{state.data.reason}

; + } + if (state.data.data.companies.length === 0) { + return ( +

+ Nothing in EDGAR matches “{query}”. Only companies that file with the + SEC are listed; try the ticker. +

+ ); + } + + return ( + + {state.data.data.companies.map((company) => ( + onPick(company.cik)} + > + {company.name} + {company.ticker ?? "—"} + {company.exchange ?? "—"} + + {company.cik} + + + ))} + + ); +} diff --git a/apps/app/app/edgar/[...path]/route.ts b/apps/app/app/edgar/[...path]/route.ts new file mode 100644 index 000000000..82e074d31 --- /dev/null +++ b/apps/app/app/edgar/[...path]/route.ts @@ -0,0 +1,54 @@ +import { connection } from "next/server"; +import { edgarConfigured, edgarHeaders, edgarTarget } from "@/lib/edgar"; +import { getSession } from "@/lib/session"; + +const TIMEOUT_MS = 60_000; + +export async function GET( + request: Request, + { params }: { params: Promise<{ path: string[] }> }, +): Promise { + await connection(); + + if (!edgarConfigured()) { + return Response.json( + { reason: "The SEC EDGAR service is not configured for this install." }, + { status: 503 }, + ); + } + + const session = await getSession(); + if (!session) { + return Response.json({ reason: "Not signed in." }, { status: 401 }); + } + + const { path } = await params; + const url = new URL(request.url); + const target = edgarTarget(path.join("/"), url.search); + if (!target) { + return Response.json({ reason: "No such route." }, { status: 404 }); + } + + try { + const upstream = await fetch(target, { + headers: edgarHeaders(), + signal: AbortSignal.timeout(TIMEOUT_MS), + cache: "no-store", + }); + const body = await upstream.text(); + return new Response(body, { + status: upstream.status, + headers: { "content-type": "application/json" }, + }); + } catch (error) { + return Response.json( + { + reason: + error instanceof Error && error.name === "TimeoutError" + ? "The SEC EDGAR service did not answer in time." + : "The SEC EDGAR service is unreachable.", + }, + { status: 502 }, + ); + } +} diff --git a/apps/app/components/app-icon-rail.tsx b/apps/app/components/app-icon-rail.tsx index ad3440e00..57b2f9c23 100644 --- a/apps/app/components/app-icon-rail.tsx +++ b/apps/app/components/app-icon-rail.tsx @@ -4,6 +4,7 @@ import Building from "@carbon/icons-react/es/Building"; import Close from "@carbon/icons-react/es/Close"; import Dashboard from "@carbon/icons-react/es/Dashboard"; import Partnership from "@carbon/icons-react/es/Partnership"; +import Search from "@carbon/icons-react/es/Search"; import Settings from "@carbon/icons-react/es/Settings"; import UserMultiple from "@carbon/icons-react/es/UserMultiple"; import { Button } from "@crm/ui/components/button"; @@ -57,6 +58,7 @@ const ITEMS: RailItem[] = [ match: "prefix", }, { title: "Deals", href: "/deals", icon: Partnership, match: "prefix" }, + { title: "Research", href: "/research/sec", icon: Search, match: "prefix" }, { title: "Settings", href: "/settings", icon: Settings, match: "prefix" }, ]; diff --git a/apps/app/lib/edgar.ts b/apps/app/lib/edgar.ts new file mode 100644 index 000000000..ecbd51ebc --- /dev/null +++ b/apps/app/lib/edgar.ts @@ -0,0 +1,34 @@ +const ALLOWED_PATHS = [ + /^health$/, + /^companies\/search$/, + /^companies\/[A-Za-z0-9.-]{1,12}$/, + /^companies\/[A-Za-z0-9.-]{1,12}\/(filings|owners|insiders|proxy)$/, + /^filings\/search$/, + /^compensation\/compare$/, +]; + +export function edgarUrl(): string | null { + const url = process.env.EDGAR_URL?.trim(); + return url ? url.replace(/\/+$/, "") : null; +} + +export function edgarConfigured(): boolean { + return edgarUrl() !== null; +} + +export function edgarPathAllowed(path: string): boolean { + return ALLOWED_PATHS.some((pattern) => pattern.test(path)); +} + +export function edgarTarget(path: string, search: string): URL | null { + const base = edgarUrl(); + if (!base || !edgarPathAllowed(path)) return null; + return new URL(`${base}/${path}${search}`); +} + +export function edgarHeaders(): HeadersInit { + const secret = process.env.EDGAR_SECRET?.trim(); + return secret + ? { accept: "application/json", authorization: `Bearer ${secret}` } + : { accept: "application/json" }; +} diff --git a/apps/app/proxy.ts b/apps/app/proxy.ts index 6040b7068..e069a3f6d 100644 --- a/apps/app/proxy.ts +++ b/apps/app/proxy.ts @@ -14,7 +14,7 @@ const LANDING_PATH = "/"; const SIGN_IN_PATH = "/sign-in"; -const UNGATED = ["/grant-access", "/eve"]; +const UNGATED = ["/grant-access", "/eve", "/edgar"]; const ANONYMOUS = ["/t"]; diff --git a/apps/app/test/edgar.spec.ts b/apps/app/test/edgar.spec.ts new file mode 100644 index 000000000..6ad68f76f --- /dev/null +++ b/apps/app/test/edgar.spec.ts @@ -0,0 +1,68 @@ +import { afterEach, describe, expect, it } from "bun:test"; +import { + edgarConfigured, + edgarHeaders, + edgarPathAllowed, + edgarTarget, +} from "../lib/edgar"; + +const savedUrl = process.env.EDGAR_URL; +const savedSecret = process.env.EDGAR_SECRET; + +afterEach(() => { + if (savedUrl === undefined) delete process.env.EDGAR_URL; + else process.env.EDGAR_URL = savedUrl; + if (savedSecret === undefined) delete process.env.EDGAR_SECRET; + else process.env.EDGAR_SECRET = savedSecret; +}); + +describe("the app's edgar helpers", () => { + it("is off without a URL", () => { + delete process.env.EDGAR_URL; + expect(edgarConfigured()).toBe(false); + expect(edgarTarget("health", "")).toBeNull(); + }); + + it("proxies only the routes the page reads", () => { + for (const path of [ + "health", + "companies/search", + "companies/320193", + "companies/AAPL", + "companies/320193/filings", + "companies/320193/owners", + "companies/320193/insiders", + "companies/320193/proxy", + "filings/search", + "compensation/compare", + ]) { + expect(edgarPathAllowed(path)).toBe(true); + } + for (const path of [ + "", + "docs", + "companies/320193/secrets", + "companies/../health", + "companies/320193/filings/x", + ]) { + expect(edgarPathAllowed(path)).toBe(false); + } + }); + + it("builds the target from the URL, the path and the query", () => { + process.env.EDGAR_URL = "http://127.0.0.1:2100/"; + expect( + edgarTarget("companies/search", "?q=apple&limit=3")?.toString(), + ).toBe("http://127.0.0.1:2100/companies/search?q=apple&limit=3"); + }); + + it("sends the secret only when it is set", () => { + process.env.EDGAR_SECRET = "s3cret"; + expect(edgarHeaders()).toEqual({ + accept: "application/json", + authorization: "Bearer s3cret", + }); + delete process.env.EDGAR_SECRET; + expect(edgarHeaders()).toEqual({ accept: "application/json" }); + }); +}); diff --git a/apps/app/test/onboarding-gate.spec.ts b/apps/app/test/onboarding-gate.spec.ts index 0480f1429..bf78246fa 100644 --- a/apps/app/test/onboarding-gate.spec.ts +++ b/apps/app/test/onboarding-gate.spec.ts @@ -292,6 +292,16 @@ describe("proxy", () => { ).toBeNull(); }); + it("leaves the SEC EDGAR proxy alone", async () => { + setup({ onboarded: false }); + + expect( + redirectedTo( + await proxy(request("/edgar/companies/search", [SESSION_COOKIE])), + ), + ).toBeNull(); + }); + it("fails open when the API is unreachable", async () => { stub(async () => { throw new Error("connect ECONNREFUSED"); diff --git a/apps/app/turbo.json b/apps/app/turbo.json index 3051a65cd..445625907 100644 --- a/apps/app/turbo.json +++ b/apps/app/turbo.json @@ -15,6 +15,8 @@ "BETTER_AUTH_SECRET", "BETTER_AUTH_URL", "DATABASE_URL", + "EDGAR_SECRET", + "EDGAR_URL", "IS_MARKETING" ] }, @@ -38,6 +40,8 @@ "BETTER_AUTH_SECRET", "BETTER_AUTH_URL", "DATABASE_URL", + "EDGAR_SECRET", + "EDGAR_URL", "IS_MARKETING", "NEXT_PUBLIC_API_URL", "NEXT_PUBLIC_AUTH_URL" diff --git a/docs/environment.md b/docs/environment.md index 948b76070..912d1c77d 100644 --- a/docs/environment.md +++ b/docs/environment.md @@ -139,7 +139,8 @@ up, every response is JSON that the CRM parses with Zod (`packages/validation/src/edgar.ts`) before use, and a missing URL turns the capability off without an error. Only the routes are SEC-specific. The same shape works for a service on another machine or in a Google Colab notebook behind a -tunnel; `services/edgar/README.md` shows both. `EDGAR_IDENTITY` is read by the +tunnel; `services/edgar/README.md` shows both. The Next.js app reads the same two +variables for its Research → SEC page, so they are in `apps/app/turbo.json` too. `EDGAR_IDENTITY` is read by the service alone: the SEC requires every automated client to name a contact email. `BLOB_READ_WRITE_TOKEN` is also in `env.validation.ts` and `apps/api/turbo.json` From 936b19928e5c0f51445bc694d26d21e2d9733095 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 3 Sep 2026 16:32:48 +0000 Subject: [PATCH 17/19] feat(edgar): tunnel.sh runs the service behind a Cloudflare tunnel on any machine One command builds the image or the venv, generates the secret, starts the service, opens a quick tunnel and prints EDGAR_URL and EDGAR_SECRET to set on the CRM. It mirrors colab.ipynb for a PC. --- docs/setup.md | 3 +- services/edgar/README.md | 11 +++++ services/edgar/tunnel.sh | 94 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 107 insertions(+), 1 deletion(-) create mode 100755 services/edgar/tunnel.sh diff --git a/docs/setup.md b/docs/setup.md index d7cb58de4..d34dc1e1f 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -18,7 +18,8 @@ agent's `sec_*` tools, on port 2100. It needs `EDGAR_IDENTITY` in `.env` (the SE asks every automated client for a contact email) and the CRM needs `EDGAR_URL=http://127.0.0.1:2100`. Leave both unset and the agent simply reports the source as unavailable. `services/edgar/README.md` covers running it on another -machine or in Google Colab. +machine (`services/edgar/tunnel.sh` starts it behind a Cloudflare tunnel and prints +the two variables to set) or in Google Colab. Prisma from the repo root: `db:generate`, `db:migrate`, `db:push`, `db:reset`, `db:seed`, `db:studio`, `db:deploy`. diff --git a/services/edgar/README.md b/services/edgar/README.md index c0c320f2f..488e6ea89 100644 --- a/services/edgar/README.md +++ b/services/edgar/README.md @@ -53,6 +53,17 @@ edgartools itself. ## Run it on another machine +One command does everything below — build or venv, secret, service, tunnel — +and prints the two variables to paste into the CRM: + +```sh +EDGAR_IDENTITY="Jane Doe jane@example.com" services/edgar/tunnel.sh +``` + +It needs Docker (or Python 3.11+) and +[cloudflared](https://developers.cloudflare.com/cloudflare-one/connections/connect-networks/downloads/). +`EDGAR_RUNTIME=python` forces the venv path when Docker is present. By hand: + ```sh cd services/edgar python -m venv .venv && . .venv/bin/activate diff --git a/services/edgar/tunnel.sh b/services/edgar/tunnel.sh new file mode 100755 index 000000000..219453afb --- /dev/null +++ b/services/edgar/tunnel.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env sh +set -eu + +HERE=$(cd "$(dirname "$0")" && pwd) +PORT=${EDGAR_PORT:-2100} +IDENTITY=${EDGAR_IDENTITY:-} +SECRET=${EDGAR_SECRET:-} +DATA_DIR=${EDGAR_DATA_DIR:-"$HERE/.cache"} +RUNTIME=${EDGAR_RUNTIME:-auto} + +if [ -z "$IDENTITY" ]; then + echo "EDGAR_IDENTITY is required: the SEC asks every automated client for a name and a real email." >&2 + echo " EDGAR_IDENTITY=\"Jane Doe jane@example.com\" $0" >&2 + exit 1 +fi + +if [ -z "$SECRET" ]; then + SECRET=$(openssl rand -hex 24 2>/dev/null || head -c 24 /dev/urandom | od -An -tx1 | tr -d ' \n') +fi + +mkdir -p "$DATA_DIR" + +if [ "$RUNTIME" = auto ]; then + if command -v docker >/dev/null 2>&1; then RUNTIME=docker; else RUNTIME=python; fi +fi + +cleanup() { + [ -n "${SERVER_PID:-}" ] && kill "$SERVER_PID" 2>/dev/null || true + [ "$RUNTIME" = docker ] && docker rm -f crm-edgar-tunnel >/dev/null 2>&1 || true + [ -n "${TUNNEL_PID:-}" ] && kill "$TUNNEL_PID" 2>/dev/null || true +} +trap cleanup EXIT INT TERM + +case "$RUNTIME" in + docker) + docker build -q -t crm-edgar "$HERE" >/dev/null + docker rm -f crm-edgar-tunnel >/dev/null 2>&1 || true + docker run -d --name crm-edgar-tunnel -p "127.0.0.1:$PORT:2100" \ + -e EDGAR_IDENTITY="$IDENTITY" -e EDGAR_SECRET="$SECRET" \ + -v "$DATA_DIR:/data" crm-edgar >/dev/null + ;; + python) + if [ ! -x "$HERE/.venv/bin/python" ]; then + python3 -m venv "$HERE/.venv" + "$HERE/.venv/bin/pip" install -q -r "$HERE/requirements.txt" + fi + PYTHONPATH="$HERE" EDGAR_IDENTITY="$IDENTITY" EDGAR_SECRET="$SECRET" \ + EDGAR_DATA_DIR="$DATA_DIR" EDGAR_LOCAL_DATA_DIR="$DATA_DIR" \ + "$HERE/.venv/bin/python" -m uvicorn edgar_service.app:app --host 127.0.0.1 --port "$PORT" >"$DATA_DIR/uvicorn.log" 2>&1 & + SERVER_PID=$! + ;; + *) + echo "EDGAR_RUNTIME must be docker or python." >&2 + exit 1 + ;; +esac + +i=0 +until curl -fsS "http://127.0.0.1:$PORT/health" >/dev/null 2>&1; do + i=$((i + 1)) + if [ "$i" -gt 60 ]; then + echo "The service did not answer on port $PORT." >&2 + exit 1 + fi + sleep 1 +done + +if ! command -v cloudflared >/dev/null 2>&1; then + echo "cloudflared is required: https://developers.cloudflare.com/cloudflare-one/connections/connect-networks/downloads/" >&2 + exit 1 +fi + +cloudflared tunnel --url "http://127.0.0.1:$PORT" --no-autoupdate >"$DATA_DIR/tunnel.log" 2>&1 & +TUNNEL_PID=$! + +URL="" +i=0 +while [ -z "$URL" ]; do + i=$((i + 1)) + if [ "$i" -gt 60 ]; then + echo "The tunnel did not come up. See $DATA_DIR/tunnel.log." >&2 + exit 1 + fi + sleep 1 + URL=$(grep -oE 'https://[a-z0-9-]+\.trycloudflare\.com' "$DATA_DIR/tunnel.log" | head -1 || true) +done + +echo +echo "Set these on the CRM (Vercel → crm-agent and crm-app, or the local .env), then redeploy:" +echo "EDGAR_URL=$URL" +echo "EDGAR_SECRET=$SECRET" +echo +echo "The service and the tunnel live as long as this shell does. Ctrl-C stops both." +wait "$TUNNEL_PID" From 560151d0b2174e1f3f70b99f933b69de4799ea3f Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 3 Sep 2026 17:09:00 +0000 Subject: [PATCH 18/19] fix(agent): lock the claimed and retired rows once, in a materialized CTE A FROM subquery with LIMIT and FOR UPDATE SKIP LOCKED is re-scanned by the planner, so retireExhausted(2) still retired three rows in CI after the last rewrite. Both claimDue and retireExhausted now select and lock their rows in a MATERIALIZED CTE, which Postgres evaluates exactly once, and update from it. --- apps/agent/agent/lib/tasks.ts | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/apps/agent/agent/lib/tasks.ts b/apps/agent/agent/lib/tasks.ts index 94ce8bfcc..5dd3ee556 100644 --- a/apps/agent/agent/lib/tasks.ts +++ b/apps/agent/agent/lib/tasks.ts @@ -42,11 +42,7 @@ export async function claimDue( const onlyMode = "only" in kinds; const claimed = await db.$queryRaw` - UPDATE "agentTask" AS t - SET "leasedUntil" = ${until}, - "startedAt" = COALESCE(t."startedAt", ${now}), - "attempts" = t."attempts" + 1 - FROM ( + WITH due AS MATERIALIZED ( SELECT t2.id FROM "agentTask" AS t2 WHERE t2."finishedAt" IS NULL AND t2."dueAt" <= ${now} @@ -59,7 +55,12 @@ export async function claimDue( ORDER BY t2."priority" DESC, t2."dueAt" ASC LIMIT ${limit} FOR UPDATE SKIP LOCKED - ) AS due + ) + UPDATE "agentTask" AS t + SET "leasedUntil" = ${until}, + "startedAt" = COALESCE(t."startedAt", ${now}), + "attempts" = t."attempts" + 1 + FROM due WHERE t.id = due.id RETURNING t.id, t."contactId", t."companyId", t."dealId", t.kind, t.reason, t.payload, t.budget, t.attempts, t.priority, t."dueAt"; @@ -76,10 +77,7 @@ export async function retireExhausted( const now = new Date(); return db.$queryRaw` - UPDATE "agentTask" AS t - SET "finishedAt" = ${now}, - "outcome" = ${RETIRED_OUTCOME} - FROM ( + WITH exhausted AS MATERIALIZED ( SELECT c.id FROM "agentTask" AS c WHERE c."finishedAt" IS NULL @@ -88,7 +86,11 @@ export async function retireExhausted( ORDER BY c."dueAt" ASC LIMIT ${limit} FOR UPDATE SKIP LOCKED - ) AS exhausted + ) + UPDATE "agentTask" AS t + SET "finishedAt" = ${now}, + "outcome" = ${RETIRED_OUTCOME} + FROM exhausted WHERE t.id = exhausted.id RETURNING t.id, t."contactId", t."companyId", t."dealId", t.kind; `; From 42cff5c8282ba52d1997b0b67803836e41e4ed0a Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 3 Sep 2026 17:17:17 +0000 Subject: [PATCH 19/19] test(agent): name the rows when retireExhausted exceeds its limit The limit assertion has failed in CI with three rows and never locally, before and after the query rewrite. The failure now lists the rows the statement returned, with their kind, attempts and lease, and the ids the test created, so the next occurrence says where the third row came from. --- apps/agent/test/tasks.integration.spec.ts | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/apps/agent/test/tasks.integration.spec.ts b/apps/agent/test/tasks.integration.spec.ts index 3100f0c1f..cfdb065f4 100644 --- a/apps/agent/test/tasks.integration.spec.ts +++ b/apps/agent/test/tasks.integration.spec.ts @@ -190,7 +190,24 @@ describe("retireExhausted", () => { } for (let pass = 0; pass < 3; pass++) { - expect((await retireExhausted(2)).length).toBeLessThanOrEqual(2); + const retired = await retireExhausted(2); + if (retired.length > 2) { + const rows = await db.agentTask.findMany({ + where: { id: { in: retired.map((task) => task.id) } }, + select: { + id: true, + kind: true, + attempts: true, + leasedUntil: true, + finishedAt: true, + outcome: true, + }, + }); + throw new Error( + `retireExhausted(2) returned ${retired.length} rows on pass ${pass}: ${JSON.stringify({ retired, rows, mine })}`, + ); + } + expect(retired.length).toBeLessThanOrEqual(2); } const open = await db.agentTask.count({