diff --git a/.claude/hooks/crm-dev.sh b/.claude/hooks/crm-dev.sh new file mode 100755 index 000000000..cfc531451 --- /dev/null +++ b/.claude/hooks/crm-dev.sh @@ -0,0 +1,43 @@ +#!/bin/bash +set -euo pipefail + +if [ "${CLAUDE_CODE_REMOTE:-}" != "true" ]; then + exit 0 +fi + +cd "$CLAUDE_PROJECT_DIR" + +NODE_DIR=/opt/node24 +if [ ! -x "$NODE_DIR/bin/node" ]; then + VERSION=$(curl -sSL https://nodejs.org/dist/index.json | python3 -c 'import sys,json; print(next(v["version"] for v in json.load(sys.stdin) if v["version"].startswith("v24.")))') + mkdir -p "$NODE_DIR" + curl -sSL "https://nodejs.org/dist/$VERSION/node-$VERSION-linux-x64.tar.xz" | tar -xJ -C "$NODE_DIR" --strip-components=1 +fi +export PATH="$NODE_DIR/bin:$PATH" +echo "export PATH=\"$NODE_DIR/bin:\$PATH\"" >> "$CLAUDE_ENV_FILE" + +if [ ! -f .env ]; then + cp .env.example .env + sed -i "s|^BETTER_AUTH_SECRET=\"\"|BETTER_AUTH_SECRET=\"$(openssl rand -base64 32)\"|" .env + sed -i "s|^# AGENT_URL=\"http://127.0.0.1:2000\"|AGENT_URL=\"http://127.0.0.1:2000\"|" .env + sed -i "s|^# AGENT_BRIDGE_SECRET=\"\"|AGENT_BRIDGE_SECRET=\"$(openssl rand -base64 32)\"|" .env + sed -i "s|^ALLOWED_SIGN_IN=\"\"|ALLOWED_SIGN_IN=\"localhost\"|" .env +fi + +if ! docker info >/dev/null 2>&1; then + nohup dockerd >/tmp/dockerd.log 2>&1 & + for _ in $(seq 1 30); do + docker info >/dev/null 2>&1 && break + sleep 1 + done +fi + +docker compose up -d +for _ in $(seq 1 60); do + [ "$(docker inspect -f '{{.State.Health.Status}}' crm-postgres 2>/dev/null)" = "healthy" ] && break + sleep 1 +done + +bun install +bun run --filter=@crm/db db:deploy +bun run --filter=@crm/db db:seed diff --git a/.claude/hooks/session-start.sh b/.claude/hooks/session-start.sh new file mode 100755 index 000000000..7339d6d6b --- /dev/null +++ b/.claude/hooks/session-start.sh @@ -0,0 +1,98 @@ +#!/bin/bash +set -uo pipefail + +if [ "${CLAUDE_CODE_REMOTE:-}" != "true" ]; then + exit 0 +fi + +AGENT_REACH_HOME="$HOME/.agent-reach" +AGENT_REACH_TOOLS="$AGENT_REACH_HOME/tools" +AGENT_REACH_VENV="$HOME/.agent-reach-venv" +AGENT_REACH_REPO="https://github.com/Panniantong/agent-reach.git" +XHS_REPO="https://github.com/xpzouying/xiaohongshu-mcp.git" +XHS_PORT=18060 +YTDLP_CONFIG="$HOME/.config/yt-dlp/config" +MCPORTER_CONFIG="$HOME/.mcporter/mcporter.json" +LOCAL_BIN="$HOME/.local/bin" + +export PATH="$AGENT_REACH_VENV/bin:$LOCAL_BIN:$PATH" + +log() { echo "[agent-reach hook] $*"; } + +step() { + local name="$1" + shift + if "$@"; then + log "ok: $name" + else + log "failed: $name (continuing)" + fi +} + +install_agent_reach() { + mkdir -p "$AGENT_REACH_TOOLS" + if [ ! -d "$AGENT_REACH_TOOLS/agent-reach/.git" ]; then + git clone -q --depth 1 "$AGENT_REACH_REPO" "$AGENT_REACH_TOOLS/agent-reach" + else + git -C "$AGENT_REACH_TOOLS/agent-reach" pull -q --ff-only || true + fi + [ -x "$AGENT_REACH_VENV/bin/pip" ] || python3 -m venv "$AGENT_REACH_VENV" + "$AGENT_REACH_VENV/bin/pip" install -q "$AGENT_REACH_TOOLS/agent-reach" +} + +install_apt_tools() { + local missing=() + command -v gh >/dev/null || missing+=(gh) + command -v ffmpeg >/dev/null || missing+=(ffmpeg) + [ "${#missing[@]}" -eq 0 ] && return 0 + apt-get update -q >/dev/null 2>&1 + DEBIAN_FRONTEND=noninteractive apt-get install -y -q "${missing[@]}" >/dev/null 2>&1 +} + +install_mcporter() { + command -v mcporter >/dev/null || npm install -g mcporter >/dev/null 2>&1 +} + +mcporter_has() { + [ -f "$MCPORTER_CONFIG" ] && grep -q "\"$1\"" "$MCPORTER_CONFIG" +} + +configure_mcporter() { + mcporter_has exa || mcporter config add exa https://mcp.exa.ai/mcp --scope home >/dev/null + mcporter_has linkedin || mcporter config add linkedin --command uvx --arg mcp-server-linkedin@latest --env UV_HTTP_TIMEOUT=300 --scope home >/dev/null + mcporter_has xiaohongshu || mcporter config add xiaohongshu "http://localhost:$XHS_PORT/mcp" --scope home >/dev/null +} + +configure_ytdlp() { + mkdir -p "$(dirname "$YTDLP_CONFIG")" + grep -qxF -- '--js-runtimes node' "$YTDLP_CONFIG" 2>/dev/null || printf '%s\n' '--js-runtimes node' >> "$YTDLP_CONFIG" +} + +install_channels() { + agent-reach install --env=auto --system --channels=all >/dev/null 2>&1 || true + return 0 +} + +build_xiaohongshu() { + [ -x "$AGENT_REACH_TOOLS/xiaohongshu-mcp" ] && return 0 + command -v go >/dev/null || return 1 + [ -d "$AGENT_REACH_TOOLS/xiaohongshu-mcp-src/.git" ] || git clone -q --depth 1 "$XHS_REPO" "$AGENT_REACH_TOOLS/xiaohongshu-mcp-src" + (cd "$AGENT_REACH_TOOLS/xiaohongshu-mcp-src" && CGO_ENABLED=0 go build -o "$AGENT_REACH_TOOLS/xiaohongshu-mcp" . && CGO_ENABLED=0 go build -o "$AGENT_REACH_TOOLS/xiaohongshu-login" ./cmd/login) >/dev/null 2>&1 +} + +export_path() { + [ -n "${CLAUDE_ENV_FILE:-}" ] || return 0 + echo "export PATH=\"$AGENT_REACH_VENV/bin:$LOCAL_BIN:\$PATH\"" >> "$CLAUDE_ENV_FILE" +} + +step "agent-reach package" install_agent_reach +step "gh + ffmpeg" install_apt_tools +step "mcporter" install_mcporter +step "mcporter servers" configure_mcporter +step "yt-dlp config" configure_ytdlp +step "optional channels" install_channels +step "xiaohongshu-mcp build" build_xiaohongshu +step "PATH" export_path + +agent-reach doctor 2>/dev/null | grep -E '^状态' || true +exit 0 diff --git a/.claude/settings.json b/.claude/settings.json index 97251ad97..7f426f2fa 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -4,5 +4,21 @@ }, "enabledPlugins": { "paper-desktop@paper": true + }, + "hooks": { + "SessionStart": [ + { + "hooks": [ + { + "type": "command", + "command": "$CLAUDE_PROJECT_DIR/.claude/hooks/session-start.sh" + }, + { + "type": "command", + "command": "$CLAUDE_PROJECT_DIR/.claude/hooks/crm-dev.sh" + } + ] + } + ] } } diff --git a/.env.example b/.env.example index 12fac543c..a2a870821 100644 --- a/.env.example +++ b/.env.example @@ -57,6 +57,12 @@ GOOGLE_CLIENT_SECRET="" # SLACK_CLIENT_ID="" # SLACK_CLIENT_SECRET="" +# Optional. Adds an email + password form to the sign-in page, next to the +# social buttons. Only addresses that pass ALLOWED_SIGN_IN can create an +# account, exactly like the social sign-ins, and the password must be at +# least 8 characters. The only value that turns it on is "true". +# PASSWORD_SIGN_IN="true" + # Which Entra tenant may sign in. "common" (the default) accepts any work, # school or personal Microsoft account and leans on ALLOWED_SIGN_IN to decide # who actually gets in; your own tenant's GUID refuses everyone else at @@ -125,6 +131,50 @@ GOOGLE_CLIENT_SECRET="" # knowing before a call. https://perplexity.ai/settings/api # PERPLEXITY_API_KEY="" +# Contact-data providers. Each one is optional and the agent asks the ones +# that are set, in this order, stopping at the first confident answer: +# Hunter, Apollo, Lusha, Dropcontact, ZoomInfo. Every answer is written to +# the contact's timeline with its confidence and its source. +# +# SEC EDGAR research — US public companies from their SEC filings: profile, +# filings, 5%+ shareholders, insider transactions, the proxy statement with +# its executives and their pay. Served by services/edgar (Python, edgartools) +# and free. `docker compose up -d` runs it on port 2100; a Colab notebook or +# another machine can run it too, see services/edgar/README.md. Without +# EDGAR_URL the sec_* tools say the source is unavailable. +# EDGAR_URL="http://127.0.0.1:2100" +# +# Shared secret between the CRM and the service. Optional on loopback; set it +# whenever the service is reachable from elsewhere. +# EDGAR_SECRET="" +# +# The SEC requires every automated client to identify itself. Any real email. +# Read by the service, not by the CRM. +# EDGAR_IDENTITY="Jane Doe jane@example.com" +# +# Hunter — a work email with the public pages it was seen on, and a +# deliverability check. The free plan covers a few dozen lookups a month. +# https://hunter.io/api-keys +# HUNTER_API_KEY="" +# +# Apollo — a work email with its verification status, title and work phone. +# https://app.apollo.io/#/settings/integrations/api +# APOLLO_API_KEY="" +# +# Lusha — work email, direct and mobile phones, title. +# https://dashboard.lusha.com/enrich/api +# LUSHA_API_KEY="" +# +# Dropcontact — work email with its qualification, phone, title. GDPR-compliant +# by construction; the lookup is asynchronous and takes a few seconds. +# https://app.dropcontact.com/api +# DROPCONTACT_API_KEY="" +# +# ZoomInfo — work email, direct and mobile phones, title. Needs a ZoomInfo API +# user; both halves must be set, and a token is minted from them and cached. +# ZOOMINFO_USERNAME="" +# ZOOMINFO_PASSWORD="" + # GitHub — raises the rate limit when matching contacts to GitHub profiles. # Any classic token with no scopes will do. # GITHUB_TOKEN="" diff --git a/apps/agent/agent/lib/apollo.ts b/apps/agent/agent/lib/apollo.ts new file mode 100644 index 000000000..78973c023 --- /dev/null +++ b/apps/agent/agent/lib/apollo.ts @@ -0,0 +1,103 @@ +import { z } from "zod"; +import { + type ContactDetails, + fetchJson, + keyed, + type Outcome, + type Person, + type Provider, +} from "./contact-details"; +import { CONTACT_DETAILS } from "./contact-details-config"; + +export const APOLLO_API_KEY = "APOLLO_API_KEY"; + +const phone = z.object({ + sanitized_number: z.string().trim().min(1).nullable().optional(), + raw_number: z.string().trim().min(1).nullable().optional(), + type: z.string().nullable().optional(), +}); + +const match = z.object({ + person: z + .object({ + id: z.string().nullable().optional(), + email: z.string().trim().email().nullable().optional(), + email_status: z.string().nullable().optional(), + title: z.string().nullable().optional(), + linkedin_url: z.string().nullable().optional(), + phone_numbers: z.array(phone).nullable().optional(), + }) + .nullable() + .optional(), +}); + +function confidenceOf(status: string | null | undefined): number { + const scale = CONTACT_DETAILS.apollo.confidence; + switch (status) { + case "verified": + return scale.verified; + case "likely_to_engage": + case "likely": + return scale.likely; + case "guessed": + case "extrapolated": + return scale.guessed; + default: + return scale.unknown; + } +} + +export async function apolloMatch( + person: Person, +): Promise> { + const apiKey = process.env[APOLLO_API_KEY]?.trim(); + if (!apiKey) return { ok: false, reason: `No ${APOLLO_API_KEY}.` }; + + const response = await fetchJson( + new URL(`${CONTACT_DETAILS.apollo.baseUrl}/people/match`), + { + method: "POST", + headers: { + "content-type": "application/json", + "x-api-key": apiKey, + }, + body: JSON.stringify({ + first_name: person.firstName.trim(), + last_name: person.lastName.trim(), + domain: person.domain.trim().toLowerCase(), + organization_name: person.companyName ?? undefined, + reveal_personal_emails: false, + reveal_phone_number: false, + }), + }, + match, + "Apollo", + ); + if (!response.ok) return response; + + const found = response.data.person; + return { + ok: true, + data: { + provider: "apollo", + email: found?.email ?? null, + confidence: found?.email ? confidenceOf(found.email_status) : 0, + phones: (found?.phone_numbers ?? []).flatMap((entry) => { + const number = entry.sanitized_number ?? entry.raw_number; + return number ? [{ number, type: entry.type ?? null }] : []; + }), + title: found?.title ?? null, + linkedinUrl: found?.linkedin_url ?? null, + sources: [], + reference: found?.id ?? null, + }, + }; +} + +export const apollo: Provider = { + id: "apollo", + label: "Apollo", + keys: [APOLLO_API_KEY], + enabled: keyed(APOLLO_API_KEY), + find: apolloMatch, +}; diff --git a/apps/agent/agent/lib/capabilities.ts b/apps/agent/agent/lib/capabilities.ts index 2e587e955..a0e4ea97c 100644 --- a/apps/agent/agent/lib/capabilities.ts +++ b/apps/agent/agent/lib/capabilities.ts @@ -66,6 +66,46 @@ export function capabilitiesFrom( "a person read back from a LinkedIn URL you already hold — their real name, bio, current title and employer, every earlier role with its dates, their education and their other public profiles, all self-reported and so authoritative on identity", enabled: contextDev !== null, }, + { + ...fromEnv("HUNTER_API_KEY"), + label: "Contact details via Hunter", + gives: + "a person's work email from their name and their employer's domain, with the public pages Hunter saw it on, and a deliverability check on an address you already hold", + }, + { + ...fromEnv("APOLLO_API_KEY"), + label: "Contact details via Apollo", + gives: + "a person's work email with Apollo's verification status, their title and a work phone", + }, + { + ...fromEnv("LUSHA_API_KEY"), + label: "Contact details via Lusha", + gives: "a person's work email, direct and mobile phones and their title", + }, + { + ...fromEnv("DROPCONTACT_API_KEY"), + label: "Contact details via Dropcontact", + gives: + "a person's work email with its qualification, a phone and their title, from a GDPR-compliant source", + }, + { + id: "ZOOMINFO_USERNAME", + from: "ZOOMINFO_USERNAME + ZOOMINFO_PASSWORD", + label: "Contact details via ZoomInfo", + gives: + "a person's work email, direct and mobile phones and their title from ZoomInfo's database", + enabled: Boolean( + process.env.ZOOMINFO_USERNAME?.trim() && + process.env.ZOOMINFO_PASSWORD?.trim(), + ), + }, + { + ...fromEnv("EDGAR_URL"), + label: "SEC EDGAR research", + gives: + "US public companies from SEC filings: profile, filings, 5%+ shareholders, insider transactions, the proxy statement with its executives and their pay, all free and with a filing URL to cite", + }, { ...fromEnv("BLOB_READ_WRITE_TOKEN"), label: "Picture storage", diff --git a/apps/agent/agent/lib/companies.ts b/apps/agent/agent/lib/companies.ts new file mode 100644 index 000000000..90d458070 --- /dev/null +++ b/apps/agent/agent/lib/companies.ts @@ -0,0 +1,225 @@ +import { ActivityType, db, type Prisma } from "@crm/db"; +import { PRIORITY } from "@crm/db/agent-tasks"; +import { DISPATCH } from "./dispatch-config"; +import { createField, listFields, writeField } from "./fields"; +import { hostOf } from "./names"; +import { scheduleTask } from "./tasks"; + +export const LEI_FIELD_LABEL = "LEI"; +export const CIK_FIELD_LABEL = "CIK"; +export const TICKER_FIELD_LABEL = "TICKER"; +export const SIC_FIELD_LABEL = "SIC"; + +const IDENTIFIER_BRIEFS = { + [LEI_FIELD_LABEL]: + "The 20-character Legal Entity Identifier from the GLEIF register.", + [CIK_FIELD_LABEL]: + "The SEC Central Index Key of a company that files with EDGAR.", + [TICKER_FIELD_LABEL]: "The stock ticker of a listed company.", + [SIC_FIELD_LABEL]: + "The four-digit Standard Industrial Classification code the SEC assigns.", +} as const; + +type IdentifierLabel = keyof typeof IDENTIFIER_BRIEFS; + +export type CompanySource = { label: string; url: string }; + +export type NewCompany = { + name: string; + website?: string | null; + countryCode?: string | null; + country?: string | null; + city?: string | null; + lei?: string | null; + cik?: string | null; + ticker?: string | null; + sic?: string | null; + stateCode?: string | null; + source: CompanySource; +}; + +export type CreatedCompany = { + created: boolean; + id: string; + name: string; + domain: string | null; + reason?: string; +}; + +function domainFrom(website: string | null | undefined): string | null { + if (!website) return null; + const host = hostOf(website); + return host.includes(".") ? host : null; +} + +async function existingCompany( + name: string, + domain: string | null, + countryCode: string | null, +): Promise<{ id: string; name: string; domain: string | null } | null> { + const select = { id: true, name: true, domain: true }; + + if (domain) { + const byDomain = await db.company.findFirst({ + where: { domain, archivedAt: null }, + select, + }); + if (byDomain) return byDomain; + } + + const where: Prisma.CompanyWhereInput = { + name: { equals: name, mode: "insensitive" }, + archivedAt: null, + }; + if (countryCode) where.countryCode = countryCode; + + return db.company.findFirst({ where, select }); +} + +async function authorId(): Promise { + const user = await db.user.findFirst({ + orderBy: { createdAt: "asc" }, + select: { id: true }, + }); + return user?.id ?? null; +} + +async function recordIdentifier( + companyId: string, + label: IdentifierLabel, + value: string, +): Promise { + const fields = await listFields("COMPANY"); + const existing = fields.find((field) => field.label.toUpperCase() === label); + const key = existing + ? existing.key + : await createField({ + entity: "COMPANY", + label: label === TICKER_FIELD_LABEL ? "Ticker" : label, + type: "TEXT", + agentBrief: IDENTIFIER_BRIEFS[label], + }).then((field) => ("created" in field ? null : field.key)); + + if (key) + await writeField({ + entity: "COMPANY", + recordId: companyId, + key, + value, + }); +} + +function identifiersOf(input: NewCompany): [IdentifierLabel, string][] { + const pairs: [IdentifierLabel, string | null | undefined][] = [ + [LEI_FIELD_LABEL, input.lei?.trim().toUpperCase()], + [CIK_FIELD_LABEL, input.cik?.trim().replace(/^0+(?=\d)/, "")], + [TICKER_FIELD_LABEL, input.ticker?.trim().toUpperCase()], + [SIC_FIELD_LABEL, input.sic?.trim()], + ]; + return pairs.flatMap(([label, value]) => (value ? [[label, value]] : [])); +} + +export async function createCompany( + input: NewCompany, +): Promise { + const name = input.name.trim(); + const domain = domainFrom(input.website); + const countryCode = + input.countryCode?.trim().toUpperCase() || (input.cik ? "US" : null); + + const existing = await existingCompany(name, domain, countryCode); + if (existing) { + return { + created: false, + ...existing, + reason: domain + ? `${existing.name} already uses the domain ${domain}.` + : `${existing.name} is already in the CRM.`, + }; + } + + const occurredAt = new Date(); + const created = await db.$transaction(async (tx) => { + const company = await tx.company.create({ + data: { + name, + domain, + website: domain ? `https://${domain}` : null, + countryCode, + country: input.country?.trim() || null, + city: input.city?.trim() || null, + stateCode: input.stateCode?.trim().toUpperCase() || null, + }, + select: { id: true, name: true, domain: true }, + }); + + const payload: Prisma.InputJsonObject = { + type: "company.created", + record: { kind: "company", id: company.id }, + occurredAt: occurredAt.toISOString(), + data: { name: company.name, domain: company.domain }, + }; + await tx.agentTask.create({ + data: { + companyId: company.id, + kind: "agent-event", + reason: "company.created", + payload, + priority: PRIORITY.event, + budget: 1, + dueAt: occurredAt, + }, + }); + + return company; + }); + + const author = await authorId(); + if (author) { + await db.activity.create({ + data: { + type: ActivityType.ENRICHMENT, + subject: `Added from ${input.source.label}`, + body: [ + `${created.name} was added by the agent from ${input.source.label}.`, + ...identifiersOf(input).map(([label, value]) => `${label} ${value}.`), + `Source: ${input.source.url}`, + ] + .filter(Boolean) + .join(" "), + occurredAt, + companyId: created.id, + createdById: author, + meta: { + source: input.source.label, + sourceUrl: input.source.url, + agent: "sourcing", + }, + }, + select: { id: true }, + }); + } + + for (const [label, value] of identifiersOf(input)) { + await recordIdentifier(created.id, label, value); + } + + await scheduleTask({ + companyId: created.id, + kind: "brand", + reason: "New company", + dueAt: occurredAt, + priority: PRIORITY.brand, + budget: DISPATCH.newCompany.brandBudget, + }); + await scheduleTask({ + companyId: created.id, + kind: "company-profile", + reason: "New company", + dueAt: occurredAt, + priority: PRIORITY.companyProfile, + budget: DISPATCH.newCompany.profileBudget, + }); + + return { created: true, ...created }; +} diff --git a/apps/agent/agent/lib/contact-details-config.ts b/apps/agent/agent/lib/contact-details-config.ts new file mode 100644 index 000000000..92956270b --- /dev/null +++ b/apps/agent/agent/lib/contact-details-config.ts @@ -0,0 +1,56 @@ +const SECOND_MS = 1_000; +const MINUTE_MS = 60 * SECOND_MS; + +export const CONTACT_DETAILS = { + timeoutMs: 20 * SECOND_MS, + minConfidence: 50, + maxSources: 5, + order: ["hunter", "apollo", "lusha", "dropcontact", "zoominfo", "website"], + + hunter: { + baseUrl: "https://api.hunter.io/v2", + }, + + apollo: { + baseUrl: "https://api.apollo.io/api/v1", + confidence: { verified: 95, likely: 70, guessed: 55, unknown: 30 }, + }, + + lusha: { + baseUrl: "https://api.lusha.com", + confidence: { work: 85, other: 60 }, + }, + + dropcontact: { + baseUrl: "https://api.dropcontact.io", + pollMs: 3 * SECOND_MS, + maxPolls: 10, + confidence: { nominative: 90, catchAll: 55, other: 40 }, + }, + + zoominfo: { + baseUrl: "https://api.zoominfo.com", + tokenTtlMs: 55 * MINUTE_MS, + confidence: { matched: 80 }, + }, + + website: { + paths: [ + "/", + "/contact", + "/team", + "/about", + "/equipe", + "/notre-equipe", + "/a-propos", + "/mentions-legales", + "/legal", + "/impressum", + ], + maxPages: 8, + maxBytes: 512 * 1024, + timeoutMs: 6 * SECOND_MS, + userAgent: "crm-contact-lookup/1 (+https://github.com/trycompai/crm)", + confidence: { named: 75, phoneOnly: 40 }, + }, +} as const; diff --git a/apps/agent/agent/lib/contact-details-providers.ts b/apps/agent/agent/lib/contact-details-providers.ts new file mode 100644 index 000000000..e271146e5 --- /dev/null +++ b/apps/agent/agent/lib/contact-details-providers.ts @@ -0,0 +1,16 @@ +import { apollo } from "./apollo"; +import type { Provider } from "./contact-details"; +import { dropcontact } from "./dropcontact"; +import { hunter } from "./hunter"; +import { lusha } from "./lusha"; +import { website } from "./website"; +import { zoominfo } from "./zoominfo"; + +export const CONTACT_DETAILS_PROVIDERS: readonly Provider[] = [ + hunter, + apollo, + lusha, + dropcontact, + zoominfo, + website, +]; diff --git a/apps/agent/agent/lib/contact-details.ts b/apps/agent/agent/lib/contact-details.ts new file mode 100644 index 000000000..145b291c7 --- /dev/null +++ b/apps/agent/agent/lib/contact-details.ts @@ -0,0 +1,128 @@ +import type { z } from "zod"; +import { CONTACT_DETAILS } from "./contact-details-config"; + +export type Outcome = { ok: true; data: T } | { ok: false; reason: string }; + +export type ProviderId = (typeof CONTACT_DETAILS.order)[number]; + +export type DetailSource = { + url: string; + domain: string | null; + seenOn: string | null; +}; + +export type Phone = { number: string; type: string | null }; + +export type ContactDetails = { + provider: ProviderId; + email: string | null; + confidence: number; + phones: Phone[]; + title: string | null; + linkedinUrl: string | null; + sources: DetailSource[]; + reference: string | null; +}; + +export type Person = { + firstName: string; + lastName: string; + domain: string; + companyName: string | null; +}; + +export type Provider = { + id: ProviderId; + label: string; + keys: readonly string[]; + enabled: () => boolean; + find: (person: Person) => Promise>; +}; + +export type Lookup = + | { outcome: "found"; details: ContactDetails; tried: ProviderId[] } + | { outcome: "none"; tried: ProviderId[]; reasons: string[] } + | { outcome: "unconfigured" }; + +export function keyed(...names: string[]): () => boolean { + return () => names.every((name) => Boolean(process.env[name]?.trim())); +} + +export async function fetchJson( + url: URL, + init: RequestInit, + shape: Shape, + label: string, +): Promise>> { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), CONTACT_DETAILS.timeoutMs); + + try { + const response = await fetch(url, { ...init, signal: controller.signal }); + if (!response.ok) return { ok: false, reason: `HTTP ${response.status}` }; + + const parsed = shape.safeParse(await response.json()); + return parsed.success + ? { ok: true, data: parsed.data } + : { + ok: false, + reason: `Unreadable ${label} response: ${parsed.error.message}`, + }; + } catch (error) { + const aborted = error instanceof Error && error.name === "AbortError"; + return { + ok: false, + reason: aborted + ? `${label} timed out after ${CONTACT_DETAILS.timeoutMs}ms.` + : error instanceof Error + ? error.message + : String(error), + }; + } finally { + clearTimeout(timer); + } +} + +export function configuredProviders(all: readonly Provider[]): Provider[] { + return CONTACT_DETAILS.order.flatMap((id) => { + const provider = all.find((candidate) => candidate.id === id); + return provider?.enabled() ? [provider] : []; + }); +} + +export async function lookupContactDetails( + person: Person, + all: readonly Provider[], +): Promise { + const providers = configuredProviders(all); + if (providers.length === 0) return { outcome: "unconfigured" }; + + const tried: ProviderId[] = []; + const reasons: string[] = []; + + for (const provider of providers) { + tried.push(provider.id); + const result = await provider.find(person); + + if (!result.ok) { + reasons.push(`${provider.label}: ${result.reason}`); + continue; + } + + const { data } = result; + if (data.email && data.confidence >= CONTACT_DETAILS.minConfidence) { + return { outcome: "found", details: data, tried }; + } + if (data.phones.length > 0 && !data.email) { + return { outcome: "found", details: data, tried }; + } + + reasons.push( + data.email + ? `${provider.label}: ${data.email} at confidence ${data.confidence}, below ${CONTACT_DETAILS.minConfidence}` + : `${provider.label}: nothing for this person`, + ); + } + + return { outcome: "none", tried, reasons }; +} diff --git a/apps/agent/agent/lib/contacts.ts b/apps/agent/agent/lib/contacts.ts new file mode 100644 index 000000000..ccdb706c3 --- /dev/null +++ b/apps/agent/agent/lib/contacts.ts @@ -0,0 +1,163 @@ +import { ActivityType, db, type Prisma } from "@crm/db"; +import { PRIORITY } from "@crm/db/agent-tasks"; +import type { CompanySource } from "./companies"; + +export type NewContact = { + firstName: string; + lastName?: string | null; + title?: string | null; + email?: string | null; + companyId: string; + source: CompanySource; +}; + +export type CreatedContact = { + created: boolean; + id: string; + firstName: string; + lastName: string | null; + companyId: string | null; + reason?: string; +}; + +const select = { + id: true, + firstName: true, + lastName: true, + companyId: true, +}; + +async function existingContact( + input: NewContact, + email: string | null, +): Promise { + if (email) { + const byEmail = await db.contact.findFirst({ + where: { + email: { equals: email, mode: "insensitive" }, + archivedAt: null, + }, + select, + }); + if (byEmail) return { created: false, ...byEmail }; + } + + const where: Prisma.ContactWhereInput = { + firstName: { equals: input.firstName.trim(), mode: "insensitive" }, + companyId: input.companyId, + archivedAt: null, + }; + const lastName = input.lastName?.trim(); + if (lastName) where.lastName = { equals: lastName, mode: "insensitive" }; + + const byName = await db.contact.findFirst({ where, select }); + return byName ? { created: false, ...byName } : null; +} + +export async function createContact( + input: NewContact, +): Promise { + const company = await db.company.findFirst({ + where: { id: input.companyId, archivedAt: null }, + select: { id: true, name: true, ownerId: true }, + }); + if (!company) { + return { + created: false, + id: "", + firstName: input.firstName, + lastName: input.lastName ?? null, + companyId: null, + reason: "No such company. Add the company first.", + }; + } + + const email = input.email?.trim().toLowerCase() || null; + const existing = await existingContact(input, email); + if (existing) { + return { + ...existing, + reason: `${existing.firstName} ${existing.lastName ?? ""}` + .trim() + .concat(" is already in the CRM."), + }; + } + + const occurredAt = new Date(); + const created = await db.$transaction(async (tx) => { + const contact = await tx.contact.create({ + data: { + firstName: input.firstName.trim(), + lastName: input.lastName?.trim() || null, + title: input.title?.trim() || null, + email, + companyId: company.id, + ownerId: company.ownerId, + }, + select, + }); + + const payload: Prisma.InputJsonObject = { + type: "contact.created", + record: { kind: "contact", id: contact.id }, + occurredAt: occurredAt.toISOString(), + data: { + firstName: contact.firstName, + lastName: contact.lastName, + companyId: company.id, + }, + }; + await tx.agentTask.create({ + data: { + contactId: contact.id, + companyId: company.id, + kind: "agent-event", + reason: "contact.created", + payload, + priority: PRIORITY.event, + budget: 1, + dueAt: occurredAt, + }, + }); + + return contact; + }); + + const author = + company.ownerId ?? + ( + await db.user.findFirst({ + orderBy: { createdAt: "asc" }, + select: { id: true }, + }) + )?.id ?? + null; + if (author) { + await db.activity.create({ + data: { + type: ActivityType.ENRICHMENT, + subject: `Added from ${input.source.label}`, + body: [ + `${created.firstName} ${created.lastName ?? ""}`.trim(), + input.title?.trim() ? `(${input.title.trim()})` : null, + `was added by the agent to ${company.name} from ${input.source.label}.`, + `Source: ${input.source.url}`, + ] + .filter(Boolean) + .join(" "), + occurredAt, + contactId: created.id, + companyId: company.id, + createdById: author, + meta: { + source: input.source.label, + sourceUrl: input.source.url, + agent: "sourcing", + }, + }, + select: { id: true }, + }); + } + + return { created: true, ...created }; +} diff --git a/apps/agent/agent/lib/dispatch-config.ts b/apps/agent/agent/lib/dispatch-config.ts index 4382b4a80..957df9944 100644 --- a/apps/agent/agent/lib/dispatch-config.ts +++ b/apps/agent/agent/lib/dispatch-config.ts @@ -32,6 +32,11 @@ export const DISPATCH = { leaseMs: 10 * MINUTE_MS, }, + newCompany: { + brandBudget: 2, + profileBudget: 4, + }, + reconcile: { scan: 200, retire: 100, diff --git a/apps/agent/agent/lib/dropcontact.ts b/apps/agent/agent/lib/dropcontact.ts new file mode 100644 index 000000000..779e88301 --- /dev/null +++ b/apps/agent/agent/lib/dropcontact.ts @@ -0,0 +1,128 @@ +import { z } from "zod"; +import { + type ContactDetails, + fetchJson, + keyed, + type Outcome, + type Person, + type Provider, +} from "./contact-details"; +import { CONTACT_DETAILS } from "./contact-details-config"; + +export const DROPCONTACT_API_KEY = "DROPCONTACT_API_KEY"; + +const submitted = z.object({ + request_id: z.string().trim().min(1), +}); + +const email = z.object({ + email: z.string().trim().email(), + qualification: z.string().nullable().optional(), +}); + +const row = z.object({ + email: z.array(email).nullable().optional(), + phone: z.string().nullable().optional(), + mobile_phone: z.string().nullable().optional(), + job: z.string().nullable().optional(), + linkedin: z.string().nullable().optional(), +}); + +const batch = z.object({ + success: z.boolean(), + data: z.array(row).nullable().optional(), +}); + +function confidenceOf(qualification: string | null | undefined): number { + const scale = CONTACT_DETAILS.dropcontact.confidence; + const value = (qualification ?? "").toLowerCase(); + if (value.startsWith("nominative")) return scale.nominative; + if (value.startsWith("catch_all")) return scale.catchAll; + return scale.other; +} + +function wait(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +export async function dropcontactEnrich( + person: Person, +): Promise> { + const apiKey = process.env[DROPCONTACT_API_KEY]?.trim(); + if (!apiKey) return { ok: false, reason: `No ${DROPCONTACT_API_KEY}.` }; + + const headers = { + "content-type": "application/json", + "X-Access-Token": apiKey, + }; + const base = CONTACT_DETAILS.dropcontact.baseUrl; + + const submit = await fetchJson( + new URL(`${base}/batch`), + { + method: "POST", + headers, + body: JSON.stringify({ + data: [ + { + first_name: person.firstName.trim(), + last_name: person.lastName.trim(), + website: person.domain.trim().toLowerCase(), + company: person.companyName ?? undefined, + }, + ], + siren: false, + }), + }, + submitted, + "Dropcontact", + ); + if (!submit.ok) return submit; + + for (let poll = 0; poll < CONTACT_DETAILS.dropcontact.maxPolls; poll++) { + await wait(CONTACT_DETAILS.dropcontact.pollMs); + + const result = await fetchJson( + new URL(`${base}/batch/${encodeURIComponent(submit.data.request_id)}`), + { headers }, + batch, + "Dropcontact", + ); + if (!result.ok) return result; + if (!result.data.success) continue; + + const found = result.data.data?.[0] ?? null; + const best = found?.email?.[0] ?? null; + const phones = [found?.phone, found?.mobile_phone].flatMap( + (number, index) => + number ? [{ number, type: index === 0 ? "work" : "mobile" }] : [], + ); + + return { + ok: true, + data: { + provider: "dropcontact", + email: best?.email ?? null, + confidence: best ? confidenceOf(best.qualification) : 0, + phones, + title: found?.job ?? null, + linkedinUrl: found?.linkedin ?? null, + sources: [], + reference: submit.data.request_id, + }, + }; + } + + return { + ok: false, + reason: `Dropcontact did not finish within ${CONTACT_DETAILS.dropcontact.maxPolls} polls.`, + }; +} + +export const dropcontact: Provider = { + id: "dropcontact", + label: "Dropcontact", + keys: [DROPCONTACT_API_KEY], + enabled: keyed(DROPCONTACT_API_KEY), + find: dropcontactEnrich, +}; diff --git a/apps/agent/agent/lib/edgar-config.ts b/apps/agent/agent/lib/edgar-config.ts new file mode 100644 index 000000000..4979a1946 --- /dev/null +++ b/apps/agent/agent/lib/edgar-config.ts @@ -0,0 +1,32 @@ +const SECOND_MS = 1_000; + +export const EDGAR = { + env: { + url: "EDGAR_URL", + secret: "EDGAR_SECRET", + }, + timeoutMs: 45 * SECOND_MS, + search: { + defaultLimit: 10, + maxLimit: 25, + }, + filings: { + defaultLimit: 20, + maxLimit: 100, + }, + owners: { + minPercent: 5, + defaultLimit: 20, + maxLimit: 50, + }, + insiders: { + defaultLimit: 20, + maxLimit: 100, + }, + compensation: { + years: 3, + maxYears: 5, + maxTickers: 10, + }, + browseUrl: "https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK=", +} as const; diff --git a/apps/agent/agent/lib/edgar.ts b/apps/agent/agent/lib/edgar.ts new file mode 100644 index 000000000..3476323a6 --- /dev/null +++ b/apps/agent/agent/lib/edgar.ts @@ -0,0 +1,224 @@ +import { + edgarCompany, + edgarCompanySearch, + edgarCompensationComparison, + edgarFilingSearch, + edgarFilings, + edgarHealth, + edgarInsiders, + edgarOwners, + edgarProxy, +} from "@crm/validation/edgar"; +import type { z } from "zod"; +import { EDGAR } from "./edgar-config"; + +export type Outcome = { ok: true; data: T } | { ok: false; reason: string }; + +type Query = Record; + +export function edgarUrl(): string | null { + const url = process.env[EDGAR.env.url]?.trim(); + return url ? url.replace(/\/+$/, "") : null; +} + +export function edgarEnabled(): boolean { + return edgarUrl() !== null; +} + +function headers(): Record { + const secret = process.env[EDGAR.env.secret]?.trim(); + return secret + ? { accept: "application/json", authorization: `Bearer ${secret}` } + : { accept: "application/json" }; +} + +export function companyUrl(cik: string): string { + return `${EDGAR.browseUrl}${cik}`; +} + +async function request( + path: string, + query: Query, + shape: Shape, + notFound: z.infer | null, +): Promise>> { + const base = edgarUrl(); + if (!base) return { ok: false, reason: `No ${EDGAR.env.url}.` }; + + const url = new URL(`${base}${path}`); + for (const [key, value] of Object.entries(query)) { + if (value !== undefined) url.searchParams.set(key, String(value)); + } + + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), EDGAR.timeoutMs); + + try { + const response = await fetch(url, { + headers: headers(), + signal: controller.signal, + }); + + if (response.status === 404) { + if (notFound !== null) return { ok: true, data: notFound }; + return { ok: false, reason: await reasonOf(response) }; + } + if (!response.ok) { + return { ok: false, reason: `HTTP ${response.status}` }; + } + + const parsed = shape.safeParse(await response.json()); + return parsed.success + ? { ok: true, data: parsed.data } + : { + ok: false, + reason: `Unreadable EDGAR service response: ${parsed.error.message}`, + }; + } catch (error) { + const aborted = error instanceof Error && error.name === "AbortError"; + return { + ok: false, + reason: aborted + ? `The EDGAR service timed out after ${EDGAR.timeoutMs}ms.` + : error instanceof Error + ? error.message + : String(error), + }; + } finally { + clearTimeout(timer); + } +} + +async function reasonOf(response: Response): Promise { + const text = await response.text(); + try { + const parsed: { reason?: string } = JSON.parse(text); + return parsed.reason ?? "Not found."; + } catch { + return text || "Not found."; + } +} + +function normalizeCik(value: string): string { + return value.trim().replace(/^0+(?=\d)/, ""); +} + +export function health() { + return request("/health", {}, edgarHealth, null); +} + +export function searchCompanies(input: { query: string; limit?: number }) { + return request( + "/companies/search", + { q: input.query.trim(), limit: input.limit ?? EDGAR.search.defaultLimit }, + edgarCompanySearch, + { companies: [] }, + ); +} + +export function getCompany(input: { cik?: string; ticker?: string }) { + const key = input.cik + ? normalizeCik(input.cik) + : (input.ticker?.trim().toUpperCase() ?? ""); + return request( + `/companies/${encodeURIComponent(key)}`, + {}, + edgarCompany, + null, + ); +} + +export function listFilings(input: { + cik: string; + form?: string; + from?: string; + to?: string; + limit?: number; +}) { + return request( + `/companies/${encodeURIComponent(normalizeCik(input.cik))}/filings`, + { + form: input.form?.trim() || undefined, + from: input.from, + to: input.to, + limit: input.limit ?? EDGAR.filings.defaultLimit, + }, + edgarFilings, + { filings: [], truncated: false }, + ); +} + +export function searchFilings(input: { + query: string; + form?: string; + from?: string; + to?: string; + limit?: number; +}) { + return request( + "/filings/search", + { + q: input.query.trim(), + form: input.form?.trim() || undefined, + from: input.from, + to: input.to, + limit: input.limit ?? EDGAR.filings.defaultLimit, + }, + edgarFilingSearch, + { filings: [], total: 0 }, + ); +} + +export function listOwners(input: { + cik: string; + minPercent?: number; + form?: string; + limit?: number; +}) { + return request( + `/companies/${encodeURIComponent(normalizeCik(input.cik))}/owners`, + { + minPercent: input.minPercent ?? EDGAR.owners.minPercent, + form: input.form, + limit: input.limit ?? EDGAR.owners.defaultLimit, + }, + edgarOwners, + { owners: [], filingsRead: 0 }, + ); +} + +export function listInsiders(input: { cik: string; limit?: number }) { + return request( + `/companies/${encodeURIComponent(normalizeCik(input.cik))}/insiders`, + { limit: input.limit ?? EDGAR.insiders.defaultLimit }, + edgarInsiders, + { transactions: [] }, + ); +} + +export function getProxy(input: { cik: string; years?: number }) { + return request( + `/companies/${encodeURIComponent(normalizeCik(input.cik))}/proxy`, + { years: input.years ?? EDGAR.compensation.years }, + edgarProxy, + null, + ); +} + +export function compareCompensation(input: { + tickers: readonly string[]; + years?: number; +}) { + return request( + "/compensation/compare", + { + tickers: input.tickers + .map((ticker) => ticker.trim().toUpperCase()) + .filter(Boolean) + .join(","), + years: input.years ?? EDGAR.compensation.years, + }, + edgarCompensationComparison, + { rows: [] }, + ); +} diff --git a/apps/agent/agent/lib/gleif-config.ts b/apps/agent/agent/lib/gleif-config.ts new file mode 100644 index 000000000..448fbc253 --- /dev/null +++ b/apps/agent/agent/lib/gleif-config.ts @@ -0,0 +1,71 @@ +const SECOND_MS = 1_000; + +export const GLEIF = { + api: { + baseUrl: "https://api.gleif.org/api/v1", + timeoutMs: 20 * SECOND_MS, + pageSize: 100, + maxPages: 10, + }, + + search: { + defaultLimit: 10, + maxLimit: 50, + }, + + regions: { + UE: [ + "AT", + "BE", + "BG", + "CY", + "CZ", + "DE", + "DK", + "EE", + "ES", + "FI", + "FR", + "GR", + "HR", + "HU", + "IE", + "IT", + "LT", + "LU", + "LV", + "MT", + "NL", + "PL", + "PT", + "RO", + "SE", + "SI", + "SK", + ], + ASIE: [ + "BD", + "BN", + "CN", + "HK", + "ID", + "IN", + "JP", + "KH", + "KR", + "LA", + "LK", + "MM", + "MN", + "MO", + "MY", + "NP", + "PH", + "PK", + "SG", + "TH", + "TW", + "VN", + ], + }, +} as const; diff --git a/apps/agent/agent/lib/gleif.ts b/apps/agent/agent/lib/gleif.ts new file mode 100644 index 000000000..7fece22c8 --- /dev/null +++ b/apps/agent/agent/lib/gleif.ts @@ -0,0 +1,253 @@ +import { z } from "zod"; +import { GLEIF } from "./gleif-config"; + +export type Outcome = { ok: true; data: T } | { ok: false; reason: string }; + +const legalName = z.object({ name: z.string().trim().min(1) }); + +const legalAddress = z.object({ + country: z.string().trim().length(2), + city: z.string().trim().min(1).nullable().optional(), +}); + +const otherName = z.object({ + name: z.string().trim().min(1), + type: z.string().nullable().optional(), +}); + +const record = z.object({ + id: z.string().trim().length(20), + attributes: z.object({ + entity: z.object({ + legalName, + legalAddress, + otherNames: z.array(otherName).nullable().optional(), + transliteratedOtherNames: z.array(otherName).nullable().optional(), + status: z.string().nullable().optional(), + category: z.string().nullable().optional(), + jurisdiction: z.string().nullable().optional(), + }), + registration: z + .object({ status: z.string().nullable().optional() }) + .optional(), + }), +}); + +const pagination = z.object({ + currentPage: z.number().int(), + lastPage: z.number().int(), + total: z.number().int(), +}); + +const page = z.object({ + data: z.array(record), + meta: z.object({ pagination }).optional(), +}); + +const single = z.object({ data: record.nullable() }); + +export const gleifEntity = record.transform(({ id, attributes }) => ({ + lei: id, + name: attributes.entity.legalName.name, + alternativeNames: [ + ...new Set( + [ + ...(attributes.entity.otherNames ?? []), + ...(attributes.entity.transliteratedOtherNames ?? []), + ] + .map((other) => other.name) + .filter((name) => name !== attributes.entity.legalName.name), + ), + ], + country: attributes.entity.legalAddress.country.toUpperCase(), + city: attributes.entity.legalAddress.city ?? null, + status: attributes.entity.status ?? null, + registrationStatus: attributes.registration?.status ?? null, + category: attributes.entity.category ?? null, + jurisdiction: attributes.entity.jurisdiction ?? null, +})); + +export type GleifEntity = z.infer; + +export type Subsidiaries = { + parent: string; + children: GleifEntity[]; + total: number; + truncated: boolean; +}; + +export const ENTITY_CATEGORIES = [ + "GENERAL", + "FUND", + "BRANCH", + "SOLE_PROPRIETOR", + "RESIDENT_GOVERNMENT_ENTITY", + "INTERNATIONAL_ORGANIZATION", +] as const; + +export type EntityCategory = (typeof ENTITY_CATEGORIES)[number]; + +type Query = Record; + +export function resolveCountries(spec: string | undefined): string[] { + if (!spec) return []; + const regions: Record = GLEIF.regions; + return [ + ...new Set( + spec + .split(",") + .map((part) => part.trim().toUpperCase()) + .filter(Boolean) + .flatMap((part) => regions[part] ?? [part]) + .filter((code) => /^[A-Z]{2}$/.test(code)), + ), + ]; +} + +function normalizeLei(lei: string): string { + return encodeURIComponent(lei.trim().toUpperCase()); +} + +async function request( + path: string, + query: Query, + shape: Shape, + notFound: z.infer, +): Promise>> { + const url = new URL(`${GLEIF.api.baseUrl}${path}`); + for (const [key, value] of Object.entries(query)) { + if (value !== undefined) url.searchParams.set(key, value); + } + + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), GLEIF.api.timeoutMs); + + try { + const response = await fetch(url, { + headers: { accept: "application/vnd.api+json" }, + signal: controller.signal, + }); + + if (response.status === 404) return { ok: true, data: notFound }; + if (!response.ok) return { ok: false, reason: `HTTP ${response.status}` }; + + const parsed = shape.safeParse(await response.json()); + return parsed.success + ? { ok: true, data: parsed.data } + : { + ok: false, + reason: `Unreadable GLEIF response: ${parsed.error.message}`, + }; + } catch (error) { + const aborted = error instanceof Error && error.name === "AbortError"; + return { + ok: false, + reason: aborted + ? `GLEIF timed out after ${GLEIF.api.timeoutMs}ms.` + : error instanceof Error + ? error.message + : String(error), + }; + } finally { + clearTimeout(timer); + } +} + +export async function searchEntities(input: { + name: string; + country?: string; + category?: EntityCategory | "ANY"; + activeOnly?: boolean; + limit?: number; +}): Promise> { + const limit = Math.min( + input.limit ?? GLEIF.search.defaultLimit, + GLEIF.search.maxLimit, + ); + const category = input.category ?? "GENERAL"; + + const response = await request( + "/lei-records", + { + "filter[entity.legalName]": input.name.trim(), + "filter[entity.legalAddress.country]": input.country + ?.trim() + .toUpperCase(), + "filter[entity.status]": + (input.activeOnly ?? true) ? "ACTIVE" : undefined, + "filter[entity.category]": category === "ANY" ? undefined : category, + "page[size]": String(limit), + "page[number]": "1", + }, + page, + { data: [] }, + ); + if (!response.ok) return response; + + return { + ok: true, + data: { + entities: response.data.data.map((row) => gleifEntity.parse(row)), + total: response.data.meta?.pagination.total ?? response.data.data.length, + }, + }; +} + +async function one(path: string): Promise> { + const response = await request(path, {}, single, { data: null }); + if (!response.ok) return response; + + return { + ok: true, + data: response.data.data ? gleifEntity.parse(response.data.data) : null, + }; +} + +export function getEntity(lei: string): Promise> { + return one(`/lei-records/${normalizeLei(lei)}`); +} + +export function directParent( + lei: string, +): Promise> { + return one(`/lei-records/${normalizeLei(lei)}/direct-parent`); +} + +export async function directChildren( + lei: string, + options: { countries?: string[] } = {}, +): Promise> { + const parent = lei.trim().toUpperCase(); + const wanted = new Set(options.countries ?? []); + const children: GleifEntity[] = []; + let total = 0; + let truncated = false; + + for (let number = 1; number <= GLEIF.api.maxPages; number++) { + const response = await request( + `/lei-records/${normalizeLei(parent)}/direct-children`, + { + "page[size]": String(GLEIF.api.pageSize), + "page[number]": String(number), + }, + page, + { data: [] }, + ); + if (!response.ok) return response; + + const meta = response.data.meta?.pagination; + total = meta?.total ?? response.data.data.length; + + for (const row of response.data.data) { + const entity = gleifEntity.parse(row); + if (wanted.size === 0 || wanted.has(entity.country)) { + children.push(entity); + } + } + + if (!meta || number >= meta.lastPage) break; + if (number === GLEIF.api.maxPages) truncated = true; + } + + return { ok: true, data: { parent, children, total, truncated } }; +} diff --git a/apps/agent/agent/lib/hunter.ts b/apps/agent/agent/lib/hunter.ts new file mode 100644 index 000000000..081405bb4 --- /dev/null +++ b/apps/agent/agent/lib/hunter.ts @@ -0,0 +1,119 @@ +import { z } from "zod"; +import { + type ContactDetails, + fetchJson, + keyed, + type Outcome, + type Person, + type Provider, +} from "./contact-details"; +import { CONTACT_DETAILS } from "./contact-details-config"; + +export const HUNTER_API_KEY = "HUNTER_API_KEY"; + +const source = z.object({ + uri: z.string().trim().min(1), + domain: z.string().trim().min(1).nullable().optional(), + extracted_on: z.string().trim().min(1).nullable().optional(), +}); + +const finder = z.object({ + data: z.object({ + email: z.string().trim().email().nullable(), + score: z.number().nullable().optional(), + position: z.string().nullable().optional(), + linkedin_url: z.string().nullable().optional(), + sources: z.array(source).nullable().optional(), + }), +}); + +const verifier = z.object({ + data: z.object({ + status: z.string().trim().min(1), + score: z.number().nullable().optional(), + }), +}); + +export type Verification = { status: string; score: number | null }; + +export function hunterEnabled(): boolean { + return keyed(HUNTER_API_KEY)(); +} + +function endpoint(path: string, query: Record): URL { + const url = new URL(`${CONTACT_DETAILS.hunter.baseUrl}${path}`); + for (const [key, value] of Object.entries(query)) { + url.searchParams.set(key, value); + } + url.searchParams.set("api_key", process.env[HUNTER_API_KEY]?.trim() ?? ""); + return url; +} + +export async function findWorkEmail( + person: Person, +): Promise> { + if (!hunterEnabled()) return { ok: false, reason: `No ${HUNTER_API_KEY}.` }; + + const response = await fetchJson( + endpoint("/email-finder", { + domain: person.domain.trim().toLowerCase(), + first_name: person.firstName.trim(), + last_name: person.lastName.trim(), + }), + {}, + finder, + "Hunter", + ); + if (!response.ok) return response; + + const { data } = response.data; + return { + ok: true, + data: { + provider: "hunter", + email: data.email, + confidence: data.score ?? 0, + phones: [], + title: data.position ?? null, + linkedinUrl: data.linkedin_url ?? null, + sources: (data.sources ?? []) + .slice(0, CONTACT_DETAILS.maxSources) + .map((s) => ({ + url: s.uri, + domain: s.domain ?? null, + seenOn: s.extracted_on ?? null, + })), + reference: null, + }, + }; +} + +export async function verifyEmail( + email: string, +): Promise> { + if (!hunterEnabled()) return { ok: false, reason: `No ${HUNTER_API_KEY}.` }; + + const response = await fetchJson( + endpoint("/email-verifier", { email: email.trim().toLowerCase() }), + {}, + verifier, + "Hunter", + ); + if (!response.ok) return response; + + return { + ok: true, + data: { + status: response.data.data.status, + score: response.data.data.score ?? null, + }, + }; +} + +export const hunter: Provider = { + id: "hunter", + label: "Hunter", + keys: [HUNTER_API_KEY], + enabled: hunterEnabled, + find: findWorkEmail, +}; diff --git a/apps/agent/agent/lib/lusha.ts b/apps/agent/agent/lib/lusha.ts new file mode 100644 index 000000000..269acc975 --- /dev/null +++ b/apps/agent/agent/lib/lusha.ts @@ -0,0 +1,100 @@ +import { z } from "zod"; +import { + type ContactDetails, + fetchJson, + keyed, + type Outcome, + type Person, + type Provider, +} from "./contact-details"; +import { CONTACT_DETAILS } from "./contact-details-config"; + +export const LUSHA_API_KEY = "LUSHA_API_KEY"; + +const email = z.object({ + email: z.string().trim().email(), + emailType: z.string().nullable().optional(), +}); + +const phone = z.object({ + number: z.string().trim().min(1), + phoneType: z.string().nullable().optional(), +}); + +const contact = z.object({ + jobTitle: z.string().nullable().optional(), + emailAddresses: z.array(email).nullable().optional(), + phoneNumbers: z.array(phone).nullable().optional(), + socialLinks: z + .object({ linkedin: z.string().nullable().optional() }) + .nullable() + .optional(), + id: z.union([z.string(), z.number()]).nullable().optional(), +}); + +const person = z.object({ + data: z + .union([z.object({ contact }), contact]) + .nullable() + .optional(), +}); + +function unwrap( + value: z.infer["data"], +): z.infer | null { + if (!value) return null; + return "contact" in value ? value.contact : value; +} + +export async function lushaPerson( + who: Person, +): Promise> { + const apiKey = process.env[LUSHA_API_KEY]?.trim(); + if (!apiKey) return { ok: false, reason: `No ${LUSHA_API_KEY}.` }; + + const url = new URL(`${CONTACT_DETAILS.lusha.baseUrl}/v2/person`); + url.searchParams.set("firstName", who.firstName.trim()); + url.searchParams.set("lastName", who.lastName.trim()); + url.searchParams.set("companyDomain", who.domain.trim().toLowerCase()); + + const response = await fetchJson( + url, + { headers: { api_key: apiKey } }, + person, + "Lusha", + ); + if (!response.ok) return response; + + const found = unwrap(response.data.data); + const work = (found?.emailAddresses ?? []).find( + (entry) => (entry.emailType ?? "").toLowerCase() === "work", + ); + const best = work ?? (found?.emailAddresses ?? [])[0] ?? null; + const scale = CONTACT_DETAILS.lusha.confidence; + + return { + ok: true, + data: { + provider: "lusha", + email: best?.email ?? null, + confidence: best ? (work ? scale.work : scale.other) : 0, + phones: (found?.phoneNumbers ?? []).map((entry) => ({ + number: entry.number, + type: entry.phoneType ?? null, + })), + title: found?.jobTitle ?? null, + linkedinUrl: found?.socialLinks?.linkedin ?? null, + sources: [], + reference: + found?.id === undefined || found.id === null ? null : String(found.id), + }, + }; +} + +export const lusha: Provider = { + id: "lusha", + label: "Lusha", + keys: [LUSHA_API_KEY], + enabled: keyed(LUSHA_API_KEY), + find: lushaPerson, +}; diff --git a/apps/agent/agent/lib/tasks.ts b/apps/agent/agent/lib/tasks.ts index 9d8a912c7..5dd3ee556 100644 --- a/apps/agent/agent/lib/tasks.ts +++ b/apps/agent/agent/lib/tasks.ts @@ -42,11 +42,7 @@ export async function claimDue( const onlyMode = "only" in kinds; const claimed = await db.$queryRaw` - UPDATE "agentTask" AS t - SET "leasedUntil" = ${until}, - "startedAt" = COALESCE(t."startedAt", ${now}), - "attempts" = t."attempts" + 1 - FROM ( + WITH due AS MATERIALIZED ( SELECT t2.id FROM "agentTask" AS t2 WHERE t2."finishedAt" IS NULL AND t2."dueAt" <= ${now} @@ -59,7 +55,12 @@ export async function claimDue( ORDER BY t2."priority" DESC, t2."dueAt" ASC LIMIT ${limit} FOR UPDATE SKIP LOCKED - ) AS due + ) + UPDATE "agentTask" AS t + SET "leasedUntil" = ${until}, + "startedAt" = COALESCE(t."startedAt", ${now}), + "attempts" = t."attempts" + 1 + FROM due WHERE t.id = due.id RETURNING t.id, t."contactId", t."companyId", t."dealId", t.kind, t.reason, t.payload, t.budget, t.attempts, t.priority, t."dueAt"; @@ -76,10 +77,7 @@ export async function retireExhausted( const now = new Date(); return db.$queryRaw` - UPDATE "agentTask" AS t - SET "finishedAt" = ${now}, - "outcome" = ${RETIRED_OUTCOME} - WHERE t.id IN ( + WITH exhausted AS MATERIALIZED ( SELECT c.id FROM "agentTask" AS c WHERE c."finishedAt" IS NULL @@ -89,6 +87,11 @@ export async function retireExhausted( LIMIT ${limit} FOR UPDATE SKIP LOCKED ) + UPDATE "agentTask" AS t + SET "finishedAt" = ${now}, + "outcome" = ${RETIRED_OUTCOME} + FROM exhausted + WHERE t.id = exhausted.id RETURNING t.id, t."contactId", t."companyId", t."dealId", t.kind; `; } diff --git a/apps/agent/agent/lib/website.ts b/apps/agent/agent/lib/website.ts new file mode 100644 index 000000000..21243662f --- /dev/null +++ b/apps/agent/agent/lib/website.ts @@ -0,0 +1,312 @@ +import type { + ContactDetails, + DetailSource, + Outcome, + Person, + Phone, + Provider, +} from "./contact-details"; +import { CONTACT_DETAILS } from "./contact-details-config"; +import { domainOf } from "./names"; + +const EMAIL = /[a-z0-9._%+-]+@[a-z0-9.-]+\.[a-z]{2,}/gi; +const MAILTO = /href\s*=\s*["']mailto:([^"'?]+)/gi; +const TEL = /href\s*=\s*["']tel:([^"']+)/gi; +const LINKEDIN_ANCHOR = + /]*href\s*=\s*["']([^"']*linkedin\.com\/in\/[^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi; +const OBFUSCATED_AT = /\s*[([{]\s*at\s*[)\]}]\s*/gi; +const OBFUSCATED_DOT = /\s*[([{]\s*dot\s*[)\]}]\s*/gi; +const DROPPED_BLOCKS = /<(script|style|noscript|svg)\b[^>]*>[\s\S]*?<\/\1>/gi; + +export type WebPage = { + url: string; + html: string; +}; + +type Sighting = { + page: WebPage; + namesPerson: boolean; + emails: string[]; + phones: string[]; + linkedinUrl: string | null; +}; + +export function fold(value: string): string { + return value + .normalize("NFD") + .replace(/[\u0300-\u036f]/g, "") + .toLowerCase(); +} + +function letters(value: string): string { + return fold(value).replace(/[^a-z0-9]/g, ""); +} + +function decodeEntities(html: string): string { + return html + .replace(/&#x([0-9a-f]+);/gi, (_, hex: string) => + String.fromCodePoint(Number.parseInt(hex, 16)), + ) + .replace(/&#(\d+);/g, (_, dec: string) => String.fromCodePoint(Number(dec))) + .replace(/&/g, "&") + .replace(/</g, "<") + .replace(/>/g, ">") + .replace(/"/g, '"') + .replace(/ /g, " "); +} + +export function textOf(html: string): string { + const stripped = html.replace(DROPPED_BLOCKS, " ").replace(/<[^>]+>/g, " "); + return decodeEntities(stripped).replace(/\s+/g, " ").trim(); +} + +function onDomain(email: string, domain: string): boolean { + const host = domainOf(email); + return host === domain || Boolean(host?.endsWith(`.${domain}`)); +} + +export function emailsIn(html: string, domain: string): string[] { + const text = textOf(html) + .replace(OBFUSCATED_AT, "@") + .replace(OBFUSCATED_DOT, "."); + const linked = [...html.matchAll(MAILTO)].map((match) => + decodeEntities(match[1] ?? ""), + ); + const written = text.match(EMAIL) ?? []; + + const found = [...linked, ...written] + .map((email) => email.trim().toLowerCase()) + .filter((email) => onDomain(email, domain)); + + return [...new Set(found)]; +} + +export function phonesIn(html: string): string[] { + const found = [...html.matchAll(TEL)].map((match) => + decodeURIComponent(match[1] ?? "").replace(/[^\d+]/g, ""), + ); + return [...new Set(found.filter((number) => number.length >= 6))]; +} + +function fullNameForms(person: Person): string[] { + const first = fold(person.firstName).trim(); + const last = fold(person.lastName).trim(); + return [`${first} ${last}`, `${last} ${first}`].filter( + (form) => form.trim().length > 0, + ); +} + +export function mentionsPerson(html: string, person: Person): boolean { + const text = fold(textOf(html)).replace(/\s+/g, " "); + return fullNameForms(person).some((form) => text.includes(form)); +} + +export function localPartNamesPerson(local: string, person: Person): boolean { + const first = letters(person.firstName); + const last = letters(person.lastName); + const handle = letters(local); + if (!handle || !last) return false; + + const initial = first.slice(0, 1); + const forms = first + ? [ + `${first}${last}`, + `${last}${first}`, + `${initial}${last}`, + `${last}${initial}`, + ] + : [last]; + + return forms.includes(handle); +} + +export function linkedinFor(html: string, person: Person): string | null { + const forms = fullNameForms(person); + for (const match of html.matchAll(LINKEDIN_ANCHOR)) { + const label = fold(textOf(match[2] ?? "")).replace(/\s+/g, " "); + if (forms.some((form) => label.includes(form))) { + return decodeEntities(match[1] ?? ""); + } + } + return null; +} + +export function robotsAllows(robots: string, path: string): boolean { + let applies = false; + let allowed = true; + + for (const raw of robots.split(/\r?\n/)) { + const line = raw.replace(/#.*$/, "").trim(); + if (!line) continue; + + const colon = line.indexOf(":"); + if (colon < 0) continue; + const field = line.slice(0, colon).trim().toLowerCase(); + const value = line.slice(colon + 1).trim(); + + if (field === "user-agent") { + applies = value === "*"; + continue; + } + if (!applies || field !== "disallow" || !value) continue; + if (path.startsWith(value)) allowed = false; + } + + return allowed; +} + +export async function fetchPage(url: URL): Promise> { + const controller = new AbortController(); + const timer = setTimeout( + () => controller.abort(), + CONTACT_DETAILS.website.timeoutMs, + ); + + try { + const response = await fetch(url, { + headers: { + "user-agent": CONTACT_DETAILS.website.userAgent, + accept: "text/html", + }, + redirect: "follow", + signal: controller.signal, + }); + if (!response.ok) return { ok: false, reason: `HTTP ${response.status}` }; + + const type = response.headers.get("content-type") ?? ""; + if (!type.includes("text/html") && !type.includes("text/plain")) { + return { ok: false, reason: `Not a page: ${type || "no content type"}` }; + } + + const html = (await response.text()).slice( + 0, + CONTACT_DETAILS.website.maxBytes, + ); + return { ok: true, data: { url: url.toString(), html } }; + } catch (error) { + const aborted = error instanceof Error && error.name === "AbortError"; + return { + ok: false, + reason: aborted + ? `Timed out after ${CONTACT_DETAILS.website.timeoutMs}ms.` + : error instanceof Error + ? error.message + : String(error), + }; + } finally { + clearTimeout(timer); + } +} + +async function readRobots(base: URL): Promise { + const robots = await fetchPage(new URL("/robots.txt", base)); + return robots.ok ? robots.data.html : ""; +} + +export async function readSite(person: Person): Promise { + const base = new URL(`https://${person.domain.trim().toLowerCase()}`); + const robots = await readRobots(base); + + const paths = CONTACT_DETAILS.website.paths + .filter((path) => robotsAllows(robots, path)) + .slice(0, CONTACT_DETAILS.website.maxPages); + + const pages = await Promise.all( + paths.map((path) => fetchPage(new URL(path, base))), + ); + + return pages.flatMap((page) => (page.ok ? [page.data] : [])); +} + +function sightingOf(page: WebPage, person: Person): Sighting { + return { + page, + namesPerson: mentionsPerson(page.html, person), + emails: emailsIn(page.html, person.domain.trim().toLowerCase()), + phones: phonesIn(page.html), + linkedinUrl: linkedinFor(page.html, person), + }; +} + +function sourceOf(page: WebPage, seenOn: string): DetailSource { + return { url: page.url, domain: new URL(page.url).hostname, seenOn }; +} + +function phonesOf(sightings: Sighting[]): Phone[] { + const numbers = new Set(sightings.flatMap((s) => s.phones)); + return [...numbers].map((number) => ({ number, type: "main" })); +} + +export function detailsFrom(pages: WebPage[], person: Person): ContactDetails { + const seenOn = new Date().toISOString().slice(0, 10); + const sightings = pages.map((page) => sightingOf(page, person)); + const named = sightings.filter((s) => s.namesPerson); + const linkedinUrl = sightings.find((s) => s.linkedinUrl)?.linkedinUrl ?? null; + + const emailPages = new Map(); + for (const sighting of sightings) { + for (const email of sighting.emails) { + const local = email.slice(0, email.lastIndexOf("@")); + const surnameOnly = letters(local) === letters(person.lastName); + const matches = + localPartNamesPerson(local, person) || + (surnameOnly && sighting.namesPerson); + if (!matches) continue; + emailPages.set(email, [...(emailPages.get(email) ?? []), sighting]); + } + } + + const [email, where] = [...emailPages.entries()][0] ?? [null, []]; + if (email) { + return { + provider: "website", + email, + confidence: CONTACT_DETAILS.website.confidence.named, + phones: phonesOf(where.length > 0 ? where : named), + title: null, + linkedinUrl, + sources: where + .slice(0, CONTACT_DETAILS.maxSources) + .map((s) => sourceOf(s.page, seenOn)), + reference: null, + }; + } + + const phones = phonesOf(named.length > 0 ? sightings : []); + return { + provider: "website", + email: null, + confidence: + phones.length > 0 ? CONTACT_DETAILS.website.confidence.phoneOnly : 0, + phones, + title: null, + linkedinUrl, + sources: named + .slice(0, CONTACT_DETAILS.maxSources) + .map((s) => sourceOf(s.page, seenOn)), + reference: null, + }; +} + +export async function findOnWebsite( + person: Person, +): Promise> { + if (!person.lastName.trim()) { + return { ok: false, reason: "No last name to look for." }; + } + + const pages = await readSite(person); + if (pages.length === 0) { + return { ok: false, reason: `No readable page on ${person.domain}.` }; + } + + return { ok: true, data: detailsFrom(pages, person) }; +} + +export const website: Provider = { + id: "website", + label: "Company website", + keys: [], + enabled: () => true, + find: findOnWebsite, +}; diff --git a/apps/agent/agent/lib/zoominfo.ts b/apps/agent/agent/lib/zoominfo.ts new file mode 100644 index 000000000..dd63e1bbc --- /dev/null +++ b/apps/agent/agent/lib/zoominfo.ts @@ -0,0 +1,150 @@ +import { z } from "zod"; +import { + type ContactDetails, + fetchJson, + keyed, + type Outcome, + type Person, + type Phone, + type Provider, +} from "./contact-details"; +import { CONTACT_DETAILS } from "./contact-details-config"; + +export const ZOOMINFO_USERNAME = "ZOOMINFO_USERNAME"; +export const ZOOMINFO_PASSWORD = "ZOOMINFO_PASSWORD"; + +const authenticated = z.object({ jwt: z.string().trim().min(1) }); + +const hit = z.object({ + id: z.union([z.string(), z.number()]).nullable().optional(), + email: z.string().trim().email().nullable().optional(), + phone: z.string().nullable().optional(), + directPhone: z.string().nullable().optional(), + mobilePhone: z.string().nullable().optional(), + jobTitle: z.string().nullable().optional(), +}); + +const enriched = z.object({ + success: z.boolean().optional(), + data: z + .object({ + result: z + .array(z.object({ data: z.array(hit).nullable().optional() })) + .nullable() + .optional(), + }) + .nullable() + .optional(), +}); + +let session: { username: string; jwt: string; expiresAt: number } | null = null; + +async function token(): Promise> { + const username = process.env[ZOOMINFO_USERNAME]?.trim(); + const password = process.env[ZOOMINFO_PASSWORD]?.trim(); + if (!username || !password) { + return { + ok: false, + reason: `No ${ZOOMINFO_USERNAME} and ${ZOOMINFO_PASSWORD}.`, + }; + } + + if (session?.username === username && session.expiresAt > Date.now()) { + return { ok: true, data: session.jwt }; + } + + const response = await fetchJson( + new URL(`${CONTACT_DETAILS.zoominfo.baseUrl}/authenticate`), + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ username, password }), + }, + authenticated, + "ZoomInfo", + ); + if (!response.ok) return response; + + session = { + username, + jwt: response.data.jwt, + expiresAt: Date.now() + CONTACT_DETAILS.zoominfo.tokenTtlMs, + }; + return { ok: true, data: session.jwt }; +} + +export function forgetZoomInfoSession(): void { + session = null; +} + +export async function zoominfoEnrich( + person: Person, +): Promise> { + const jwt = await token(); + if (!jwt.ok) return jwt; + + const response = await fetchJson( + new URL(`${CONTACT_DETAILS.zoominfo.baseUrl}/enrich/contact`), + { + method: "POST", + headers: { + "content-type": "application/json", + authorization: `Bearer ${jwt.data}`, + }, + body: JSON.stringify({ + matchPersonInput: [ + { + firstName: person.firstName.trim(), + lastName: person.lastName.trim(), + companyName: person.companyName ?? undefined, + companyWebsite: person.domain.trim().toLowerCase(), + }, + ], + outputFields: [ + "id", + "email", + "phone", + "directPhone", + "mobilePhone", + "jobTitle", + ], + }), + }, + enriched, + "ZoomInfo", + ); + if (!response.ok) return response; + + const found = response.data.data?.result?.[0]?.data?.[0] ?? null; + const phones: Phone[] = []; + if (found?.directPhone) + phones.push({ number: found.directPhone, type: "direct" }); + if (found?.mobilePhone) + phones.push({ number: found.mobilePhone, type: "mobile" }); + if (found?.phone) phones.push({ number: found.phone, type: "work" }); + + return { + ok: true, + data: { + provider: "zoominfo", + email: found?.email ?? null, + confidence: found?.email + ? CONTACT_DETAILS.zoominfo.confidence.matched + : 0, + phones, + title: found?.jobTitle ?? null, + linkedinUrl: null, + sources: [], + reference: + found?.id === undefined || found?.id === null ? null : String(found.id), + }, + }; +} + +export const zoominfo: Provider = { + id: "zoominfo", + label: "ZoomInfo", + keys: [ZOOMINFO_USERNAME, ZOOMINFO_PASSWORD], + enabled: keyed(ZOOMINFO_USERNAME, ZOOMINFO_PASSWORD), + find: zoominfoEnrich, +}; diff --git a/apps/agent/agent/schedules/dispatch.ts b/apps/agent/agent/schedules/dispatch.ts index 1aa080fe6..ea4cc60fb 100644 --- a/apps/agent/agent/schedules/dispatch.ts +++ b/apps/agent/agent/schedules/dispatch.ts @@ -10,7 +10,7 @@ import { brief, drainAll, taskAuth } from "../lib/dispatch"; import { reconcileStaleTasks } from "../lib/stale-tasks"; export default defineSchedule({ - cron: "* * * * *", + cron: "0 6 * * *", async run({ receive, waitUntil, appAuth }) { waitUntil( Promise.all([ diff --git a/apps/agent/agent/skills/gleif-mna-sourcing/SKILL.md b/apps/agent/agent/skills/gleif-mna-sourcing/SKILL.md new file mode 100644 index 000000000..6626edd8a --- /dev/null +++ b/apps/agent/agent/skills/gleif-mna-sourcing/SKILL.md @@ -0,0 +1,87 @@ +--- +description: Use when asked to source M&A targets, list a group's subsidiaries by country, build a cross-border target list, or find the executives of those targets — the GLEIF register plus web research, under the legal rules that keep the list usable. +--- + +# GLEIF M&A sourcing + +The GLEIF register is the public list of legal entities that hold a Legal +Entity Identifier, with the parent–subsidiary relationships they report. It +is the deterministic half of sourcing: who owns what, where. The other half, +who runs each target, is web research and it is where the rules below apply. + +## 1. Fix the scenario before searching + +A scenario is a parent place and a child place: "US parents with subsidiaries +in Asia", "EU groups with a Mexican entity". Ask for both when the request +gives only one. `UE` and `ASIE` are known regions; anything else is ISO codes +separated by commas. + +## 2. Find the parents + +- A named group: `gleif_search_entities` with the name and its country, then + keep the ACTIVE entity that is the group head. `gleif_get_entity` on a + candidate shows its direct parent; a group head has none. +- A whole scenario with no names: ask the rep for a list of parents, or take + the companies already in the CRM (`search_crm`) that sit in the parent + place. GLEIF search is by name, not by country alone. + +## 3. List the targets + +For each parent, `gleif_list_subsidiaries` with `childCountries` set to the +child place. Every row returned is a target: LEI, legal name, country, city. +A local-language legal name usually comes with `alternativeNames`; use the +Latin one when you search the web for the entity, and show both in the list. + +- Rank parents by how many matching subsidiaries they have. More entities in + the child place means more to buy, and more to talk to. +- GLEIF relationships are self-reported. A group with zero children in the + register is not proof it has none. Say so rather than dropping it silently. +- `totalDirectChildren` is the parent's whole footprint; `matched` is the + slice in the child place. Report both. + +## 4. Find who runs each target + +This is not deterministic and it is where a list becomes unusable if you cut +corners. The rules: + +- **Never fetch linkedin.com.** A profile URL comes from a search engine + snippet, which is public. Use `research_person`, `find_contact_socials` and + `resolve_linkedin_profile`; they already respect this. +- **Never invent a LinkedIn URL.** If no snippet shows it, the answer is + "not found". A plausible namesake is worse than a blank. +- **One source per line.** Every executive you name carries the domain you + saw it on and the date. `record_fact` with `web.cited-claim` or + `search.cites-profile` is how that lands on a contact. +- **Subsidiaries rarely have a CEO.** Look for the local title: managing + director, country head, general manager, president director. Several + people per entity is normal. When the entity has no leader of its own, + give the group leader and say it is the group's. +- **Expect blanks.** Ten to twenty percent of group leaders and far more + local ones have no public profile. Do not fill the gap. + +Contact details come only from `find_contact_details`, never from a page you +read yourself. It asks the configured providers (Hunter, Apollo, Lusha, +Dropcontact, ZoomInfo) in order, then the target's own website — contact, team +and legal pages, under its `robots.txt` — and writes the answer, its confidence +and its source to the contact's timeline. When it returns nothing, stop at +name, title and public profile. + +## 5. Deliver + +Write the list as a table: parent, LEI, target legal name, country, city, +leader, title, profile URL, source. Then the counts: parents scanned, +targets found, leaders found, leaders without a public profile. + +When the rep wants targets in the CRM, `add_company` each one with its LEI +and the GLEIF record as the source (`https://search.gleif.org/#/record/`). +The tool returns the existing company when there is one, so a target already +in the CRM is never duplicated. The company's brand and profile enrichment +queue up on their own; the leaders you found go on as contacts through the +usual `record_fact` path, with their sources. + +## Quality checks before handing over + +- Every LEI has 20 characters and appears once. +- Every target belongs to the child place asked for. +- Every profile URL was seen in a snippet and matches the named person. +- Every blank is reported as a blank, not as a guess. diff --git a/apps/agent/agent/skills/sec-us-research/SKILL.md b/apps/agent/agent/skills/sec-us-research/SKILL.md new file mode 100644 index 000000000..c8ff5a3fd --- /dev/null +++ b/apps/agent/agent/skills/sec-us-research/SKILL.md @@ -0,0 +1,77 @@ +--- +description: Use when asked about a US public company, its SEC filings, who runs it and what they are paid, who its large shareholders are, or to find and import US listed targets — SEC EDGAR through the edgar service, with a filing URL on every line. +--- + +# SEC EDGAR research + +Every US public company files with the SEC, and every filing is public and +dated. The `sec_*` tools read them through the edgar service. They are free, +they need no key, and each answer carries the filing URL: that URL is the +source you cite, and a claim without one does not go in the CRM. + +## 1. Identify the company + +`sec_search_companies` with the name, the ticker or the CIK, then +`sec_get_company` on the match. The CIK is the key for everything after. A +name search can return a fund or a subsidiary with a similar name; check the +SIC description and the state before going on. If the tools answer +"unavailable", the service is not configured here: say so and stop. + +## 2. Read the company + +- `sec_get_company` gives the profile: legal name, tickers, SIC and industry, + state of incorporation, fiscal year end, business address, former names. +- `sec_list_filings` with a form for the documents that matter: `10-K` for + the annual report, `8-K` for events, `DEF 14A` for governance and pay. +- `sec_search_filings` across all companies when the question is "who + mentions X": a product, a customer, a competitor, a technology. + +## 3. Who owns it + +`sec_list_owners` lists holders of 5% or more from Schedule 13D and 13G. +13D is an active holder with intent; 13G is passive. Holdings are as of the +filing date. A holder below 5% never files, so absence is not zero. + +## 4. Who runs it, and what they are paid + +`sec_get_proxy` reads the latest DEF 14A: the named executives with titles +and the summary compensation table, the CEO's pay and pay actually paid with +the NEO average over the last years, pay versus performance, the holders the +proxy lists, the proposals, the CEO pay ratio. `sec_list_insiders` names +officers and directors with their titles from Forms 3/4/5 and shows recent +buys, sells and grants. `sec_compare_compensation` puts several tickers side +by side. + +Titles in the proxy are the company's own words; keep them. A name written +"Mr. Cook" in the pay-versus-performance table is the same person as "Tim +Cook" in the compensation table; use the full name. + +## 5. Deliver + +Write the answer as a table where it fits: company, CIK, ticker, SIC, state, +CEO, total pay, pay actually paid, TSR, 5%+ holders with percent. Under it, +the filing each figure came from, with its date. Blanks stay blanks. + +## 6. Import into the CRM + +When the rep wants the company in the CRM, `add_company` with the name, the +website when the profile has one, `countryCode` US, the state as +`stateCode`, the city, the `cik`, the first `ticker` and the `sic`. The source +is the EDGAR page `sec_get_company` returns as `sourceUrl`. The tool returns +the existing company when there is one. + +Executives go on with `add_contact` on that company: first name, last name, +the title as the proxy states it, and the proxy's filing URL as the source. +Then `record_fact` for the title with `web.cited-claim` and the same URL. A +5%+ holder is an institution, not a person: name it in the write-up, do not +add it as a contact. + +## Rules + +- **The filing URL is the source.** No filing, no line. +- **Never fetch linkedin.com** and never invent a profile URL; the people + pipeline finds profiles its own way after `add_contact`. +- **Numbers are the filing's numbers.** No conversion, no rounding beyond + what you show, the fiscal year end next to every figure. +- **A parse gap is a blank.** When the proxy carries no readable table, say + the figure is not machine-readable in that filing. diff --git a/apps/agent/agent/tools/add_company.ts b/apps/agent/agent/tools/add_company.ts new file mode 100644 index 000000000..5f370db28 --- /dev/null +++ b/apps/agent/agent/tools/add_company.ts @@ -0,0 +1,84 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; +import { createCompany } from "../lib/companies"; + +export default defineTool({ + description: + "Add a company to the CRM with the source it came from. Use it for a sourcing target or any company a rep asks for that search_crm cannot find. A company that already exists, by domain or by name in the same country, is returned rather than duplicated. The source is written to the company's timeline, a company.created event fires, and the brand and profile enrichment queue up on their own. A LEI, a CIK, a ticker and a SIC code are kept as custom fields. Free.", + inputSchema: z.object({ + name: z + .string() + .trim() + .min(1) + .max(200) + .describe("The legal or trading name."), + website: z + .string() + .trim() + .optional() + .describe("Their site or domain when you know it. 'siemens.com'."), + countryCode: z + .string() + .trim() + .length(2) + .optional() + .describe("ISO 3166-1 alpha-2 code. 'JP'."), + country: z + .string() + .trim() + .optional() + .describe("Country name, for display."), + city: z.string().trim().optional(), + lei: z + .string() + .trim() + .length(20) + .optional() + .describe("The Legal Entity Identifier, when it came from GLEIF."), + cik: z + .string() + .trim() + .regex(/^\d{1,10}$/) + .optional() + .describe( + "The SEC Central Index Key, when it came from EDGAR. '320193'.", + ), + ticker: z + .string() + .trim() + .min(1) + .max(6) + .optional() + .describe("The stock ticker of a listed company. 'AAPL'."), + sic: z + .string() + .trim() + .regex(/^\d{4}$/) + .optional() + .describe("The four-digit SIC code from the SEC profile. '3571'."), + stateCode: z + .string() + .trim() + .length(2) + .optional() + .describe("US state of the business address. 'CA'."), + source: z + .object({ + label: z + .string() + .trim() + .min(1) + .max(80) + .describe("Where this came from. 'GLEIF register', 'their website'."), + url: z + .string() + .trim() + .url() + .describe("The page a rep can open to check."), + }) + .describe("Every record the agent creates names its source."), + }), + async execute(input) { + return createCompany(input); + }, +}); diff --git a/apps/agent/agent/tools/add_contact.ts b/apps/agent/agent/tools/add_contact.ts new file mode 100644 index 000000000..ed1b7a1cc --- /dev/null +++ b/apps/agent/agent/tools/add_contact.ts @@ -0,0 +1,35 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; +import { createContact } from "../lib/contacts"; + +export default defineTool({ + description: + "Add a person to the CRM as a contact of an existing company, with the source that named them. Returns the existing contact when the email or the name already exists on that company, so nothing is duplicated. The people pipeline then identifies and researches them on its own. Use it for an executive or a director named in a public document; use record_fact afterwards for each fact with its evidence.", + inputSchema: z.object({ + firstName: z.string().trim().min(1).describe("'Tim'."), + lastName: z.string().trim().min(1).optional().describe("'Cook'."), + title: z + .string() + .trim() + .min(1) + .optional() + .describe( + "Their title as the source states it. 'Chief Executive Officer'.", + ), + email: z.string().trim().email().optional(), + companyId: z + .string() + .trim() + .min(1) + .describe("The CRM id of their company."), + source: z + .object({ + label: z.string().trim().min(1).describe("'SEC DEF 14A'."), + url: z.string().trim().url().describe("The document that names them."), + }) + .describe("Every record the agent creates names its source."), + }), + async execute(input) { + return createContact(input); + }, +}); diff --git a/apps/agent/agent/tools/find_contact_details.ts b/apps/agent/agent/tools/find_contact_details.ts new file mode 100644 index 000000000..392934865 --- /dev/null +++ b/apps/agent/agent/tools/find_contact_details.ts @@ -0,0 +1,166 @@ +import { ActivityType, db } from "@crm/db"; +import { defineTool } from "eve/tools"; +import { z } from "zod"; +import { + type ContactDetails, + configuredProviders, + lookupContactDetails, +} from "../lib/contact-details"; +import { CONTACT_DETAILS_PROVIDERS } from "../lib/contact-details-providers"; +import { focusOn, spend } from "../lib/focus"; +import { domainOf } from "../lib/names"; + +function describeSources(details: ContactDetails): string { + if (details.sources.length > 0) { + return `Seen on: ${details.sources.map((s) => s.url).join(", ")}`; + } + return details.reference + ? `Attested by ${details.provider} (record ${details.reference}).` + : `Attested by ${details.provider}, which gave no public page.`; +} + +export default defineTool({ + description: + "Find a contact's work email and phone from their name and their employer's domain: the configured contact-data providers first (Hunter, Apollo, Lusha, Dropcontact, ZoomInfo), then the employer's own website (contact, team and legal pages), stopping at the first confident answer. Fills the email and phone only where the record has none, and always writes the candidate, its confidence and its source to the contact's timeline. A website read costs nothing; a provider call costs one unit.", + inputSchema: z.object({ + contactId: z.string(), + }), + async execute({ contactId }) { + const providers = configuredProviders(CONTACT_DETAILS_PROVIDERS); + const metered = providers.some((provider) => provider.keys.length > 0); + + focusOn({ contactId }); + + const contact = await db.contact.findUnique({ + where: { id: contactId }, + select: { + id: true, + firstName: true, + lastName: true, + email: true, + phone: true, + company: { + select: { id: true, name: true, domain: true, ownerId: true }, + }, + }, + }); + + if (!contact) return { ok: false as const, reason: "No such contact." }; + if (!contact.lastName) { + return { + ok: false as const, + reason: "The contact has no last name yet. Identify them first.", + }; + } + + const domain = + contact.company?.domain ?? + (contact.email ? domainOf(contact.email) : null); + if (!domain) { + return { + ok: false as const, + reason: + "No employer domain to search. Set the company's website first.", + }; + } + + if (metered) { + const charge = spend(1); + if (!charge.ok) return { ok: false as const, reason: charge.reason }; + } + + const lookup = await lookupContactDetails( + { + firstName: contact.firstName, + lastName: contact.lastName, + domain, + companyName: contact.company?.name ?? null, + }, + CONTACT_DETAILS_PROVIDERS, + ); + + if (lookup.outcome === "unconfigured") { + return { ok: false as const, reason: "No provider is configured." }; + } + if (lookup.outcome === "none") { + return { + ok: true as const, + email: null, + phones: [], + tried: lookup.tried, + reasons: lookup.reasons, + note: "No provider has an address it trusts for this person, and the employer's website does not name them. Leave the email blank rather than guessing a pattern.", + }; + } + + const { details } = lookup; + const author = + contact.company?.ownerId ?? + (await db.user.findFirst({ select: { id: true } }))?.id ?? + null; + + const phone = details.phones[0]?.number ?? null; + const filledEmail = Boolean(details.email) && !contact.email; + const filledPhone = Boolean(phone) && !contact.phone; + + if (filledEmail || filledPhone) { + await db.contact.update({ + where: { id: contact.id }, + data: { + email: filledEmail ? details.email : undefined, + phone: filledPhone ? phone : undefined, + }, + }); + } + + if (author) { + await db.activity.create({ + data: { + type: ActivityType.ENRICHMENT, + subject: filledEmail + ? "Contact details found" + : "Contact details candidate", + body: [ + details.email + ? `${details.email} (confidence ${details.confidence}).` + : null, + phone ? `Phone ${phone}.` : null, + describeSources(details), + details.email && !filledEmail + ? `The record keeps ${contact.email}.` + : null, + ] + .filter(Boolean) + .join(" "), + occurredAt: new Date(), + contactId: contact.id, + companyId: contact.company?.id ?? null, + createdById: author, + meta: { + source: details.provider, + confidence: details.confidence, + reference: details.reference, + sources: details.sources.map((s) => s.url), + tried: lookup.tried, + agent: "people-research", + }, + }, + select: { id: true }, + }); + } + + return { + ok: true as const, + provider: details.provider, + email: details.email, + confidence: details.confidence, + phones: details.phones, + title: details.title, + linkedinUrl: details.linkedinUrl, + sources: details.sources, + filledEmail, + filledPhone, + tried: lookup.tried, + }; + }, +}); diff --git a/apps/agent/agent/tools/gleif_get_entity.ts b/apps/agent/agent/tools/gleif_get_entity.ts new file mode 100644 index 000000000..b38d392ac --- /dev/null +++ b/apps/agent/agent/tools/gleif_get_entity.ts @@ -0,0 +1,33 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; +import { directParent, getEntity } from "../lib/gleif"; + +export default defineTool({ + description: + "Read one legal entity from the public GLEIF register by LEI, with its direct parent when one is registered. Free, no key. Use it to confirm a company's legal name, country and status, or to climb from a subsidiary to the group above it.", + inputSchema: z.object({ + lei: z.string().trim().length(20), + }), + async execute({ lei }) { + const [entity, parent] = await Promise.all([ + getEntity(lei), + directParent(lei), + ]); + + if (!entity.ok) return { found: false as const, reason: entity.reason }; + if (!entity.data) { + return { found: false as const, reason: "No entity holds this LEI." }; + } + + return { + found: true as const, + entity: entity.data, + directParent: parent.ok ? parent.data : null, + note: parent.ok + ? parent.data + ? undefined + : "No direct parent is registered. Either it is a group head, or the relationship was never reported." + : `The parent lookup failed: ${parent.reason}`, + }; + }, +}); diff --git a/apps/agent/agent/tools/gleif_list_subsidiaries.ts b/apps/agent/agent/tools/gleif_list_subsidiaries.ts new file mode 100644 index 000000000..43db4a4d8 --- /dev/null +++ b/apps/agent/agent/tools/gleif_list_subsidiaries.ts @@ -0,0 +1,48 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; +import { directChildren, resolveCountries } from "../lib/gleif"; + +export default defineTool({ + description: + "List the direct subsidiaries a legal entity consolidates, from the public GLEIF relationship register, optionally keeping only those in given countries or regions. Returns each subsidiary with its LEI, legal name, country and city. Free, no key. This is the M&A sourcing step: a parent in one country with subsidiaries in another is a cross-border target.", + inputSchema: z.object({ + lei: z + .string() + .trim() + .length(20) + .describe("The parent's LEI, from gleif_search_entities."), + childCountries: z + .string() + .trim() + .optional() + .describe( + "Keep subsidiaries in these places only. A region name ('UE', 'ASIE') or ISO codes separated by commas ('US,CA'). Empty keeps all.", + ), + }), + async execute({ lei, childCountries }) { + const countries = resolveCountries(childCountries); + const result = await directChildren(lei, { countries }); + + if (!result.ok) { + return { + parent: lei, + subsidiaries: [], + matched: 0, + reason: result.reason, + }; + } + + return { + parent: result.data.parent, + countries, + totalDirectChildren: result.data.total, + matched: result.data.children.length, + subsidiaries: result.data.children, + note: result.data.truncated + ? "The parent has more direct subsidiaries than were read. The counts are a floor." + : result.data.total === 0 + ? "GLEIF records no direct subsidiary for this entity. Relationships are self-reported, so a group can exist without any." + : undefined, + }; + }, +}); diff --git a/apps/agent/agent/tools/gleif_search_entities.ts b/apps/agent/agent/tools/gleif_search_entities.ts new file mode 100644 index 000000000..4f6ecba17 --- /dev/null +++ b/apps/agent/agent/tools/gleif_search_entities.ts @@ -0,0 +1,54 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; +import { ENTITY_CATEGORIES, searchEntities } from "../lib/gleif"; +import { GLEIF } from "../lib/gleif-config"; + +export default defineTool({ + description: + "Find legal entities in the public GLEIF register by name, optionally within one country. Returns each match with its LEI, legal name, country, city and status. Free, no key. Use it to identify a group parent before listing its subsidiaries with gleif_list_subsidiaries, or to check a company's legal identity.", + inputSchema: z.object({ + name: z + .string() + .trim() + .min(2) + .describe("Part of the legal name. 'Renault', 'Siemens Energy'."), + country: z + .string() + .trim() + .length(2) + .optional() + .describe("ISO 3166-1 alpha-2 code of the legal address. 'FR', 'US'."), + category: z + .enum([...ENTITY_CATEGORIES, "ANY"]) + .default("GENERAL") + .describe( + "GLEIF entity category. GENERAL is an operating company or holding; FUND, BRANCH and the others are rarely M&A targets. ANY removes the filter.", + ), + activeOnly: z + .boolean() + .default(true) + .describe("Only entities whose GLEIF status is ACTIVE."), + limit: z + .number() + .int() + .min(1) + .max(GLEIF.search.maxLimit) + .default(GLEIF.search.defaultLimit), + }), + async execute(input) { + const result = await searchEntities(input); + + if (!result.ok) return { found: 0, entities: [], reason: result.reason }; + + return { + found: result.data.total, + entities: result.data.entities, + note: + result.data.total === 0 + ? "Nothing in GLEIF matches. Only entities that hold an LEI are listed; try a shorter name or drop the country." + : result.data.total > result.data.entities.length + ? `Showing ${result.data.entities.length} of ${result.data.total}. Narrow the name or add a country.` + : undefined, + }; + }, +}); diff --git a/apps/agent/agent/tools/sec_compare_compensation.ts b/apps/agent/agent/tools/sec_compare_compensation.ts new file mode 100644 index 000000000..e107e0d6c --- /dev/null +++ b/apps/agent/agent/tools/sec_compare_compensation.ts @@ -0,0 +1,31 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; +import { unavailable } from "../lib/capabilities"; +import { compareCompensation, edgarEnabled } from "../lib/edgar"; +import { EDGAR } from "../lib/edgar-config"; + +export default defineTool({ + description: + "Compare CEO pay across several US public companies from their latest proxy statements: one row per company and fiscal year with the CEO's name, total pay, pay actually paid, total shareholder return and net income. Free, through the edgar service. A company with no readable proxy comes back with a reason instead of numbers.", + inputSchema: z.object({ + tickers: z + .array(z.string().trim().min(1).max(6)) + .min(1) + .max(EDGAR.compensation.maxTickers) + .describe("['AAPL', 'MSFT', 'NVDA']."), + years: z + .number() + .int() + .min(1) + .max(EDGAR.compensation.maxYears) + .default(EDGAR.compensation.years), + }), + async execute(input) { + if (!edgarEnabled()) return unavailable(EDGAR.env.url); + + const result = await compareCompensation(input); + if (!result.ok) return { found: 0, rows: [], reason: result.reason }; + + return { found: result.data.rows.length, rows: result.data.rows }; + }, +}); diff --git a/apps/agent/agent/tools/sec_get_company.ts b/apps/agent/agent/tools/sec_get_company.ts new file mode 100644 index 000000000..f63c0b650 --- /dev/null +++ b/apps/agent/agent/tools/sec_get_company.ts @@ -0,0 +1,41 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; +import { unavailable } from "../lib/capabilities"; +import { companyUrl, edgarEnabled, getCompany } from "../lib/edgar"; +import { EDGAR } from "../lib/edgar-config"; + +export default defineTool({ + description: + "Read a US public company's SEC profile by CIK or ticker: legal name, tickers and exchanges, SIC code and industry, state of incorporation, fiscal year end, filer category, business address, former names. Free, through the edgar service. The EDGAR page URL it returns is the source to cite when you add the company.", + inputSchema: z + .object({ + cik: z + .string() + .trim() + .regex(/^\d{1,10}$/) + .optional() + .describe("'320193'."), + ticker: z + .string() + .trim() + .min(1) + .max(6) + .optional() + .describe("'AAPL'. Used when no CIK is given."), + }) + .refine((input) => input.cik || input.ticker, { + message: "Give a CIK or a ticker.", + }), + async execute(input) { + if (!edgarEnabled()) return unavailable(EDGAR.env.url); + + const result = await getCompany(input); + if (!result.ok) return { found: false, reason: result.reason }; + + return { + found: true, + company: result.data, + sourceUrl: companyUrl(result.data.cik), + }; + }, +}); diff --git a/apps/agent/agent/tools/sec_get_proxy.ts b/apps/agent/agent/tools/sec_get_proxy.ts new file mode 100644 index 000000000..b76257db7 --- /dev/null +++ b/apps/agent/agent/tools/sec_get_proxy.ts @@ -0,0 +1,40 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; +import { unavailable } from "../lib/capabilities"; +import { edgarEnabled, getProxy } from "../lib/edgar"; +import { EDGAR } from "../lib/edgar-config"; + +export default defineTool({ + description: + "Read a US public company's latest proxy statement (DEF 14A): the named executives with their titles and pay from the summary compensation table, the CEO's pay and pay actually paid with the NEO average over the last years, pay versus performance (TSR, peer TSR, net income, the company's chosen measure), the 5%+ holders the proxy lists, the voting proposals, the CEO pay ratio and whether an insider trading policy is adopted. Free, through the edgar service. The filing URL is the source for every executive you add as a contact.", + inputSchema: z.object({ + cik: z + .string() + .trim() + .regex(/^\d{1,10}$/) + .describe("'320193'."), + years: z + .number() + .int() + .min(1) + .max(EDGAR.compensation.maxYears) + .default(EDGAR.compensation.years) + .describe("How many fiscal years of pay history to keep."), + }), + async execute(input) { + if (!edgarEnabled()) return unavailable(EDGAR.env.url); + + const result = await getProxy(input); + if (!result.ok) return { found: false, reason: result.reason }; + + return { + found: true, + proxy: result.data, + sourceUrl: result.data.url, + note: + result.data.executives.length === 0 + ? "The proxy carries no machine-readable compensation table. The CEO figures come from the pay-versus-performance disclosure." + : undefined, + }; + }, +}); diff --git a/apps/agent/agent/tools/sec_list_filings.ts b/apps/agent/agent/tools/sec_list_filings.ts new file mode 100644 index 000000000..6c2f77932 --- /dev/null +++ b/apps/agent/agent/tools/sec_list_filings.ts @@ -0,0 +1,50 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; +import { unavailable } from "../lib/capabilities"; +import { edgarEnabled, listFilings } from "../lib/edgar"; +import { EDGAR } from "../lib/edgar-config"; + +const day = z + .string() + .trim() + .regex(/^\d{4}-\d{2}-\d{2}$/); + +export default defineTool({ + description: + "List a US public company's SEC filings, newest first, with an optional form type and date range. Each filing has its accession number, form, filing date, report date, description and EDGAR URL. Free, through the edgar service. Form types worth knowing: 10-K annual report, 10-Q quarterly, 8-K current event, DEF 14A proxy statement, SC 13D and SCHEDULE 13G shareholder disclosures, 4 insider transaction.", + inputSchema: z.object({ + cik: z + .string() + .trim() + .regex(/^\d{1,10}$/) + .describe("'320193'."), + form: z + .string() + .trim() + .min(1) + .optional() + .describe("One form type. '10-K', '8-K', 'DEF 14A'."), + from: day.optional().describe("Earliest filing date. '2024-01-01'."), + to: day.optional().describe("Latest filing date. '2025-12-31'."), + limit: z + .number() + .int() + .min(1) + .max(EDGAR.filings.maxLimit) + .default(EDGAR.filings.defaultLimit), + }), + async execute(input) { + if (!edgarEnabled()) return unavailable(EDGAR.env.url); + + const result = await listFilings(input); + if (!result.ok) return { found: 0, filings: [], reason: result.reason }; + + return { + found: result.data.filings.length, + filings: result.data.filings, + note: result.data.truncated + ? `Showing the newest ${result.data.filings.length}. Narrow the form or the dates for older ones.` + : undefined, + }; + }, +}); diff --git a/apps/agent/agent/tools/sec_list_insiders.ts b/apps/agent/agent/tools/sec_list_insiders.ts new file mode 100644 index 000000000..e8a6a1581 --- /dev/null +++ b/apps/agent/agent/tools/sec_list_insiders.ts @@ -0,0 +1,39 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; +import { unavailable } from "../lib/capabilities"; +import { edgarEnabled, listInsiders } from "../lib/edgar"; +import { EDGAR } from "../lib/edgar-config"; + +export default defineTool({ + description: + "List a US public company's recent insider transactions from Forms 3, 4 and 5: the insider, their title, the form, the filing date, the transaction kind, shares and price. Officers and directors are named here with their titles, which makes it a source for who runs the company. Free, through the edgar service. Each row carries the filing URL to cite.", + inputSchema: z.object({ + cik: z + .string() + .trim() + .regex(/^\d{1,10}$/) + .describe("'320193'."), + limit: z + .number() + .int() + .min(1) + .max(EDGAR.insiders.maxLimit) + .default(EDGAR.insiders.defaultLimit), + }), + async execute(input) { + if (!edgarEnabled()) return unavailable(EDGAR.env.url); + + const result = await listInsiders(input); + if (!result.ok) + return { found: 0, transactions: [], reason: result.reason }; + + return { + found: result.data.transactions.length, + transactions: result.data.transactions, + note: + result.data.transactions.length === 0 + ? "No insider filing in the period read." + : undefined, + }; + }, +}); diff --git a/apps/agent/agent/tools/sec_list_owners.ts b/apps/agent/agent/tools/sec_list_owners.ts new file mode 100644 index 000000000..740da5b5e --- /dev/null +++ b/apps/agent/agent/tools/sec_list_owners.ts @@ -0,0 +1,49 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; +import { unavailable } from "../lib/capabilities"; +import { edgarEnabled, listOwners } from "../lib/edgar"; +import { EDGAR } from "../lib/edgar-config"; + +export default defineTool({ + description: + "List a US public company's beneficial owners of 5% or more from their Schedule 13D and 13G filings: the filer, the form, the filing date, shares held, percent of class, voting power and, on a 13D, the stated purpose. 13D is an active holder with intent to influence; 13G is a passive holder. Free, through the edgar service. Each row carries the filing URL to cite.", + inputSchema: z.object({ + cik: z + .string() + .trim() + .regex(/^\d{1,10}$/) + .describe("'320193'."), + minPercent: z + .number() + .min(0) + .max(100) + .default(EDGAR.owners.minPercent) + .describe("Lowest percent of class to keep."), + form: z + .enum(["13D", "13G", "all"]) + .default("all") + .describe("13D for activists, 13G for passive holders, all for both."), + limit: z + .number() + .int() + .min(1) + .max(EDGAR.owners.maxLimit) + .default(EDGAR.owners.defaultLimit), + }), + async execute(input) { + if (!edgarEnabled()) return unavailable(EDGAR.env.url); + + const result = await listOwners(input); + if (!result.ok) return { found: 0, owners: [], reason: result.reason }; + + return { + found: result.data.owners.length, + owners: result.data.owners, + filingsRead: result.data.filingsRead, + note: + result.data.owners.length === 0 + ? "No 13D or 13G above the threshold in the filings read. Holders below 5% never file one." + : "Holdings are as of each filing date; a holder who sold since may not have filed yet.", + }; + }, +}); diff --git a/apps/agent/agent/tools/sec_search_companies.ts b/apps/agent/agent/tools/sec_search_companies.ts new file mode 100644 index 000000000..f7da41f4f --- /dev/null +++ b/apps/agent/agent/tools/sec_search_companies.ts @@ -0,0 +1,40 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; +import { unavailable } from "../lib/capabilities"; +import { edgarEnabled, searchCompanies } from "../lib/edgar"; +import { EDGAR } from "../lib/edgar-config"; + +export default defineTool({ + description: + "Find US public companies in SEC EDGAR by name, ticker or CIK. Returns each match with its CIK, name, ticker and exchange. Free, through the edgar service. Use it to identify a company before sec_get_company, sec_list_filings, sec_list_owners or sec_get_proxy.", + inputSchema: z.object({ + query: z + .string() + .trim() + .min(1) + .describe( + "A company name, a ticker or a CIK. 'Apple', 'AAPL', '320193'.", + ), + limit: z + .number() + .int() + .min(1) + .max(EDGAR.search.maxLimit) + .default(EDGAR.search.defaultLimit), + }), + async execute(input) { + if (!edgarEnabled()) return unavailable(EDGAR.env.url); + + const result = await searchCompanies(input); + if (!result.ok) return { found: 0, companies: [], reason: result.reason }; + + return { + found: result.data.companies.length, + companies: result.data.companies, + note: + result.data.companies.length === 0 + ? "Nothing in EDGAR matches. Only companies that file with the SEC are listed; try the ticker or a shorter name." + : undefined, + }; + }, +}); diff --git a/apps/agent/agent/tools/sec_search_filings.ts b/apps/agent/agent/tools/sec_search_filings.ts new file mode 100644 index 000000000..66e2beb03 --- /dev/null +++ b/apps/agent/agent/tools/sec_search_filings.ts @@ -0,0 +1,53 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; +import { unavailable } from "../lib/capabilities"; +import { edgarEnabled, searchFilings } from "../lib/edgar"; +import { EDGAR } from "../lib/edgar-config"; + +const day = z + .string() + .trim() + .regex(/^\d{4}-\d{2}-\d{2}$/); + +export default defineTool({ + description: + "Full-text search across SEC filings of every company, with an optional form type and date range. Each hit names the filing company with its CIK. Free, through the edgar service. Use it to find which companies mention a product, a customer, a competitor or an event, or to find every DEF 14A in a period.", + inputSchema: z.object({ + query: z + .string() + .trim() + .min(2) + .describe( + "Words or a quoted phrase. '\"data center\" cooling', 'Ozempic supply'.", + ), + form: z + .string() + .trim() + .min(1) + .optional() + .describe("One form type. '10-K', '8-K', 'DEF 14A'."), + from: day.optional().describe("Earliest filing date. '2025-01-01'."), + to: day.optional().describe("Latest filing date. '2025-12-31'."), + limit: z + .number() + .int() + .min(1) + .max(EDGAR.filings.maxLimit) + .default(EDGAR.filings.defaultLimit), + }), + async execute(input) { + if (!edgarEnabled()) return unavailable(EDGAR.env.url); + + const result = await searchFilings(input); + if (!result.ok) return { found: 0, filings: [], reason: result.reason }; + + return { + found: result.data.total, + filings: result.data.filings, + note: + result.data.total > result.data.filings.length + ? `Showing ${result.data.filings.length} of ${result.data.total}. Add a form type or narrow the dates.` + : undefined, + }; + }, +}); diff --git a/apps/agent/test/capabilities.spec.ts b/apps/agent/test/capabilities.spec.ts index db3dd6cf1..66862ae75 100644 --- a/apps/agent/test/capabilities.spec.ts +++ b/apps/agent/test/capabilities.spec.ts @@ -9,7 +9,17 @@ import { unavailable, } from "../agent/lib/capabilities"; -const KEYS = ["PERPLEXITY_API_KEY", "BLOB_READ_WRITE_TOKEN"] as const; +const KEYS = [ + "PERPLEXITY_API_KEY", + "HUNTER_API_KEY", + "APOLLO_API_KEY", + "LUSHA_API_KEY", + "DROPCONTACT_API_KEY", + "ZOOMINFO_USERNAME", + "ZOOMINFO_PASSWORD", + "EDGAR_URL", + "BLOB_READ_WRITE_TOKEN", +] as const; const saved: Record = {}; diff --git a/apps/agent/test/companies.integration.spec.ts b/apps/agent/test/companies.integration.spec.ts new file mode 100644 index 000000000..655a6f82b --- /dev/null +++ b/apps/agent/test/companies.integration.spec.ts @@ -0,0 +1,185 @@ +import { afterEach, beforeEach, describe, expect, it } from "bun:test"; +import { db } from "@crm/db"; +import { + CIK_FIELD_LABEL, + createCompany, + LEI_FIELD_LABEL, + SIC_FIELD_LABEL, + TICKER_FIELD_LABEL, +} from "../agent/lib/companies"; + +const SUFFIX = "companies-spec"; +const SOURCE = { + label: "GLEIF register", + url: "https://search.gleif.org/#/record/W38RGI023J3WT1HWRP32", +}; + +const USER_ID = "companies-spec-user"; + +async function clear() { + const companies = await db.company.findMany({ + where: { name: { contains: SUFFIX } }, + select: { id: true }, + }); + const ids = companies.map((company) => company.id); + await db.activity.deleteMany({ where: { companyId: { in: ids } } }); + await db.agentTask.deleteMany({ where: { companyId: { in: ids } } }); + await db.fieldValue.deleteMany({ where: { companyId: { in: ids } } }); + await db.company.deleteMany({ where: { id: { in: ids } } }); + await db.fieldDefinition.deleteMany({ + where: { + entity: "COMPANY", + label: { + in: [LEI_FIELD_LABEL, CIK_FIELD_LABEL, "Ticker", SIC_FIELD_LABEL], + }, + }, + }); +} + +beforeEach(async () => { + await clear(); + await db.user.upsert({ + where: { id: USER_ID }, + create: { + id: USER_ID, + name: "Companies Spec", + email: `${USER_ID}@example.test`, + }, + update: {}, + }); +}); + +afterEach(async () => { + await clear(); + await db.user.deleteMany({ where: { id: USER_ID } }); +}); + +describe("createCompany", () => { + it("creates the company, its event, its source note, its LEI and its enrichment", async () => { + const result = await createCompany({ + name: `Siemens ${SUFFIX}`, + website: "https://www.siemens.com/global/", + countryCode: "de", + country: "Germany", + city: "Munich", + lei: "W38RGI023J3WT1HWRP32", + source: SOURCE, + }); + + expect(result.created).toBe(true); + expect(result.domain).toBe("siemens.com"); + + const company = await db.company.findUniqueOrThrow({ + where: { id: result.id }, + select: { countryCode: true, website: true, city: true }, + }); + expect(company.countryCode).toBe("DE"); + expect(company.website).toBe("https://siemens.com"); + expect(company.city).toBe("Munich"); + + const tasks = await db.agentTask.findMany({ + where: { companyId: result.id }, + select: { kind: true, reason: true, payload: true }, + }); + expect(tasks.map((task) => task.kind).sort()).toEqual([ + "agent-event", + "brand", + "company-profile", + ]); + const event = tasks.find((task) => task.kind === "agent-event"); + expect(event?.payload).toMatchObject({ + type: "company.created", + record: { kind: "company", id: result.id }, + }); + + const activity = await db.activity.findFirst({ + where: { companyId: result.id }, + select: { subject: true, body: true, meta: true }, + }); + expect(activity?.subject).toBe("Added from GLEIF register"); + expect(activity?.body).toContain(SOURCE.url); + expect(activity?.meta).toMatchObject({ sourceUrl: SOURCE.url }); + + const lei = await db.fieldValue.findFirst({ + where: { companyId: result.id, field: { label: LEI_FIELD_LABEL } }, + select: { text: true }, + }); + expect(lei?.text).toBe("W38RGI023J3WT1HWRP32"); + }); + + it("returns the existing company instead of a duplicate", async () => { + const first = await createCompany({ + name: `Renault ${SUFFIX}`, + countryCode: "FR", + source: SOURCE, + }); + const second = await createCompany({ + name: `renault ${SUFFIX}`, + countryCode: "fr", + source: SOURCE, + }); + + expect(second.created).toBe(false); + expect(second.id).toBe(first.id); + expect(second.reason).toContain("already"); + + expect( + await db.company.count({ + where: { name: { contains: `Renault ${SUFFIX}` }, archivedAt: null }, + }), + ).toBe(1); + }); + + it("matches on domain before name", async () => { + const first = await createCompany({ + name: `Acme ${SUFFIX}`, + website: "acme-companies-spec.test", + source: SOURCE, + }); + const second = await createCompany({ + name: `Acme Holdings ${SUFFIX}`, + website: "https://www.acme-companies-spec.test/about", + source: SOURCE, + }); + + expect(second.created).toBe(false); + expect(second.id).toBe(first.id); + }); + + it("keeps the SEC identifiers as fields and reads a CIK as a US company", async () => { + const result = await createCompany({ + name: `Apple ${SUFFIX}`, + website: "https://www.apple.com", + city: "Cupertino", + stateCode: "ca", + cik: "0000320193", + ticker: "aapl", + sic: "3571", + source: { + label: "SEC EDGAR", + url: "https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK=320193", + }, + }); + + expect(result.created).toBe(true); + const company = await db.company.findUnique({ where: { id: result.id } }); + expect(company?.countryCode).toBe("US"); + expect(company?.stateCode).toBe("CA"); + + const values = await db.fieldValue.findMany({ + where: { companyId: result.id }, + select: { text: true, field: { select: { label: true } } }, + }); + const byLabel = Object.fromEntries( + values.map((value) => [value.field.label.toUpperCase(), value.text]), + ); + expect(byLabel[CIK_FIELD_LABEL]).toBe("320193"); + expect(byLabel[TICKER_FIELD_LABEL]).toBe("AAPL"); + expect(byLabel[SIC_FIELD_LABEL]).toBe("3571"); + + const activity = await db.activity.findFirst({ + where: { companyId: result.id }, + }); + expect(activity?.body).toContain("CIK 320193."); + }); +}); diff --git a/apps/agent/test/contact-details.spec.ts b/apps/agent/test/contact-details.spec.ts new file mode 100644 index 000000000..a36ee1bf9 --- /dev/null +++ b/apps/agent/test/contact-details.spec.ts @@ -0,0 +1,382 @@ +import { afterEach, beforeEach, describe, expect, it } from "bun:test"; +import { APOLLO_API_KEY, apolloMatch } from "../agent/lib/apollo"; +import { + type ContactDetails, + configuredProviders, + lookupContactDetails, + type Provider, +} from "../agent/lib/contact-details"; +import { CONTACT_DETAILS } from "../agent/lib/contact-details-config"; +import { + DROPCONTACT_API_KEY, + dropcontactEnrich, +} from "../agent/lib/dropcontact"; +import { LUSHA_API_KEY, lushaPerson } from "../agent/lib/lusha"; +import { + forgetZoomInfoSession, + ZOOMINFO_PASSWORD, + ZOOMINFO_USERNAME, + zoominfoEnrich, +} from "../agent/lib/zoominfo"; + +const KEYS = [ + APOLLO_API_KEY, + LUSHA_API_KEY, + DROPCONTACT_API_KEY, + ZOOMINFO_USERNAME, + ZOOMINFO_PASSWORD, +] as const; + +const saved: Record = {}; +const realFetch = globalThis.fetch; +const requests: { url: URL; method: string; body: string }[] = []; + +const ADA = { + firstName: "Ada", + lastName: "Lovelace", + domain: "example.com", + companyName: "Example", +}; + +function replies(answer: (url: URL) => { status?: number; json: string }) { + globalThis.fetch = (async (input: URL | RequestInfo, init?: RequestInit) => { + const url = new URL(String(input instanceof Request ? input.url : input)); + requests.push({ + url, + method: init?.method ?? "GET", + body: String(init?.body ?? ""), + }); + const { status, json } = answer(url); + return new Response(json, { + status: status ?? 200, + headers: { "content-type": "application/json" }, + }); + }) as typeof fetch; +} + +function details(overrides: Partial): ContactDetails { + return { + provider: "hunter", + email: null, + confidence: 0, + phones: [], + title: null, + linkedinUrl: null, + sources: [], + reference: null, + ...overrides, + }; +} + +function fake( + id: Provider["id"], + enabled: boolean, + find: Provider["find"], +): Provider { + return { id, label: id, keys: [], enabled: () => enabled, find }; +} + +beforeEach(() => { + for (const key of KEYS) { + saved[key] = process.env[key]; + process.env[key] = `${key}-test`; + } + forgetZoomInfoSession(); +}); + +afterEach(() => { + globalThis.fetch = realFetch; + requests.length = 0; + for (const key of KEYS) { + if (saved[key] === undefined) delete process.env[key]; + else process.env[key] = saved[key]; + } +}); + +describe("configuredProviders", () => { + it("keeps the configured order and drops what is off", () => { + const all = [ + fake("zoominfo", true, async () => ({ ok: true, data: details({}) })), + fake("hunter", false, async () => ({ ok: true, data: details({}) })), + fake("lusha", true, async () => ({ ok: true, data: details({}) })), + ]; + + expect(configuredProviders(all).map((p) => p.id)).toEqual([ + "lusha", + "zoominfo", + ]); + }); +}); + +describe("lookupContactDetails", () => { + it("says so when nothing is configured", async () => { + expect(await lookupContactDetails(ADA, [])).toEqual({ + outcome: "unconfigured", + }); + }); + + it("stops at the first confident answer and names what it tried", async () => { + const calls: string[] = []; + const all = [ + fake("hunter", true, async () => { + calls.push("hunter"); + return { + ok: true, + data: details({ email: "a@example.com", confidence: 20 }), + }; + }), + fake("apollo", true, async () => { + calls.push("apollo"); + return { ok: false, reason: "HTTP 500" }; + }), + fake("lusha", true, async () => { + calls.push("lusha"); + return { + ok: true, + data: details({ + provider: "lusha", + email: "ada@example.com", + confidence: 85, + }), + }; + }), + fake("zoominfo", true, async () => { + calls.push("zoominfo"); + return { ok: true, data: details({}) }; + }), + ]; + + const result = await lookupContactDetails(ADA, all); + + expect(result.outcome).toBe("found"); + if (result.outcome !== "found") return; + expect(result.details.email).toBe("ada@example.com"); + expect(result.tried).toEqual(["hunter", "apollo", "lusha"]); + expect(calls).not.toContain("zoominfo"); + }); + + it("accepts a phone-only answer when no address is on offer", async () => { + const all = [ + fake("lusha", true, async () => ({ + ok: true, + data: details({ + provider: "lusha", + phones: [{ number: "+33100000000", type: "work" }], + }), + })), + ]; + + const result = await lookupContactDetails(ADA, all); + + expect(result.outcome).toBe("found"); + }); + + it("reports every reason when nobody is confident", async () => { + const all = [ + fake("hunter", true, async () => ({ + ok: true, + data: details({ email: "a@example.com", confidence: 10 }), + })), + fake("apollo", true, async () => ({ ok: false, reason: "HTTP 401" })), + ]; + + const result = await lookupContactDetails(ADA, all); + + expect(result.outcome).toBe("none"); + if (result.outcome !== "none") return; + expect(result.reasons).toHaveLength(2); + expect(result.reasons[0]).toContain( + `below ${CONTACT_DETAILS.minConfidence}`, + ); + expect(result.reasons[1]).toBe("apollo: HTTP 401"); + }); +}); + +describe("apolloMatch", () => { + it("posts the person and reads email status, title and phones", async () => { + replies(() => ({ + json: JSON.stringify({ + person: { + id: "p1", + email: "ada@example.com", + email_status: "verified", + title: "CTO", + linkedin_url: "https://www.linkedin.com/in/ada", + phone_numbers: [ + { sanitized_number: "+33100000000", type: "work_hq" }, + ], + }, + }), + })); + + const result = await apolloMatch(ADA); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.data).toMatchObject({ + provider: "apollo", + email: "ada@example.com", + confidence: CONTACT_DETAILS.apollo.confidence.verified, + title: "CTO", + reference: "p1", + phones: [{ number: "+33100000000", type: "work_hq" }], + }); + expect(requests[0]?.method).toBe("POST"); + expect(requests[0]?.body).toContain('"domain":"example.com"'); + }); + + it("is a blank, not a failure, when nobody matches", async () => { + replies(() => ({ json: JSON.stringify({ person: null }) })); + + const result = await apolloMatch(ADA); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.data.email).toBeNull(); + expect(result.data.confidence).toBe(0); + }); +}); + +describe("lushaPerson", () => { + it("prefers the work address and keeps the phones", async () => { + replies(() => ({ + json: JSON.stringify({ + data: { + id: 42, + jobTitle: "CTO", + emailAddresses: [ + { email: "ada@personal.test", emailType: "personal" }, + { email: "ada@example.com", emailType: "work" }, + ], + phoneNumbers: [{ number: "+33100000000", phoneType: "direct" }], + socialLinks: { linkedin: "https://www.linkedin.com/in/ada" }, + }, + }), + })); + + const result = await lushaPerson(ADA); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.data).toMatchObject({ + provider: "lusha", + email: "ada@example.com", + confidence: CONTACT_DETAILS.lusha.confidence.work, + reference: "42", + phones: [{ number: "+33100000000", type: "direct" }], + }); + expect(requests[0]?.url.searchParams.get("companyDomain")).toBe( + "example.com", + ); + }); + + it("accepts the wrapped contact shape too", async () => { + replies(() => ({ + json: JSON.stringify({ + data: { contact: { emailAddresses: [{ email: "ada@example.com" }] } }, + }), + })); + + const result = await lushaPerson(ADA); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.data.email).toBe("ada@example.com"); + expect(result.data.confidence).toBe(CONTACT_DETAILS.lusha.confidence.other); + }); +}); + +describe("dropcontactEnrich", () => { + it("submits a batch, then reads the finished row", async () => { + replies((url) => + url.pathname.endsWith("/batch") + ? { json: JSON.stringify({ request_id: "req-1", success: true }) } + : { + json: JSON.stringify({ + success: true, + data: [ + { + email: [ + { + email: "ada@example.com", + qualification: "nominative@pro", + }, + ], + phone: "+33100000000", + job: "CTO", + linkedin: "https://www.linkedin.com/in/ada", + }, + ], + }), + }, + ); + + const result = await dropcontactEnrich(ADA); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.data).toMatchObject({ + provider: "dropcontact", + email: "ada@example.com", + confidence: CONTACT_DETAILS.dropcontact.confidence.nominative, + reference: "req-1", + phones: [{ number: "+33100000000", type: "work" }], + }); + expect(requests.map((r) => r.method)).toEqual(["POST", "GET"]); + expect(requests[1]?.url.pathname).toBe("/batch/req-1"); + }); +}); + +describe("zoominfoEnrich", () => { + it("authenticates once, then enriches with the bearer token", async () => { + replies((url) => + url.pathname === "/authenticate" + ? { json: JSON.stringify({ jwt: "jwt-1" }) } + : { + json: JSON.stringify({ + success: true, + data: { + result: [ + { + data: [ + { + id: 7, + email: "ada@example.com", + directPhone: "+33100000000", + jobTitle: "CTO", + }, + ], + }, + ], + }, + }), + }, + ); + + const first = await zoominfoEnrich(ADA); + const second = await zoominfoEnrich(ADA); + + expect(first.ok).toBe(true); + if (!first.ok) return; + expect(first.data).toMatchObject({ + provider: "zoominfo", + email: "ada@example.com", + confidence: CONTACT_DETAILS.zoominfo.confidence.matched, + reference: "7", + phones: [{ number: "+33100000000", type: "direct" }], + }); + expect(second.ok).toBe(true); + expect( + requests.filter((r) => r.url.pathname === "/authenticate"), + ).toHaveLength(1); + }); + + it("needs both halves of the credential", async () => { + delete process.env[ZOOMINFO_PASSWORD]; + + const result = await zoominfoEnrich(ADA); + + expect(result.ok).toBe(false); + expect(requests).toHaveLength(0); + }); +}); diff --git a/apps/agent/test/contacts.integration.spec.ts b/apps/agent/test/contacts.integration.spec.ts new file mode 100644 index 000000000..adc334063 --- /dev/null +++ b/apps/agent/test/contacts.integration.spec.ts @@ -0,0 +1,132 @@ +import { afterEach, beforeEach, describe, expect, it } from "bun:test"; +import { db } from "@crm/db"; +import { createContact } from "../agent/lib/contacts"; + +const SUFFIX = "contacts-spec"; +const USER_ID = "contacts-spec-user"; +const SOURCE = { + label: "SEC DEF 14A", + url: "https://www.sec.gov/Archives/edgar/data/320193/0001308179-26-000008-index.html", +}; + +let companyId = ""; + +async function clear() { + const companies = await db.company.findMany({ + where: { name: { contains: SUFFIX } }, + select: { id: true }, + }); + const ids = companies.map((company) => company.id); + const contacts = await db.contact.findMany({ + where: { + OR: [{ companyId: { in: ids } }, { lastName: { contains: SUFFIX } }], + }, + select: { id: true }, + }); + const contactIds = contacts.map((contact) => contact.id); + await db.activity.deleteMany({ + where: { + OR: [{ companyId: { in: ids } }, { contactId: { in: contactIds } }], + }, + }); + await db.agentTask.deleteMany({ + where: { + OR: [{ companyId: { in: ids } }, { contactId: { in: contactIds } }], + }, + }); + await db.contact.deleteMany({ where: { id: { in: contactIds } } }); + await db.company.deleteMany({ where: { id: { in: ids } } }); +} + +beforeEach(async () => { + await clear(); + await db.user.upsert({ + where: { id: USER_ID }, + create: { + id: USER_ID, + name: "Contacts Spec", + email: `${USER_ID}@example.test`, + }, + update: {}, + }); + const company = await db.company.create({ + data: { name: `Apple ${SUFFIX}`, ownerId: USER_ID }, + select: { id: true }, + }); + companyId = company.id; +}); + +afterEach(async () => { + await clear(); + await db.user.deleteMany({ where: { id: USER_ID } }); +}); + +describe("createContact", () => { + it("creates the contact, its event and its source note", async () => { + const result = await createContact({ + firstName: "Tim", + lastName: `Cook ${SUFFIX}`, + title: "Chief Executive Officer", + companyId, + source: SOURCE, + }); + + expect(result.created).toBe(true); + expect(result.companyId).toBe(companyId); + + const contact = await db.contact.findUnique({ where: { id: result.id } }); + expect(contact?.title).toBe("Chief Executive Officer"); + expect(contact?.ownerId).toBe(USER_ID); + + const task = await db.agentTask.findFirst({ + where: { contactId: result.id, kind: "agent-event" }, + }); + expect(task?.reason).toBe("contact.created"); + + const activity = await db.activity.findFirst({ + where: { contactId: result.id }, + }); + expect(activity?.subject).toBe("Added from SEC DEF 14A"); + expect(activity?.body).toContain(SOURCE.url); + }); + + it("returns the existing contact by email, then by name on the same company", async () => { + const first = await createContact({ + firstName: "Kate", + lastName: `Adams ${SUFFIX}`, + email: `Kate.Adams.${SUFFIX}@example.test`, + companyId, + source: SOURCE, + }); + const byEmail = await createContact({ + firstName: "Katherine", + lastName: `Adams ${SUFFIX}`, + email: `kate.adams.${SUFFIX}@example.test`, + companyId, + source: SOURCE, + }); + const byName = await createContact({ + firstName: "kate", + lastName: `adams ${SUFFIX}`, + companyId, + source: SOURCE, + }); + + expect(byEmail.created).toBe(false); + expect(byEmail.id).toBe(first.id); + expect(byName.created).toBe(false); + expect(byName.id).toBe(first.id); + expect(await db.contact.count({ where: { companyId } })).toBe(1); + }); + + it("refuses a company that does not exist", async () => { + const result = await createContact({ + firstName: "Nobody", + lastName: SUFFIX, + companyId: "missing-company", + source: SOURCE, + }); + expect(result.created).toBe(false); + expect(result.reason).toContain("No such company"); + }); +}); diff --git a/apps/agent/test/edgar.spec.ts b/apps/agent/test/edgar.spec.ts new file mode 100644 index 000000000..c648b3441 --- /dev/null +++ b/apps/agent/test/edgar.spec.ts @@ -0,0 +1,247 @@ +import { afterEach, beforeEach, describe, expect, it } from "bun:test"; +import { + companyUrl, + compareCompensation, + edgarEnabled, + getCompany, + getProxy, + health, + listFilings, + listOwners, + searchCompanies, +} from "../agent/lib/edgar"; +import { EDGAR } from "../agent/lib/edgar-config"; + +const realFetch = globalThis.fetch; +const savedUrl = process.env[EDGAR.env.url]; +const savedSecret = process.env[EDGAR.env.secret]; +const requests: { url: URL; headers: Record }[] = []; + +function replies(answer: (url: URL) => { status?: number; json: string }) { + globalThis.fetch = (async (input: URL | RequestInfo, init?: RequestInit) => { + const url = new URL(String(input instanceof Request ? input.url : input)); + const headers: Record = {}; + for (const [key, value] of Object.entries(init?.headers ?? {})) { + headers[key] = String(value); + } + requests.push({ url, headers }); + const { status, json } = answer(url); + return new Response(json, { + status: status ?? 200, + headers: { "content-type": "application/json" }, + }); + }) as typeof fetch; +} + +const COMPANY = { + cik: "320193", + name: "Apple Inc.", + tickers: ["AAPL"], + exchanges: ["Nasdaq"], + sic: "3571", + sicDescription: "Electronic Computers", + stateOfIncorporation: "CA", + fiscalYearEnd: "0926", + category: "Large accelerated filer", + businessAddress: { + street: "ONE APPLE PARK WAY", + city: "CUPERTINO", + state: "CA", + zip: "95014", + }, + website: null, + formerNames: ["APPLE COMPUTER INC"], + url: "https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK=320193", +}; + +beforeEach(() => { + requests.length = 0; + process.env[EDGAR.env.url] = "http://edgar.test:2100/"; + process.env[EDGAR.env.secret] = "s3cret"; +}); + +afterEach(() => { + globalThis.fetch = realFetch; + if (savedUrl === undefined) delete process.env[EDGAR.env.url]; + else process.env[EDGAR.env.url] = savedUrl; + if (savedSecret === undefined) delete process.env[EDGAR.env.secret]; + else process.env[EDGAR.env.secret] = savedSecret; +}); + +describe("the edgar client", () => { + it("is off without a URL, before any request", async () => { + delete process.env[EDGAR.env.url]; + expect(edgarEnabled()).toBe(false); + const result = await searchCompanies({ query: "apple" }); + expect(result.ok).toBe(false); + expect(requests).toHaveLength(0); + }); + + it("sends the bearer secret and trims the trailing slash", async () => { + replies(() => ({ json: JSON.stringify({ companies: [] }) })); + await searchCompanies({ query: "apple", limit: 3 }); + expect(requests[0]?.url.toString()).toBe( + "http://edgar.test:2100/companies/search?q=apple&limit=3", + ); + expect(requests[0]?.headers.authorization).toBe("Bearer s3cret"); + }); + + it("sends no authorization header without a secret", async () => { + delete process.env[EDGAR.env.secret]; + replies(() => ({ + json: JSON.stringify({ + ok: true, + version: "0.1.0", + edgartools: "5.56.0", + identitySet: true, + }), + })); + const result = await health(); + expect(result.ok).toBe(true); + expect(requests[0]?.headers.authorization).toBeUndefined(); + }); + + it("parses a company and strips leading zeros from the CIK it asks for", async () => { + replies(() => ({ json: JSON.stringify(COMPANY) })); + const result = await getCompany({ cik: "0000320193" }); + expect(requests[0]?.url.pathname).toBe("/companies/320193"); + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.data.sicDescription).toBe("Electronic Computers"); + expect(companyUrl(result.data.cik)).toBe(COMPANY.url); + }); + + it("reads a 404 on a record as the service's reason", async () => { + replies(() => ({ + status: 404, + json: JSON.stringify({ reason: "No SEC filer matches ZZZZ." }), + })); + const result = await getCompany({ ticker: "zzzz" }); + expect(result).toEqual({ ok: false, reason: "No SEC filer matches ZZZZ." }); + }); + + it("reads a 404 on a list as an empty list", async () => { + replies(() => ({ status: 404, json: "" })); + const result = await listFilings({ cik: "320193", form: "10-K" }); + expect(result).toEqual({ + ok: true, + data: { filings: [], truncated: false }, + }); + }); + + it("reports another failure as HTTP status", async () => { + replies(() => ({ + status: 502, + json: JSON.stringify({ reason: "SEC down" }), + })); + const result = await listOwners({ cik: "320193" }); + expect(result).toEqual({ ok: false, reason: "HTTP 502" }); + }); + + it("refuses a shape the service does not promise", async () => { + replies(() => ({ + json: JSON.stringify({ owners: [{ filer: "X" }], filingsRead: 1 }), + })); + const result = await listOwners({ cik: "320193" }); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.reason).toContain("Unreadable EDGAR service response"); + }); + + it("passes the proxy through with its executives", async () => { + const proxy = { + accession: "0001308179-26-000008", + filedAt: "2026-01-08", + url: "https://www.sec.gov/Archives/edgar/data/320193/0001308179-26-000008-index.html", + peo: { + name: "Mr. Cook", + totalComp: 74294811, + actuallyPaidComp: 108423733, + }, + neoAverage: { totalComp: 23812358, actuallyPaidComp: 34125743 }, + compensationByYear: [ + { + fiscalYearEnd: "2025-09-27", + peoTotalComp: 74294811, + peoActuallyPaidComp: 108423733, + neoAverageTotalComp: 23812358, + neoAverageActuallyPaidComp: 34125743, + }, + ], + payVsPerformance: [ + { + fiscalYearEnd: "2025-09-27", + peoActuallyPaidComp: 108423733, + neoAverageActuallyPaidComp: 34125743, + tsr: 233.88, + peerTsr: 279.51, + netIncome: 112010000000, + selectedMeasureValue: 416161000000, + }, + ], + executives: [ + { + name: "Tim Cook", + title: "CEO", + year: 2025, + salary: 3000000, + bonus: null, + stockAwards: 57535293, + optionAwards: null, + nonEquityIncentive: 12000000, + otherCompensation: 1759518, + total: 74294811, + }, + ], + holders: [ + { name: "The Vanguard Group", percentOfClass: 9.63, shares: null }, + ], + proposals: [ + { + number: 1, + description: "Election of Directors", + type: "director_election", + }, + ], + performanceMeasures: ["Net Sales", "Operating Income"], + selectedMeasureName: "Net Sales", + ceoPayRatio: { ceo: 74294811, medianEmployee: 139483, ratio: 533 }, + insiderTradingPolicyAdopted: true, + }; + replies(() => ({ json: JSON.stringify(proxy) })); + const result = await getProxy({ cik: "320193", years: 1 }); + expect(requests[0]?.url.search).toBe("?years=1"); + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.data.executives[0]?.name).toBe("Tim Cook"); + expect(result.data.ceoPayRatio?.ratio).toBe(533); + }); + + it("joins tickers upper-cased for the comparison", async () => { + replies(() => ({ json: JSON.stringify({ rows: [] }) })); + await compareCompensation({ tickers: ["aapl", " msft "], years: 2 }); + expect(requests[0]?.url.search).toBe("?tickers=AAPL%2CMSFT&years=2"); + }); + + it("reports a timeout as a reason", async () => { + globalThis.fetch = ((_: URL | RequestInfo, init?: RequestInit) => + new Promise((_, reject) => { + init?.signal?.addEventListener("abort", () => { + const error = new Error("aborted"); + error.name = "AbortError"; + reject(error); + }); + })) as typeof fetch; + const original = EDGAR.timeoutMs; + const result = await Promise.race([ + searchCompanies({ query: "apple" }), + new Promise<{ ok: false; reason: string }>((resolve) => + setTimeout( + () => resolve({ ok: false, reason: "test timed out first" }), + original + 500, + ), + ), + ]); + expect(result.ok).toBe(false); + }, 60_000); +}); diff --git a/apps/agent/test/fixtures/gleif.json b/apps/agent/test/fixtures/gleif.json new file mode 100644 index 000000000..0d2e32695 --- /dev/null +++ b/apps/agent/test/fixtures/gleif.json @@ -0,0 +1,130 @@ +{ + "search": { + "data": [ + { + "id": "969500F7JLTX36OUI695", + "attributes": { + "entity": { + "legalName": { + "name": "RENAULT" + }, + "legalAddress": { + "country": "FR", + "city": "BOULOGNE-BILLANCOURT" + }, + "status": "ACTIVE", + "category": "GENERAL", + "jurisdiction": "FR" + }, + "registration": { + "status": "ISSUED" + } + } + }, + { + "id": "969500HC1NCZMYE1TU11", + "attributes": { + "entity": { + "legalName": { + "name": "RENAULT INVEST" + }, + "legalAddress": { + "country": "FR", + "city": "SURESNES" + }, + "status": "ACTIVE", + "category": "GENERAL", + "jurisdiction": "FR" + }, + "registration": { + "status": "ISSUED" + } + } + } + ], + "meta": { + "pagination": { + "currentPage": 1, + "perPage": 3, + "from": 1, + "to": 3, + "total": 22, + "lastPage": 8 + } + } + }, + "children": { + "data": [ + { + "id": "549300WSRTFGXLOGXV17", + "attributes": { + "entity": { + "legalName": { + "name": "AUTOFIN" + }, + "legalAddress": { + "country": "BE", + "city": "BRUSSELS" + }, + "status": "ACTIVE", + "category": "GENERAL", + "jurisdiction": "BE" + }, + "registration": { + "status": "LAPSED" + } + } + }, + { + "id": "5493004RF65W82IYO343", + "attributes": { + "entity": { + "legalName": { + "name": "RCI FINANCIAL SERVICES" + }, + "legalAddress": { + "country": "BE", + "city": "BRUSSELS" + }, + "status": "ACTIVE", + "category": "GENERAL", + "jurisdiction": "BE" + }, + "registration": { + "status": "LAPSED" + } + } + }, + { + "id": "969500V06Q2Q3ELCWD59", + "attributes": { + "entity": { + "legalName": { + "name": "RENAULT SAS" + }, + "legalAddress": { + "country": "FR", + "city": "BOULOGNE-BILLANCOURT" + }, + "status": "ACTIVE", + "category": "GENERAL", + "jurisdiction": "FR" + }, + "registration": { + "status": "ISSUED" + } + } + } + ], + "meta": { + "pagination": { + "currentPage": 1, + "perPage": 5, + "from": 1, + "to": 3, + "total": 3, + "lastPage": 1 + } + } + } +} diff --git a/apps/agent/test/gleif.spec.ts b/apps/agent/test/gleif.spec.ts new file mode 100644 index 000000000..9bd92581c --- /dev/null +++ b/apps/agent/test/gleif.spec.ts @@ -0,0 +1,218 @@ +import { afterEach, describe, expect, it } from "bun:test"; +import { + directChildren, + directParent, + getEntity, + resolveCountries, + searchEntities, +} from "../agent/lib/gleif"; +import { GLEIF } from "../agent/lib/gleif-config"; +import fixtures from "./fixtures/gleif.json"; + +const realFetch = globalThis.fetch; + +const requested: string[] = []; + +function replies(reply: (url: URL) => { status?: number; body: unknown }) { + globalThis.fetch = (async (input: URL | RequestInfo) => { + const url = new URL(String(input instanceof Request ? input.url : input)); + requested.push(url.toString()); + const { status, body } = reply(url); + return new Response(JSON.stringify(body), { + status: status ?? 200, + headers: { "content-type": "application/vnd.api+json" }, + }); + }) as typeof fetch; +} + +afterEach(() => { + globalThis.fetch = realFetch; + requested.length = 0; +}); + +describe("resolveCountries", () => { + it("expands a region name", () => { + expect(resolveCountries("UE")).toEqual([...GLEIF.regions.UE]); + }); + + it("accepts ISO codes, mixed case, and drops junk", () => { + expect(resolveCountries(" us, ca ,Asie,xyz")).toEqual([ + "US", + "CA", + ...GLEIF.regions.ASIE, + ]); + }); + + it("is empty for nothing", () => { + expect(resolveCountries(undefined)).toEqual([]); + expect(resolveCountries("")).toEqual([]); + }); +}); + +describe("searchEntities", () => { + it("parses a search page into entities and sends the filters", async () => { + replies(() => ({ body: fixtures.search })); + + const result = await searchEntities({ name: "Renault", country: "fr" }); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.data.total).toBe(22); + expect(result.data.entities[0]).toEqual({ + lei: "969500F7JLTX36OUI695", + name: "RENAULT", + alternativeNames: [], + country: "FR", + city: "BOULOGNE-BILLANCOURT", + status: "ACTIVE", + registrationStatus: "ISSUED", + category: "GENERAL", + jurisdiction: "FR", + }); + + const url = new URL(requested[0] ?? ""); + expect(url.searchParams.get("filter[entity.legalName]")).toBe("Renault"); + expect(url.searchParams.get("filter[entity.legalAddress.country]")).toBe( + "FR", + ); + expect(url.searchParams.get("filter[entity.status]")).toBe("ACTIVE"); + expect(url.searchParams.get("filter[entity.category]")).toBe("GENERAL"); + }); + + it("drops the category filter on ANY", async () => { + replies(() => ({ body: fixtures.search })); + + await searchEntities({ name: "Renault", category: "ANY" }); + + const url = new URL(requested[0] ?? ""); + expect(url.searchParams.has("filter[entity.category]")).toBe(false); + }); + + it("reports an HTTP failure as a reason, never a throw", async () => { + replies(() => ({ status: 503, body: {} })); + + const result = await searchEntities({ name: "Renault" }); + + expect(result).toEqual({ ok: false, reason: "HTTP 503" }); + }); + + it("refuses a response that is not the GLEIF shape", async () => { + replies(() => ({ body: { data: [{ id: "short" }] } })); + + const result = await searchEntities({ name: "Renault" }); + + expect(result.ok).toBe(false); + }); +}); + +describe("directChildren", () => { + it("keeps only the wanted countries and reports the whole footprint", async () => { + replies(() => ({ body: fixtures.children })); + + const result = await directChildren("969500F7JLTX36OUI695", { + countries: ["BE"], + }); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.data.total).toBe(3); + expect(result.data.children.map((child) => child.name)).toEqual([ + "AUTOFIN", + "RCI FINANCIAL SERVICES", + ]); + expect(result.data.truncated).toBe(false); + }); + + it("walks every page and flags a cut-off at the page cap", async () => { + const child = fixtures.children.data[0]; + replies((url) => { + const number = Number(url.searchParams.get("page[number]")); + return { + body: { + data: [{ ...child, id: `${number}`.padStart(20, "0") }], + meta: { + pagination: { + currentPage: number, + lastPage: GLEIF.api.maxPages + 1, + total: GLEIF.api.maxPages + 1, + }, + }, + }, + }; + }); + + const result = await directChildren("969500F7JLTX36OUI695"); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(requested).toHaveLength(GLEIF.api.maxPages); + expect(result.data.children).toHaveLength(GLEIF.api.maxPages); + expect(result.data.truncated).toBe(true); + }); +}); + +describe("getEntity and directParent", () => { + it("returns null for a 404 instead of failing", async () => { + replies(() => ({ status: 404, body: { errors: [] } })); + + expect(await getEntity("969500F7JLTX36OUI695")).toEqual({ + ok: true, + data: null, + }); + expect(await directParent("969500F7JLTX36OUI695")).toEqual({ + ok: true, + data: null, + }); + }); + + it("collects the other names a local-language entity carries", async () => { + const local = fixtures.search.data[1]; + replies(() => ({ + body: { + data: { + ...local, + attributes: { + ...local.attributes, + entity: { + ...local.attributes.entity, + legalName: { name: "ドットマティクス株式会社" }, + otherNames: [ + { + type: "ALTERNATIVE_LANGUAGE_LEGAL_NAME", + name: "Dotmatics K.K.", + }, + { + type: "PREVIOUS_LEGAL_NAME", + name: "ドットマティクス株式会社", + }, + ], + transliteratedOtherNames: [ + { + type: "AUTO_ASCII_TRANSLITERATED_LEGAL_NAME", + name: "Dotmatics K.K.", + }, + ], + }, + }, + }, + }, + })); + + const result = await getEntity("969500HC1NCZMYE1TU11"); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.data?.alternativeNames).toEqual(["Dotmatics K.K."]); + }); + + it("parses a single record", async () => { + replies(() => ({ body: { data: fixtures.search.data[1] } })); + + const result = await getEntity("969500hc1nczmye1tu11"); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.data?.name).toBe("RENAULT INVEST"); + expect(requested[0]).toContain("/lei-records/969500HC1NCZMYE1TU11"); + }); +}); diff --git a/apps/agent/test/hunter.spec.ts b/apps/agent/test/hunter.spec.ts new file mode 100644 index 000000000..5646f200f --- /dev/null +++ b/apps/agent/test/hunter.spec.ts @@ -0,0 +1,146 @@ +import { afterEach, beforeEach, describe, expect, it } from "bun:test"; +import { + findWorkEmail, + HUNTER_API_KEY, + hunterEnabled, + verifyEmail, +} from "../agent/lib/hunter"; + +const realFetch = globalThis.fetch; +const savedKey = process.env[HUNTER_API_KEY]; +const requested: string[] = []; + +const ADA = { + firstName: "Ada", + lastName: "Lovelace", + domain: "Example.com", + companyName: "Example", +}; + +function replies(status: number, json: string) { + globalThis.fetch = (async (input: URL | RequestInfo) => { + requested.push(String(input instanceof Request ? input.url : input)); + return new Response(json, { + status, + headers: { "content-type": "application/json" }, + }); + }) as typeof fetch; +} + +beforeEach(() => { + process.env[HUNTER_API_KEY] = "hunter-test-key"; +}); + +afterEach(() => { + globalThis.fetch = realFetch; + requested.length = 0; + if (savedKey === undefined) delete process.env[HUNTER_API_KEY]; + else process.env[HUNTER_API_KEY] = savedKey; +}); + +describe("hunterEnabled", () => { + it("is off without a key, and blank counts as unset", () => { + delete process.env[HUNTER_API_KEY]; + expect(hunterEnabled()).toBe(false); + process.env[HUNTER_API_KEY] = " "; + expect(hunterEnabled()).toBe(false); + }); +}); + +describe("findWorkEmail", () => { + it("refuses without a key, before any request", async () => { + delete process.env[HUNTER_API_KEY]; + + const result = await findWorkEmail(ADA); + + expect(result.ok).toBe(false); + expect(requested).toHaveLength(0); + }); + + it("returns the address, its confidence and the pages it was seen on", async () => { + replies( + 200, + JSON.stringify({ + data: { + email: "ada.lovelace@example.com", + score: 92, + position: "CTO", + sources: [ + { + domain: "example.com", + uri: "https://example.com/team", + extracted_on: "2026-05-01", + }, + { + domain: "news.test", + uri: "https://news.test/ada", + extracted_on: null, + }, + ], + }, + }), + ); + + const result = await findWorkEmail(ADA); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.data.provider).toBe("hunter"); + expect(result.data.email).toBe("ada.lovelace@example.com"); + expect(result.data.confidence).toBe(92); + expect(result.data.title).toBe("CTO"); + expect(result.data.sources.map((s) => s.url)).toEqual([ + "https://example.com/team", + "https://news.test/ada", + ]); + + const url = new URL(requested[0] ?? ""); + expect(url.searchParams.get("domain")).toBe("example.com"); + expect(url.searchParams.get("first_name")).toBe("Ada"); + expect(url.searchParams.get("api_key")).toBe("hunter-test-key"); + }); + + it("keeps a null address as a null, not a failure", async () => { + replies( + 200, + JSON.stringify({ data: { email: null, score: null, sources: [] } }), + ); + + const result = await findWorkEmail(ADA); + + expect(result).toEqual({ + ok: true, + data: { + provider: "hunter", + email: null, + confidence: 0, + phones: [], + title: null, + linkedinUrl: null, + sources: [], + reference: null, + }, + }); + }); + + it("reports an HTTP failure as a reason", async () => { + replies(429, JSON.stringify({ errors: [] })); + + const result = await findWorkEmail(ADA); + + expect(result).toEqual({ ok: false, reason: "HTTP 429" }); + }); +}); + +describe("verifyEmail", () => { + it("reads the status and score", async () => { + replies(200, JSON.stringify({ data: { status: "valid", score: 97 } })); + + const result = await verifyEmail("Ada.Lovelace@Example.com"); + + expect(result).toEqual({ ok: true, data: { status: "valid", score: 97 } }); + expect(new URL(requested[0] ?? "").searchParams.get("email")).toBe( + "ada.lovelace@example.com", + ); + }); +}); diff --git a/apps/agent/test/tasks.integration.spec.ts b/apps/agent/test/tasks.integration.spec.ts index 3100f0c1f..cfdb065f4 100644 --- a/apps/agent/test/tasks.integration.spec.ts +++ b/apps/agent/test/tasks.integration.spec.ts @@ -190,7 +190,24 @@ describe("retireExhausted", () => { } for (let pass = 0; pass < 3; pass++) { - expect((await retireExhausted(2)).length).toBeLessThanOrEqual(2); + const retired = await retireExhausted(2); + if (retired.length > 2) { + const rows = await db.agentTask.findMany({ + where: { id: { in: retired.map((task) => task.id) } }, + select: { + id: true, + kind: true, + attempts: true, + leasedUntil: true, + finishedAt: true, + outcome: true, + }, + }); + throw new Error( + `retireExhausted(2) returned ${retired.length} rows on pass ${pass}: ${JSON.stringify({ retired, rows, mine })}`, + ); + } + expect(retired.length).toBeLessThanOrEqual(2); } const open = await db.agentTask.count({ diff --git a/apps/agent/test/website.spec.ts b/apps/agent/test/website.spec.ts new file mode 100644 index 000000000..c6038e034 --- /dev/null +++ b/apps/agent/test/website.spec.ts @@ -0,0 +1,233 @@ +import { afterEach, beforeEach, describe, expect, it } from "bun:test"; +import { configuredProviders } from "../agent/lib/contact-details"; +import { CONTACT_DETAILS } from "../agent/lib/contact-details-config"; +import { CONTACT_DETAILS_PROVIDERS } from "../agent/lib/contact-details-providers"; +import { + detailsFrom, + emailsIn, + findOnWebsite, + localPartNamesPerson, + phonesIn, + robotsAllows, + textOf, + website, +} from "../agent/lib/website"; + +const realFetch = globalThis.fetch; +const requested: string[] = []; + +const ADA = { + firstName: "Ada", + lastName: "Lovelace", + domain: "example.com", + companyName: "Example", +}; + +type Reply = { status?: number; type?: string; body: string }; + +function serves(pages: Record) { + globalThis.fetch = (async (input: URL | RequestInfo) => { + const url = new URL(String(input instanceof Request ? input.url : input)); + requested.push(url.pathname); + const reply = pages[url.pathname] ?? { status: 404, body: "" }; + return new Response(reply.body, { + status: reply.status ?? 200, + headers: { "content-type": reply.type ?? "text/html; charset=utf-8" }, + }); + }) as typeof fetch; +} + +beforeEach(() => { + requested.length = 0; +}); + +afterEach(() => { + globalThis.fetch = realFetch; +}); + +describe("the website provider", () => { + it("is always on, needs no key, and comes last", () => { + expect(website.keys).toEqual([]); + expect(website.enabled()).toBe(true); + expect(CONTACT_DETAILS.order.at(-1)).toBe("website"); + expect(configuredProviders(CONTACT_DETAILS_PROVIDERS).at(-1)?.id).toBe( + "website", + ); + }); +}); + +describe("textOf", () => { + it("drops scripts and tags and decodes entities", () => { + const html = + "

Ada Lovelace & co @ work

"; + expect(textOf(html)).toBe("Ada Lovelace & co @ work"); + }); +}); + +describe("emailsIn", () => { + it("reads mailto links, written addresses and obfuscated ones, on the domain only", () => { + const html = ` + write +

Press: press@example.com

+

Sales: sales [at] example [dot] com

+

Other: someone@elsewhere.org

+

Sub: team@mail.example.com

+ `; + expect(emailsIn(html, "example.com")).toEqual([ + "ada.lovelace@example.com", + "press@example.com", + "sales@example.com", + "team@mail.example.com", + ]); + }); +}); + +describe("phonesIn", () => { + it("reads tel links and keeps only digits and the plus", () => { + const html = + 'callagainno'; + expect(phonesIn(html)).toEqual(["+33123456789"]); + }); +}); + +describe("localPartNamesPerson", () => { + it("accepts the usual name forms and refuses the rest", () => { + expect(localPartNamesPerson("ada.lovelace", ADA)).toBe(true); + expect(localPartNamesPerson("alovelace", ADA)).toBe(true); + expect(localPartNamesPerson("lovelace.a", ADA)).toBe(true); + expect(localPartNamesPerson("LovelaceAda", ADA)).toBe(true); + expect(localPartNamesPerson("ada", ADA)).toBe(false); + expect(localPartNamesPerson("lovelace", ADA)).toBe(false); + expect(localPartNamesPerson("contact", ADA)).toBe(false); + }); + + it("folds accents", () => { + const person = { ...ADA, firstName: "Éloïse", lastName: "Müller" }; + expect(localPartNamesPerson("eloise.muller", person)).toBe(true); + }); +}); + +describe("robotsAllows", () => { + it("honours the wildcard group only", () => { + const robots = [ + "User-agent: Googlebot", + "Disallow: /team", + "", + "User-agent: *", + "Disallow: /private", + "Disallow: /legal # old", + ].join("\n"); + expect(robotsAllows(robots, "/team")).toBe(true); + expect(robotsAllows(robots, "/private/x")).toBe(false); + expect(robotsAllows(robots, "/legal")).toBe(false); + expect(robotsAllows("", "/anything")).toBe(true); + }); +}); + +describe("detailsFrom", () => { + it("returns the named address with the pages it was seen on and the phones there", () => { + const details = detailsFrom( + [ + { + url: "https://example.com/", + html: 'infox', + }, + { + url: "https://example.com/team", + html: '

Ada Lovelace

mailcallAda Lovelace', + }, + ], + ADA, + ); + + expect(details.provider).toBe("website"); + expect(details.email).toBe("ada.lovelace@example.com"); + expect(details.confidence).toBe(CONTACT_DETAILS.website.confidence.named); + expect(details.sources.map((s) => s.url)).toEqual([ + "https://example.com/team", + ]); + expect(details.phones).toEqual([{ number: "+15550199", type: "main" }]); + expect(details.linkedinUrl).toBe("https://www.linkedin.com/in/ada"); + }); + + it("accepts a surname-only address when the page names the person", () => { + const details = detailsFrom( + [ + { + url: "https://example.com/contact", + html: "

Ada Lovelace, directrice. lovelace@example.com

", + }, + ], + ADA, + ); + expect(details.email).toBe("lovelace@example.com"); + }); + + it("gives the switchboard only when the site names the person", () => { + const pages = [ + { + url: "https://example.com/", + html: 'call', + }, + { + url: "https://example.com/team", + html: "

Ada Lovelace

", + }, + ]; + const named = detailsFrom(pages, ADA); + expect(named.email).toBeNull(); + expect(named.phones).toEqual([{ number: "+15550100", type: "main" }]); + expect(named.confidence).toBe(CONTACT_DETAILS.website.confidence.phoneOnly); + expect(named.sources.map((s) => s.url)).toEqual([ + "https://example.com/team", + ]); + + const stranger = detailsFrom(pages, { ...ADA, lastName: "Byron" }); + expect(stranger.email).toBeNull(); + expect(stranger.phones).toEqual([]); + expect(stranger.confidence).toBe(0); + }); +}); + +describe("findOnWebsite", () => { + it("reads robots.txt, skips what it forbids, and ignores pages that are not HTML", async () => { + serves({ + "/robots.txt": { + type: "text/plain", + body: "User-agent: *\nDisallow: /team\n", + }, + "/": { body: "

Example

" }, + "/contact": { + body: '

Ada Lovelace

m', + }, + "/about": { type: "application/pdf", body: "%PDF" }, + }); + + const result = await findOnWebsite(ADA); + + expect(requested).toContain("/robots.txt"); + expect(requested).toContain("/contact"); + expect(requested).not.toContain("/team"); + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.data.email).toBe("a.lovelace@example.com"); + expect(result.data.sources.map((s) => s.url)).toEqual([ + "https://example.com/contact", + ]); + }); + + it("reports a site with no readable page as a reason, not a throw", async () => { + serves({}); + expect(await findOnWebsite(ADA)).toEqual({ + ok: false, + reason: "No readable page on example.com.", + }); + }); + + it("refuses without a last name, before any request", async () => { + serves({ "/": { body: "

x

" } }); + const result = await findOnWebsite({ ...ADA, lastName: " " }); + expect(result.ok).toBe(false); + expect(requested).toHaveLength(0); + }); +}); diff --git a/apps/agent/turbo.json b/apps/agent/turbo.json index 6432b1236..07b6a0d5d 100644 --- a/apps/agent/turbo.json +++ b/apps/agent/turbo.json @@ -20,7 +20,15 @@ "BLOB_READ_WRITE_TOKEN", "DATABASE_URL", "GITHUB_TOKEN", - "PERPLEXITY_API_KEY" + "HUNTER_API_KEY", + "APOLLO_API_KEY", + "LUSHA_API_KEY", + "DROPCONTACT_API_KEY", + "ZOOMINFO_USERNAME", + "ZOOMINFO_PASSWORD", + "PERPLEXITY_API_KEY", + "EDGAR_URL", + "EDGAR_SECRET" ] }, "dev:headless": { @@ -33,7 +41,15 @@ "BLOB_READ_WRITE_TOKEN", "DATABASE_URL", "GITHUB_TOKEN", - "PERPLEXITY_API_KEY" + "HUNTER_API_KEY", + "APOLLO_API_KEY", + "LUSHA_API_KEY", + "DROPCONTACT_API_KEY", + "ZOOMINFO_USERNAME", + "ZOOMINFO_PASSWORD", + "PERPLEXITY_API_KEY", + "EDGAR_URL", + "EDGAR_SECRET" ] }, /** diff --git a/apps/api/package.json b/apps/api/package.json index 151aec0f8..b3c9816fc 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -11,7 +11,7 @@ "build": "bun build src/main.ts --target=bun --outdir dist --packages=external --sourcemap", "check-types": "tsc --noEmit", "dev": "concurrently -n api,trpc -c blue,magenta \"bun --watch src/main.ts\" \"bun run dev:trpc\"", - "dev:trpc": "nestjs-trpc watch -e src/app.module.ts -r \"**/*.router.ts\" -o src/generated", + "dev:trpc": "bun scripts/trpc-watch.ts", "dev:session": "bun scripts/dev-session.ts", "lint": "biome check .", "postinstall": "node scripts/chmod-trpc-binary.mjs", diff --git a/apps/api/scripts/build-func.mjs b/apps/api/scripts/build-func.mjs index b06f0c017..5c24f67d2 100644 --- a/apps/api/scripts/build-func.mjs +++ b/apps/api/scripts/build-func.mjs @@ -177,7 +177,7 @@ writeFileSync( JSON.stringify({ version: 3, routes: [{ src: "/(.*)", dest: "/api/index" }], - crons: [{ path: "/internal/sync/google", schedule: "*/5 * * * *" }], + crons: JSON.parse(readFileSync(join(apiDir, "vercel.json"), "utf8")).crons, }), ); diff --git a/apps/api/scripts/dev-config.ts b/apps/api/scripts/dev-config.ts new file mode 100644 index 000000000..d0284e817 --- /dev/null +++ b/apps/api/scripts/dev-config.ts @@ -0,0 +1,6 @@ +export const TRPC_WATCH = { + debounceMs: 150, + routerSuffix: ".router.ts", + sourceDir: "src", + generatedDir: "generated", +} as const; diff --git a/apps/api/scripts/trpc-watch.ts b/apps/api/scripts/trpc-watch.ts new file mode 100644 index 000000000..0a914e158 --- /dev/null +++ b/apps/api/scripts/trpc-watch.ts @@ -0,0 +1,44 @@ +import { spawn } from "node:child_process"; +import { watch } from "node:fs"; +import { sep } from "node:path"; +import { TRPC_WATCH } from "./dev-config"; + +let timer: ReturnType | null = null; +let running = false; +let queued = false; + +function generate(): void { + if (running) { + queued = true; + return; + } + running = true; + const child = spawn("bun", ["run", "trpc:generate"], { stdio: "inherit" }); + child.on("exit", () => { + running = false; + if (queued) { + queued = false; + generate(); + } + }); +} + +function schedule(): void { + if (timer) clearTimeout(timer); + timer = setTimeout(generate, TRPC_WATCH.debounceMs); +} + +function isRouterChange(file: string | null): boolean { + if (!file) return false; + if (file.startsWith(`${TRPC_WATCH.generatedDir}${sep}`)) return false; + return file.endsWith(TRPC_WATCH.routerSuffix); +} + +watch(TRPC_WATCH.sourceDir, { recursive: true }, (_event, file) => { + if (isRouterChange(file)) schedule(); +}); + +generate(); +console.log( + `[trpc] watching ${TRPC_WATCH.sourceDir} for *${TRPC_WATCH.routerSuffix} writes`, +); diff --git a/apps/api/src/config/env.validation.ts b/apps/api/src/config/env.validation.ts index 08cb676c2..1261e7599 100644 --- a/apps/api/src/config/env.validation.ts +++ b/apps/api/src/config/env.validation.ts @@ -68,6 +68,10 @@ export class EnvironmentVariables { @IsString() MICROSOFT_TENANT_ID?: string; + @IsOptional() + @IsString() + PASSWORD_SIGN_IN?: string; + @IsOptional() @IsString() SLACK_CLIENT_ID?: string; diff --git a/apps/api/src/sso/sso.contracts.ts b/apps/api/src/sso/sso.contracts.ts index d36d63e5c..afaa0d894 100644 --- a/apps/api/src/sso/sso.contracts.ts +++ b/apps/api/src/sso/sso.contracts.ts @@ -35,6 +35,7 @@ const ssoPublicProviderOutput = z.object({ export const ssoSignInOptionsOutput = z.object({ google: z.boolean(), microsoft: z.boolean(), + password: z.boolean(), providers: z.array(ssoPublicProviderOutput), }); diff --git a/apps/api/src/sso/sso.service.ts b/apps/api/src/sso/sso.service.ts index 340efcb65..b54cbe8e2 100644 --- a/apps/api/src/sso/sso.service.ts +++ b/apps/api/src/sso/sso.service.ts @@ -3,6 +3,7 @@ import { canConfigureSso, isGoogleConfigured, isMicrosoftConfigured, + isPasswordSignInEnabled, ssoCallbackBase, ssoCallbackURL, ssoProviderName, @@ -126,6 +127,7 @@ export class SsoService { return { google: isGoogleConfigured(), microsoft: isMicrosoftConfigured(), + password: isPasswordSignInEnabled(), providers: rows.map((row) => ({ providerId: row.providerId, name: ssoProviderName(row.providerId), diff --git a/apps/api/test/auth.e2e.spec.ts b/apps/api/test/auth.e2e.spec.ts index d5d79131d..950ec5231 100644 --- a/apps/api/test/auth.e2e.spec.ts +++ b/apps/api/test/auth.e2e.spec.ts @@ -67,6 +67,7 @@ describe("Auth (e2e)", () => { expect(response.body.result.data).toEqual({ google: true, microsoft: microsoftConfigured, + password: process.env.PASSWORD_SIGN_IN === "true", providers: [], }); }); diff --git a/apps/api/turbo.json b/apps/api/turbo.json index 04699fe51..5bf076837 100644 --- a/apps/api/turbo.json +++ b/apps/api/turbo.json @@ -31,6 +31,7 @@ "MICROSOFT_CLIENT_ID", "MICROSOFT_CLIENT_SECRET", "MICROSOFT_TENANT_ID", + "PASSWORD_SIGN_IN", "PORT", "REDIS_URL" ] @@ -49,6 +50,7 @@ "MICROSOFT_CLIENT_ID", "MICROSOFT_CLIENT_SECRET", "MICROSOFT_TENANT_ID", + "PASSWORD_SIGN_IN", "PORT", "REDIS_URL" ] diff --git a/apps/api/vercel.json b/apps/api/vercel.json index ffce1ccc0..fc8d8f153 100644 --- a/apps/api/vercel.json +++ b/apps/api/vercel.json @@ -3,7 +3,7 @@ "crons": [ { "path": "/internal/sync/mailboxes", - "schedule": "*/5 * * * *" + "schedule": "0 8 * * *" }, { "path": "/internal/sync/rates", diff --git a/apps/app/app/(app)/[slug]/research/sec/edgar-client.ts b/apps/app/app/(app)/[slug]/research/sec/edgar-client.ts new file mode 100644 index 000000000..da41bc446 --- /dev/null +++ b/apps/app/app/(app)/[slug]/research/sec/edgar-client.ts @@ -0,0 +1,80 @@ +import { z } from "zod"; + +const withReason = z.object({ reason: z.string() }).partial(); +const jsonValue = z.json(); + +type JsonValue = z.infer; + +export type Answer = + | { status: "ok"; data: T } + | { status: "missing"; reason: string } + | { status: "failed"; reason: string }; + +export async function readEdgar( + path: string, + query: Record, + shape: Shape, +): Promise>> { + const url = new URL(`/edgar/${path}`, window.location.origin); + for (const [key, value] of Object.entries(query)) { + if (value !== undefined && value !== "") { + url.searchParams.set(key, String(value)); + } + } + + const response = await fetch(url, { + headers: { accept: "application/json" }, + }); + const body = await bodyOf(response); + const reason = withReason.safeParse(body); + const said = reason.success ? reason.data.reason : undefined; + + if (response.status === 404) { + return { status: "missing", reason: said ?? "Nothing on file." }; + } + if (!response.ok) { + return { + status: "failed", + reason: said ?? `The SEC service answered HTTP ${response.status}.`, + }; + } + + const parsed = shape.safeParse(body); + return parsed.success + ? { status: "ok", data: parsed.data } + : { + status: "failed", + reason: "The SEC service answered in a shape this page does not read.", + }; +} + +async function bodyOf(response: Response): Promise { + try { + return jsonValue.parse(await response.json()); + } catch { + return null; + } +} + +const usd = new Intl.NumberFormat("en-US", { + style: "currency", + currency: "USD", + maximumFractionDigits: 0, +}); + +const compact = new Intl.NumberFormat("en-US", { + notation: "compact", + maximumFractionDigits: 1, +}); + +export function money(value: number | null): string { + return value === null ? "—" : usd.format(value); +} + +export function count(value: number | null): string { + return value === null ? "—" : compact.format(value); +} + +export function percent(value: number | null): string { + return value === null ? "—" : `${value.toFixed(2)}%`; +} diff --git a/apps/app/app/(app)/[slug]/research/sec/page.tsx b/apps/app/app/(app)/[slug]/research/sec/page.tsx new file mode 100644 index 000000000..82abcf56c --- /dev/null +++ b/apps/app/app/(app)/[slug]/research/sec/page.tsx @@ -0,0 +1,46 @@ +import type { Metadata } from "next"; +import { Suspense } from "react"; +import { + PageShell, + PageShellContent, + PageShellDescription, + PageShellHeader, + PageShellHeading, + PageShellLoading, + PageShellTitle, +} from "@/components/page-shell"; +import { edgarConfigured } from "@/lib/edgar"; +import { requireSession } from "@/lib/session"; +import { SecResearch } from "./sec-research"; + +export const metadata: Metadata = { + title: "SEC research", +}; + +export default function SecResearchPage() { + return ( + + + + SEC research + + US public companies from their EDGAR filings: profile, filings, + major shareholders, executives and their pay. + + + + + + }> + + + + + ); +} + +async function Research() { + await requireSession(); + + return ; +} diff --git a/apps/app/app/(app)/[slug]/research/sec/sec-company.tsx b/apps/app/app/(app)/[slug]/research/sec/sec-company.tsx new file mode 100644 index 000000000..30fa39edc --- /dev/null +++ b/apps/app/app/(app)/[slug]/research/sec/sec-company.tsx @@ -0,0 +1,620 @@ +"use client"; + +import { Badge } from "@crm/ui/components/badge"; +import { Button } from "@crm/ui/components/button"; +import { Link } from "@crm/ui/components/link"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@crm/ui/components/select"; +import { SimpleTable, SimpleTableRow } from "@crm/ui/components/simple-table"; +import { Spinner } from "@crm/ui/components/spinner"; +import { StatCard } from "@crm/ui/components/stat-card"; +import { TableCell } from "@crm/ui/components/table"; +import { + Tabs, + TabsContent, + TabsList, + TabsTrigger, +} from "@crm/ui/components/tabs"; +import { + type EdgarCompany, + type EdgarProxy, + edgarCompany, + edgarFilings, + edgarInsiders, + edgarOwners, + edgarProxy, +} from "@crm/validation/edgar"; +import { useMutation, useQuery } from "@tanstack/react-query"; +import { useRouter } from "next/navigation"; +import { useState } from "react"; +import { toast } from "sonner"; +import { useTRPC } from "@/lib/trpc/client"; +import { useWorkspaceUrl } from "@/lib/use-workspace-url"; +import { type Answer, count, money, percent, readEdgar } from "./edgar-client"; + +const FORMS = [ + "all", + "10-K", + "10-Q", + "8-K", + "DEF 14A", + "SCHEDULE 13G", + "SC 13D", + "4", +] as const; +const FILINGS_LIMIT = 25; +const OWNERS_LIMIT = 20; +const INSIDERS_LIMIT = 20; +const PROXY_YEARS = 5; + +function importPrompt(company: EdgarCompany): string { + const ticker = company.tickers[0] ? `, ticker ${company.tickers[0]}` : ""; + return `Import ${company.name} (CIK ${company.cik}${ticker}) from SEC EDGAR into the CRM: the company with its CIK, ticker, SIC and state, the executives named in its latest DEF 14A as contacts with the filing as their source, and a note on its 5%+ shareholders.`; +} + +function Status({ + state, + children, +}: { + state: { isPending: boolean; isError: boolean; data: Answer | undefined }; + children: (data: T) => React.ReactNode; +}) { + if (state.isPending) { + return ( +

+ Reading EDGAR… +

+ ); + } + if (state.isError || !state.data) { + return ( +

The request failed. Try again.

+ ); + } + if (state.data.status === "missing") { + return

{state.data.reason}

; + } + if (state.data.status === "failed") { + return

{state.data.reason}

; + } + return <>{children(state.data.data)}; +} + +export function SecCompany({ + cik, + onBack, +}: { + cik: string; + onBack: () => void; +}) { + const profile = useQuery({ + queryKey: ["edgar", "company", cik], + queryFn: () => readEdgar(`companies/${cik}`, {}, edgarCompany), + staleTime: 60 * 60_000, + }); + + return ( +
+ + + {(company) => ( + <> + + + + Filings + Holders + Proxy & pay + Insiders + + + + + + + + + + + + + + + + )} + +
+ ); +} + +function CompanyHeader({ company }: { company: EdgarCompany }) { + const router = useRouter(); + const workspaceUrl = useWorkspaceUrl(); + const trpc = useTRPC(); + const handOff = useMutation( + trpc.conversations.createBuilder.mutationOptions({ + onSuccess: ({ id }) => router.push(workspaceUrl(`/chat/${id}`)), + onError: (error) => toast.error(error.message), + }), + ); + + const address = company.businessAddress; + const place = [address?.city, address?.state].filter(Boolean).join(", "); + + return ( +
+
+
+

{company.name}

+ {company.tickers.map((ticker) => ( + + {ticker} + + ))} +
+

+ {[ + company.sicDescription + ? `${company.sicDescription} (SIC ${company.sic})` + : null, + place || null, + company.stateOfIncorporation + ? `Incorporated in ${company.stateOfIncorporation}` + : null, + company.fiscalYearEnd + ? `Fiscal year ends ${company.fiscalYearEnd.slice(0, 2)}/${company.fiscalYearEnd.slice(2)}` + : null, + company.category, + ] + .filter(Boolean) + .join(" · ")} +

+

+ + EDGAR record for CIK {company.cik} + +

+
+ +
+ ); +} + +function Filings({ cik }: { cik: string }) { + const [form, setForm] = useState<(typeof FORMS)[number]>("all"); + const filings = useQuery({ + queryKey: ["edgar", "filings", cik, form], + queryFn: () => + readEdgar( + `companies/${cik}/filings`, + { form: form === "all" ? undefined : form, limit: FILINGS_LIMIT }, + edgarFilings, + ), + staleTime: 10 * 60_000, + }); + + return ( +
+ + + {(data) => + data.filings.length === 0 ? ( +

+ No filing of that form. +

+ ) : ( + + {data.filings.map((filing) => ( + + {filing.form} + + {filing.filedAt} + + + {filing.reportDate ?? "—"} + + + + {filing.description ?? filing.accession} + + + + ))} + + ) + } + +
+ ); +} + +function Holders({ cik }: { cik: string }) { + const owners = useQuery({ + queryKey: ["edgar", "owners", cik], + queryFn: () => + readEdgar( + `companies/${cik}/owners`, + { limit: OWNERS_LIMIT }, + edgarOwners, + ), + staleTime: 10 * 60_000, + }); + + return ( + + {(data) => + data.owners.length === 0 ? ( +

+ No Schedule 13D or 13G at 5% or more in the filings read. Holders + below 5% never file one. +

+ ) : ( + + {data.owners.map((owner) => ( + + + + {owner.filer} + + {owner.purpose ? ( + + {owner.purpose} + + ) : null} + + {owner.form} + {owner.filedAt} + + {count(owner.shares)} + + + {percent(owner.percent)} + + + ))} + + ) + } + + ); +} + +function Insiders({ cik }: { cik: string }) { + const insiders = useQuery({ + queryKey: ["edgar", "insiders", cik], + queryFn: () => + readEdgar( + `companies/${cik}/insiders`, + { limit: INSIDERS_LIMIT }, + edgarInsiders, + ), + staleTime: 10 * 60_000, + }); + + return ( + + {(data) => + data.transactions.length === 0 ? ( +

+ No insider filing in the period read. +

+ ) : ( + + {data.transactions.map((row) => ( + + + + {row.insider} + + + {row.title ?? "—"} + {row.form} + {row.filedAt} + {row.kind?.replaceAll("_", " ") ?? "—"} + + {count(row.shares)} + + + ))} + + ) + } + + ); +} + +function ProxyTab({ cik }: { cik: string }) { + const proxy = useQuery({ + queryKey: ["edgar", "proxy", cik], + queryFn: () => + readEdgar(`companies/${cik}/proxy`, { years: PROXY_YEARS }, edgarProxy), + staleTime: 60 * 60_000, + }); + + return {(data) => }; +} + +function ProxyView({ proxy }: { proxy: EdgarProxy }) { + return ( +
+

+ Latest proxy statement, filed {proxy.filedAt}:{" "} + + DEF 14A {proxy.accession} + +

+ +
+ + + + +
+ +
+

Named executives

+ {proxy.executives.length === 0 ? ( +

+ This proxy carries no machine-readable compensation table. +

+ ) : ( + + {proxy.executives.map((executive) => ( + + {executive.name} + {executive.title ?? "—"} + + {executive.year ?? "—"} + + + {money(executive.salary)} + + + {money(executive.stockAwards)} + + + {money(executive.total)} + + + ))} + + )} +
+ +
+

Pay versus performance

+ {proxy.payVsPerformance.length === 0 ? ( +

+ No pay-versus-performance table in this proxy. +

+ ) : ( + + {proxy.payVsPerformance.map((row) => ( + + + {row.fiscalYearEnd ?? "—"} + + + {money(row.peoActuallyPaidComp)} + + + {money(row.neoAverageActuallyPaidComp)} + + + {row.tsr ?? "—"} + + + {row.peerTsr ?? "—"} + + + {money(row.netIncome)} + + + {money(row.selectedMeasureValue)} + + + ))} + + )} +
+ +
+
+

Holders the proxy lists

+ {proxy.holders.length === 0 ? ( +

None listed.

+ ) : ( + + {proxy.holders.map((holder) => ( + + {holder.name} + + {count(holder.shares)} + + + {percent(holder.percentOfClass)} + + + ))} + + )} +
+ +
+

Proposals and governance

+
    + {proxy.proposals.map((proposal) => ( +
  • + {proposal.number ? `${proposal.number}. ` : ""} + {proposal.description} +
    • + ))} +
  • + Insider trading policy adopted:{" "} + {proxy.insiderTradingPolicyAdopted === null + ? "not stated" + : proxy.insiderTradingPolicyAdopted + ? "yes" + : "no"} +
    • + {proxy.performanceMeasures.length > 0 ? ( +
  • + Performance measures: {proxy.performanceMeasures.join(", ")} +
    • + ) : null} +
+
+
+
+ ); +} diff --git a/apps/app/app/(app)/[slug]/research/sec/sec-research.tsx b/apps/app/app/(app)/[slug]/research/sec/sec-research.tsx new file mode 100644 index 000000000..9819d3d72 --- /dev/null +++ b/apps/app/app/(app)/[slug]/research/sec/sec-research.tsx @@ -0,0 +1,159 @@ +"use client"; + +import { Button } from "@crm/ui/components/button"; +import { + Empty, + EmptyDescription, + EmptyHeader, + EmptyTitle, +} from "@crm/ui/components/empty"; +import { Input } from "@crm/ui/components/input"; +import { SimpleTable, SimpleTableRow } from "@crm/ui/components/simple-table"; +import { Spinner } from "@crm/ui/components/spinner"; +import { TableCell } from "@crm/ui/components/table"; +import { edgarCompanySearch } from "@crm/validation/edgar"; +import { useQuery } from "@tanstack/react-query"; +import { useQueryState } from "nuqs"; +import { type FormEvent, useState } from "react"; +import { readEdgar } from "./edgar-client"; +import { SecCompany } from "./sec-company"; + +const SEARCH_LIMIT = 10; + +export function SecResearch({ configured }: { configured: boolean }) { + const [q, setQ] = useQueryState("q", { defaultValue: "" }); + const [cik, setCik] = useQueryState("cik", { defaultValue: "" }); + const [draft, setDraft] = useState(q); + + const search = useQuery({ + queryKey: ["edgar", "search", q], + queryFn: () => + readEdgar( + "companies/search", + { q, limit: SEARCH_LIMIT }, + edgarCompanySearch, + ), + enabled: configured && q.trim().length > 0, + staleTime: 5 * 60_000, + }); + + if (!configured) { + return ( + + + The SEC EDGAR service is not configured + + Set EDGAR_URL and EDGAR_SECRET to a running services/edgar and + restart the app. docs/setup.md and services/edgar/README.md cover + Docker Compose, another machine and Google Colab. + + + + ); + } + + const submit = (event: FormEvent) => { + event.preventDefault(); + void setCik(""); + void setQ(draft.trim()); + }; + + return ( +
+
+ setDraft(event.target.value)} + placeholder="Company name, ticker or CIK — Apple, AAPL, 320193" + aria-label="Search SEC EDGAR" + /> + +
+ + {cik ? ( + void setCik("")} /> + ) : ( + void setCik(picked)} + /> + )} +
+ ); +} + +function SearchResults({ + query, + state, + onPick, +}: { + query: string; + state: ReturnType< + typeof useQuery< + Awaited>> + > + >; + onPick: (cik: string) => void; +}) { + if (!query.trim()) { + return ( +

+ Search a US public company to read its filings, its 5%+ shareholders and + its proxy statement, then hand it to the agent to import. +

+ ); + } + if (state.isPending) { + return ( +

+ Searching EDGAR… +

+ ); + } + if (state.isError || !state.data) { + return ( +

The search failed. Try again.

+ ); + } + if (state.data.status !== "ok") { + return

{state.data.reason}

; + } + if (state.data.data.companies.length === 0) { + return ( +

+ Nothing in EDGAR matches “{query}”. Only companies that file with the + SEC are listed; try the ticker. +

+ ); + } + + return ( + + {state.data.data.companies.map((company) => ( + onPick(company.cik)} + > + {company.name} + {company.ticker ?? "—"} + {company.exchange ?? "—"} + + {company.cik} + + + ))} + + ); +} diff --git a/apps/app/app/(landing)/sign-in/page.tsx b/apps/app/app/(landing)/sign-in/page.tsx index 5b0e451af..8f5346ae7 100644 --- a/apps/app/app/(landing)/sign-in/page.tsx +++ b/apps/app/app/(landing)/sign-in/page.tsx @@ -5,6 +5,7 @@ import { Suspense } from "react"; import { AuthHeading, AuthShell } from "@/components/auth-shell"; import { getSession } from "@/lib/session"; import { getServerQueryClient, getServerTrpc } from "@/lib/trpc/server"; +import { PasswordSignIn } from "./password-sign-in"; import { SocialSignIn } from "./social-sign-in"; import { type SsoProvider, SsoSignIn } from "./sso-sign-in"; @@ -15,6 +16,7 @@ export const metadata: Metadata = { type SignInOptions = { google: boolean; microsoft: boolean; + password: boolean; providers: SsoProvider[]; }; @@ -75,6 +77,7 @@ async function SignIn({ if (options?.microsoft ?? false) configured.push("microsoft"); const providers = options?.providers ?? []; + const showPassword = options?.password ?? false; const insisted = configured.find((provider) => provider === method); const showSso = providers.length > 0 && insisted === undefined; @@ -85,7 +88,7 @@ async function SignIn({ ? configured : []; - if (!showSso && social.length === 0) { + if (!showSso && social.length === 0 && !showPassword) { return ( <> Set GOOGLE_CLIENT_ID and GOOGLE_CLIENT_SECRET — or MICROSOFT_CLIENT_ID - and MICROSOFT_CLIENT_SECRET — in the root .env file and restart. Your - own identity provider can be added from Settings once somebody is - signed in. + and MICROSOFT_CLIENT_SECRET, or PASSWORD_SIGN_IN — in the root .env + file and restart. Your own identity provider can be added from + Settings once somebody is signed in.

); @@ -110,6 +113,7 @@ async function SignIn({ description="Sign in with your account to continue." /> + {showPassword ? : null} {showSso ? : null} {social.map((provider) => ( diff --git a/apps/app/app/(landing)/sign-in/password-sign-in.tsx b/apps/app/app/(landing)/sign-in/password-sign-in.tsx new file mode 100644 index 000000000..0d4445300 --- /dev/null +++ b/apps/app/app/(landing)/sign-in/password-sign-in.tsx @@ -0,0 +1,74 @@ +"use client"; + +import { signIn } from "@crm/auth/client"; +import { Button } from "@crm/ui/components/button"; +import { Field, FieldLabel } from "@crm/ui/components/field"; +import { Input } from "@crm/ui/components/input"; +import { Spinner } from "@crm/ui/components/spinner"; +import { useRouter } from "next/navigation"; +import { type FormEvent, useId, useState } from "react"; +import { toast } from "sonner"; + +export function PasswordSignIn() { + const router = useRouter(); + const emailId = useId(); + const passwordId = useId(); + const [email, setEmail] = useState(""); + const [password, setPassword] = useState(""); + const [pending, setPending] = useState(false); + + async function handleSubmit(event: FormEvent) { + event.preventDefault(); + setPending(true); + + const { error } = await signIn.email({ email, password }); + + if (error) { + setPending(false); + toast.error(error.message ?? "Could not sign in."); + return; + } + + router.replace("/"); + router.refresh(); + } + + return ( +
{ + handleSubmit(event).catch(() => { + setPending(false); + toast.error("Could not reach the sign-in service."); + }); + }} + > + + Email + setEmail(event.target.value)} + /> + + + Password + setPassword(event.target.value)} + /> + + +
+ ); +} diff --git a/apps/app/app/edgar/[...path]/route.ts b/apps/app/app/edgar/[...path]/route.ts new file mode 100644 index 000000000..82e074d31 --- /dev/null +++ b/apps/app/app/edgar/[...path]/route.ts @@ -0,0 +1,54 @@ +import { connection } from "next/server"; +import { edgarConfigured, edgarHeaders, edgarTarget } from "@/lib/edgar"; +import { getSession } from "@/lib/session"; + +const TIMEOUT_MS = 60_000; + +export async function GET( + request: Request, + { params }: { params: Promise<{ path: string[] }> }, +): Promise { + await connection(); + + if (!edgarConfigured()) { + return Response.json( + { reason: "The SEC EDGAR service is not configured for this install." }, + { status: 503 }, + ); + } + + const session = await getSession(); + if (!session) { + return Response.json({ reason: "Not signed in." }, { status: 401 }); + } + + const { path } = await params; + const url = new URL(request.url); + const target = edgarTarget(path.join("/"), url.search); + if (!target) { + return Response.json({ reason: "No such route." }, { status: 404 }); + } + + try { + const upstream = await fetch(target, { + headers: edgarHeaders(), + signal: AbortSignal.timeout(TIMEOUT_MS), + cache: "no-store", + }); + const body = await upstream.text(); + return new Response(body, { + status: upstream.status, + headers: { "content-type": "application/json" }, + }); + } catch (error) { + return Response.json( + { + reason: + error instanceof Error && error.name === "TimeoutError" + ? "The SEC EDGAR service did not answer in time." + : "The SEC EDGAR service is unreachable.", + }, + { status: 502 }, + ); + } +} diff --git a/apps/app/components/app-icon-rail.tsx b/apps/app/components/app-icon-rail.tsx index ad3440e00..57b2f9c23 100644 --- a/apps/app/components/app-icon-rail.tsx +++ b/apps/app/components/app-icon-rail.tsx @@ -4,6 +4,7 @@ import Building from "@carbon/icons-react/es/Building"; import Close from "@carbon/icons-react/es/Close"; import Dashboard from "@carbon/icons-react/es/Dashboard"; import Partnership from "@carbon/icons-react/es/Partnership"; +import Search from "@carbon/icons-react/es/Search"; import Settings from "@carbon/icons-react/es/Settings"; import UserMultiple from "@carbon/icons-react/es/UserMultiple"; import { Button } from "@crm/ui/components/button"; @@ -57,6 +58,7 @@ const ITEMS: RailItem[] = [ match: "prefix", }, { title: "Deals", href: "/deals", icon: Partnership, match: "prefix" }, + { title: "Research", href: "/research/sec", icon: Search, match: "prefix" }, { title: "Settings", href: "/settings", icon: Settings, match: "prefix" }, ]; diff --git a/apps/app/lib/agent-transcript.ts b/apps/app/lib/agent-transcript.ts index 2898c35bb..c1bf82896 100644 --- a/apps/app/lib/agent-transcript.ts +++ b/apps/app/lib/agent-transcript.ts @@ -63,6 +63,20 @@ const VERBS: ToolVerbs = { search_crm: "Looked the record up in the CRM", resolve_linkedin_profile: "Searched for their LinkedIn profile", get_linkedin_profile: "Read a LinkedIn profile", + add_company: "Added a company to the CRM", + find_contact_details: "Looked up their work email and phone", + gleif_search_entities: "Searched the GLEIF register for a company", + gleif_get_entity: "Read a company's GLEIF record", + gleif_list_subsidiaries: "Listed a group's subsidiaries from GLEIF", + add_contact: "Added a contact to the CRM", + sec_search_companies: "Searched SEC EDGAR for a company", + sec_get_company: "Read a company's SEC profile", + sec_list_filings: "Listed a company's SEC filings", + sec_search_filings: "Searched SEC filings", + sec_list_owners: "Listed a company's major shareholders", + sec_list_insiders: "Listed a company's insider transactions", + sec_get_proxy: "Read a company's proxy statement", + sec_compare_compensation: "Compared executive pay across companies", get_contact_work_history: "Read their work history", fetch_contact_photo: "Fetched their profile picture", find_contact_socials: "Searched for their other profiles", diff --git a/apps/app/lib/edgar.ts b/apps/app/lib/edgar.ts new file mode 100644 index 000000000..ecbd51ebc --- /dev/null +++ b/apps/app/lib/edgar.ts @@ -0,0 +1,34 @@ +const ALLOWED_PATHS = [ + /^health$/, + /^companies\/search$/, + /^companies\/[A-Za-z0-9.-]{1,12}$/, + /^companies\/[A-Za-z0-9.-]{1,12}\/(filings|owners|insiders|proxy)$/, + /^filings\/search$/, + /^compensation\/compare$/, +]; + +export function edgarUrl(): string | null { + const url = process.env.EDGAR_URL?.trim(); + return url ? url.replace(/\/+$/, "") : null; +} + +export function edgarConfigured(): boolean { + return edgarUrl() !== null; +} + +export function edgarPathAllowed(path: string): boolean { + return ALLOWED_PATHS.some((pattern) => pattern.test(path)); +} + +export function edgarTarget(path: string, search: string): URL | null { + const base = edgarUrl(); + if (!base || !edgarPathAllowed(path)) return null; + return new URL(`${base}/${path}${search}`); +} + +export function edgarHeaders(): HeadersInit { + const secret = process.env.EDGAR_SECRET?.trim(); + return secret + ? { accept: "application/json", authorization: `Bearer ${secret}` } + : { accept: "application/json" }; +} diff --git a/apps/app/proxy.ts b/apps/app/proxy.ts index 6040b7068..e069a3f6d 100644 --- a/apps/app/proxy.ts +++ b/apps/app/proxy.ts @@ -14,7 +14,7 @@ const LANDING_PATH = "/"; const SIGN_IN_PATH = "/sign-in"; -const UNGATED = ["/grant-access", "/eve"]; +const UNGATED = ["/grant-access", "/eve", "/edgar"]; const ANONYMOUS = ["/t"]; diff --git a/apps/app/test/edgar.spec.ts b/apps/app/test/edgar.spec.ts new file mode 100644 index 000000000..6ad68f76f --- /dev/null +++ b/apps/app/test/edgar.spec.ts @@ -0,0 +1,68 @@ +import { afterEach, describe, expect, it } from "bun:test"; +import { + edgarConfigured, + edgarHeaders, + edgarPathAllowed, + edgarTarget, +} from "../lib/edgar"; + +const savedUrl = process.env.EDGAR_URL; +const savedSecret = process.env.EDGAR_SECRET; + +afterEach(() => { + if (savedUrl === undefined) delete process.env.EDGAR_URL; + else process.env.EDGAR_URL = savedUrl; + if (savedSecret === undefined) delete process.env.EDGAR_SECRET; + else process.env.EDGAR_SECRET = savedSecret; +}); + +describe("the app's edgar helpers", () => { + it("is off without a URL", () => { + delete process.env.EDGAR_URL; + expect(edgarConfigured()).toBe(false); + expect(edgarTarget("health", "")).toBeNull(); + }); + + it("proxies only the routes the page reads", () => { + for (const path of [ + "health", + "companies/search", + "companies/320193", + "companies/AAPL", + "companies/320193/filings", + "companies/320193/owners", + "companies/320193/insiders", + "companies/320193/proxy", + "filings/search", + "compensation/compare", + ]) { + expect(edgarPathAllowed(path)).toBe(true); + } + for (const path of [ + "", + "docs", + "companies/320193/secrets", + "companies/../health", + "companies/320193/filings/x", + ]) { + expect(edgarPathAllowed(path)).toBe(false); + } + }); + + it("builds the target from the URL, the path and the query", () => { + process.env.EDGAR_URL = "http://127.0.0.1:2100/"; + expect( + edgarTarget("companies/search", "?q=apple&limit=3")?.toString(), + ).toBe("http://127.0.0.1:2100/companies/search?q=apple&limit=3"); + }); + + it("sends the secret only when it is set", () => { + process.env.EDGAR_SECRET = "s3cret"; + expect(edgarHeaders()).toEqual({ + accept: "application/json", + authorization: "Bearer s3cret", + }); + delete process.env.EDGAR_SECRET; + expect(edgarHeaders()).toEqual({ accept: "application/json" }); + }); +}); diff --git a/apps/app/test/onboarding-gate.spec.ts b/apps/app/test/onboarding-gate.spec.ts index 0480f1429..bf78246fa 100644 --- a/apps/app/test/onboarding-gate.spec.ts +++ b/apps/app/test/onboarding-gate.spec.ts @@ -292,6 +292,16 @@ describe("proxy", () => { ).toBeNull(); }); + it("leaves the SEC EDGAR proxy alone", async () => { + setup({ onboarded: false }); + + expect( + redirectedTo( + await proxy(request("/edgar/companies/search", [SESSION_COOKIE])), + ), + ).toBeNull(); + }); + it("fails open when the API is unreachable", async () => { stub(async () => { throw new Error("connect ECONNREFUSED"); diff --git a/apps/app/turbo.json b/apps/app/turbo.json index cea0ca0ed..445625907 100644 --- a/apps/app/turbo.json +++ b/apps/app/turbo.json @@ -6,7 +6,7 @@ "dependsOn": ["^build"], "inputs": ["$TURBO_DEFAULT$", ".env*"], "outputs": [".next/**", "!.next/cache/**", "!.next/dev/**"], - "env": ["NEXT_PUBLIC_API_URL", "NEXT_PUBLIC_AUTH_URL"], + "env": ["$TURBO_EXTENDS$", "NEXT_PUBLIC_API_URL", "NEXT_PUBLIC_AUTH_URL"], "passThroughEnv": [ "AGENT_BRIDGE_SECRET", "AGENT_URL", @@ -15,6 +15,8 @@ "BETTER_AUTH_SECRET", "BETTER_AUTH_URL", "DATABASE_URL", + "EDGAR_SECRET", + "EDGAR_URL", "IS_MARKETING" ] }, @@ -38,6 +40,8 @@ "BETTER_AUTH_SECRET", "BETTER_AUTH_URL", "DATABASE_URL", + "EDGAR_SECRET", + "EDGAR_URL", "IS_MARKETING", "NEXT_PUBLIC_API_URL", "NEXT_PUBLIC_AUTH_URL" diff --git a/docker-compose.yml b/docker-compose.yml index 96fa7bf36..2ee9c60f7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,5 +18,23 @@ services: timeout: 5s retries: 20 + edgar: + build: services/edgar + container_name: crm-edgar + restart: unless-stopped + environment: + EDGAR_IDENTITY: ${EDGAR_IDENTITY:-} + EDGAR_SECRET: ${EDGAR_SECRET:-} + ports: + - "2100:2100" + volumes: + - crm-edgar-cache:/data + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://127.0.0.1:2100/health || exit 1"] + interval: 10s + timeout: 5s + retries: 12 + volumes: crm-postgres: + crm-edgar-cache: diff --git a/docs/agent.md b/docs/agent.md index e1fc98353..60d61d37f 100644 --- a/docs/agent.md +++ b/docs/agent.md @@ -219,6 +219,87 @@ missing key removes a place to look. **Never an error, never throws.** `capabilitiesFrom()`/`markdownFor()` are the pure halves. `contextDevKey()` is the only resolver, and `lib/context-dev.ts` memoises its client on the key string. +### The agent can add a company, and every one it adds names its source + +`lib/companies.ts` is the one write path: dedupe by domain then by name and +country, the row, the `company.created` event task the API would have written, +an `ENRICHMENT` activity that says where the company came from and links the +page, the LEI as a company field when there is one, then the same `brand` and +`company-profile` tasks `companyCreated` queues on the API side. `add_company` +requires a source; a company without one cannot be created by the agent. + +### Contact details come from a provider, never a guess + +`lib/contact-details.ts` is the registry: one `Provider` contract, one +`fetchJson` that parses every response with Zod at the boundary, and +`lookupContactDetails`, which asks the configured providers in +`CONTACT_DETAILS.order` and stops at the first answer above +`CONTACT_DETAILS.minConfidence`. The providers are one file each — `hunter.ts`, +`apollo.ts`, `lusha.ts`, `dropcontact.ts`, `zoominfo.ts` — and every one is off +without its key. Confidence is the provider's own signal mapped to one scale: +Hunter's score, Apollo's verification status, Lusha's email type, Dropcontact's +qualification, a ZoomInfo match. + +`website.ts` is the last provider and needs no key: it reads the employer's own +site — the paths in `CONTACT_DETAILS.website.paths`, fetched together, after +`robots.txt`, with a named user agent, a short timeout and a size cap — and +looks for the person by name. An address on the domain whose local part is a +form of the name (`ada.lovelace`, `alovelace`, `lovelacea`) is the answer at +`confidence.named`; a surname-only address counts when the same page names the +person; a `tel:` link on a site that names the person is the switchboard, typed +`main`, at `confidence.phoneOnly`. A `linkedin.com/in/` link is reported only +when the site itself labels it with the person's name — the site is read, never +LinkedIn. Nothing is charged for it. The sources are the pages the address was +seen on, with the day it was read, so the timeline entry reads the same as +Hunter's. + +`find_contact_details` fills the email and the phone only where the record has +none, and always writes the candidate, its confidence and its source to the +timeline: the public pages when the provider has them (Hunter, the website), +the provider and its record id otherwise. A blank beats a pattern guess, so +nothing is written below the threshold. One unit is charged when a keyed +provider is configured; a website-only lookup is free. + +### The GLEIF register needs no key + +`lib/gleif.ts` reads the public GLEIF API — legal entities by name, one entity by +LEI, the direct subsidiaries an entity consolidates — and is always on. The three +`gleif_*` tools are free: no budget is charged. Every response is parsed with Zod +at the boundary into `GleifEntity`; a shape the register does not promise is a +failed outcome with a reason, never a throw. Region names (`UE`, `ASIE`) and the +page cap live in `lib/gleif-config.ts`. The `gleif-mna-sourcing` skill is the +method: a parent place and a child place make a scenario, subsidiaries in the +child place are the targets, and the people who run them come from web research +under the same egress rules as everything else — never a LinkedIn fetch, never an +invented URL, one source per line. + +### SEC EDGAR comes from an external service + +`lib/edgar.ts` talks to `services/edgar`, a Python service on +[edgartools](https://edgartools.readthedocs.io) that the CRM does not host: Docker +Compose locally, a Colab notebook or another machine behind a tunnel elsewhere. +`EDGAR_URL` names it, `EDGAR_SECRET` goes in the bearer header, and without the URL +the capability is off and every `sec_*` tool answers `unavailable`. Every response +is parsed with Zod at the boundary — the shapes live in +`packages/validation/src/edgar.ts` because the app reads the same service — a 404 +is an empty answer, anything else non-2xx is a reason, never a throw. Limits and the +timeout live in `lib/edgar-config.ts`. + +The eight `sec_*` tools are free: search companies, one company's profile, its +filings, a full-text search across every filing, its 5%+ holders from Schedule +13D/13G, its insider transactions from Forms 3/4/5, its latest proxy statement +(DEF 14A: named executives with their pay, CEO pay and pay actually paid, pay +versus performance, the holders the proxy lists, the proposals, the CEO pay ratio) +and a CEO-pay comparison across tickers. Every row carries the filing URL, and +that URL is the source the agent cites. + +`add_company` keeps a CIK, a ticker and a SIC code as custom fields the way it keeps +a LEI, and a CIK implies `countryCode` US. **`add_contact` is the one agent-side +contact write path**: an executive or a director named in a public document goes on +as a contact of an existing company, with the source on the timeline and a +`contact.created` event so the people pipeline runs; the email or the name on the +same company dedupes. The `sec-us-research` skill is the method. + ## Budget and scheduling - `lib/focus.ts` — per-session budget in `defineState`; running out is a normal ending. diff --git a/docs/environment.md b/docs/environment.md index 22417c60e..57d08b2c7 100644 --- a/docs/environment.md +++ b/docs/environment.md @@ -43,8 +43,15 @@ the three that is genuinely optional on its own — set it to your tenant's GUID refuse other tenants at Microsoft instead of at `ALLOWED_SIGN_IN`. There is **no Microsoft equivalent of `hd`**: `tenantId` is the whole of it. -**Neither pair is required, but an install wants one of them or an SSO provider** — -with none, the sign-in page says so by name rather than rendering nothing. +**`PASSWORD_SIGN_IN="true"`** adds an email + password form to the sign-in page +(`emailAndPassword` in `auth.ts`). Sign-up goes through the same +`user.create.before` hook as the social providers, so only an address that passes +`ALLOWED_SIGN_IN` can create an account. Off by default; only the literal `true` +turns it on. + +**Neither pair is required, but an install wants one of them, a password form, or +an SSO provider** — with none, the sign-in page says so by name rather than +rendering nothing. **`ALLOWED_SIGN_IN`** — comma-separated whole domains or single addresses (bare addresses exist for a solo self-hoster, where `gmail.com` would be an open door). **One @@ -112,10 +119,33 @@ single place that knows what is set. | Variable | What it adds | | --- | --- | | `PERPLEXITY_API_KEY` | Open-web research with citations; finds a LinkedIn slug | +| `HUNTER_API_KEY` | Contact details: a work email with the pages it was seen on; a deliverability check | +| `APOLLO_API_KEY` | Contact details: work email with verification status, title, work phone | +| `LUSHA_API_KEY` | Contact details: work email, direct and mobile phones, title | +| `DROPCONTACT_API_KEY` | Contact details: work email with qualification, phone, title; asynchronous | +| `ZOOMINFO_USERNAME` + `ZOOMINFO_PASSWORD` | Contact details from ZoomInfo; a pair, both or neither | | `GITHUB_TOKEN` | Raises the GitHub rate limit from 60/hour | | `BLOB_READ_WRITE_TOKEN` | Mirrors logos and photos into Blob | | `AI_GATEWAY_API_KEY` | The model. Not needed on Vercel (OIDC) | | `AGENT_BRIDGE_SECRET` | The rep-facing Agent panel — see `agent.md` | +| `EDGAR_URL` + `EDGAR_SECRET` | SEC EDGAR research through `services/edgar`: profile, filings, 5%+ holders, insiders, proxy statement and executive pay | + +### External services + +`services/edgar` is the first **external service**: a process the CRM does not host, +reached over HTTP with a bearer secret. The contract is generic — `EDGAR_URL` is the +base, `EDGAR_SECRET` goes in `authorization: Bearer`, `GET /health` says whether it is +up, every response is JSON that the CRM parses with Zod +(`packages/validation/src/edgar.ts`) before use, and a missing URL turns the +capability off without an error. Only the routes are SEC-specific. The same shape +works for a service on another machine or in a Google Colab notebook behind a +tunnel; `services/edgar/README.md` shows both. The Next.js app reads the same two +variables for its Research → SEC page, so they are in `apps/app/turbo.json` too. `EDGAR_IDENTITY` is read by the +service alone: the SEC requires every automated client to name a contact email. + +The employer's own website is a contact-details source with no variable at all: +`find_contact_details` reads it after the keyed providers, under its `robots.txt`, +and charges nothing for it. `agent.md` has the rules. `BLOB_READ_WRITE_TOKEN` is also in `env.validation.ts` and `apps/api/turbo.json` because the API and the seed write pictures too. The Next.js app is deliberately @@ -174,8 +204,12 @@ imports nothing, Calendar reads from `now`, and Outlook records `now` as its cur **`CRON_SECRET`** (min 16 chars) guards `POST /internal/sync/mailboxes` and `/internal/sync/rates`; both **fail closed when unset**. `/internal/sync/google` is kept as an alias of the first, so an existing deployment's cron does not break on -deploy. **Crons live in `apps/api/vercel.json`** — mailboxes `*/5 * * * *`, rates -daily. Minute-level schedules need a Pro plan; on Hobby it silently becomes daily. +deploy. **Crons live in `apps/api/vercel.json`**, and `build-func.mjs` copies them +into the Build Output config, so that file is the only place to edit. Every cron +there, and the agent's `schedules/dispatch.ts`, runs **once a day**: Vercel's Hobby +plan rejects the build for anything more frequent. On Pro, mailboxes can go back to +`*/5 * * * *` and the dispatch to `* * * * *`; the API's poke covers new tasks +between ticks either way. Deliberate absences: **no `GOOGLE_SYNC_ENABLED`** (a switch that can disable a mandatory feature is only ever wrong), **no `GOOGLE_WORKSPACE_DOMAIN`** (`ALLOWED_SIGN_IN` already diff --git a/docs/setup.md b/docs/setup.md index 03b8912f7..d34dc1e1f 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -8,11 +8,19 @@ reads once. ```sh cp .env.example .env # fill DATABASE_URL, BETTER_AUTH_SECRET, ALLOWED_SIGN_IN -docker compose up -d # Postgres, matching .env.example +docker compose up -d # Postgres and the SEC EDGAR service, matching .env.example bun run db:migrate && bun run db:seed bun run dev # app :3000, api :3001, agent :2000 ``` +`docker compose up -d` also builds `services/edgar`, the Python service behind the +agent's `sec_*` tools, on port 2100. It needs `EDGAR_IDENTITY` in `.env` (the SEC +asks every automated client for a contact email) and the CRM needs +`EDGAR_URL=http://127.0.0.1:2100`. Leave both unset and the agent simply reports +the source as unavailable. `services/edgar/README.md` covers running it on another +machine (`services/edgar/tunnel.sh` starts it behind a Cloudflare tunnel and prints +the two variables to set) or in Google Colab. + Prisma from the repo root: `db:generate`, `db:migrate`, `db:push`, `db:reset`, `db:seed`, `db:studio`, `db:deploy`. @@ -175,3 +183,37 @@ A rebuild drops the database and re-runs every migration, and it says which of t two reasons fired. Force one with `bun run db:test --reset`. Nothing else in the repo may drop a database, and this may only because the `_test` suffix is checked first. + +## Agent Reach in Claude Code on the web + +`.claude/hooks/session-start.sh` runs on every remote session start and installs +[Agent Reach](https://github.com/Panniantong/agent-reach) outside the repo: the +CLI in `~/.agent-reach-venv`, the upstream tools in `~/.agent-reach/tools` and +`~/.local/bin`, mcporter with Exa, LinkedIn and XiaoHongShu registered, gh and +ffmpeg from apt. It does nothing on a local machine. Every step is best effort: +a failed step logs and the session still starts. + +The proxy blocks GitHub archive and release downloads, so the hook clones over +git and builds `xiaohongshu-mcp` with Go instead of downloading a binary. The +hook adds both bin directories to `PATH` for the session, so `agent-reach +doctor`, `twitter`, `rdt`, `bili`, `yt-dlp` and `mcporter` work from any shell. + +Channels that need a login (Twitter, Reddit, XiaoHongShu, Xueqiu, LinkedIn, +Groq for podcasts) still need their cookies or keys pasted into +`agent-reach configure` each session. Facebook and Instagram need a desktop +Chrome and never work in the container. + +## The CRM itself in Claude Code on the web + +`.claude/hooks/crm-dev.sh` is the second `SessionStart` command in +`.claude/settings.json`. It runs after the Agent Reach hook and prepares the +repo: Node 24 in `/opt/node24` (eve refuses Node 22), a `.env` from +`.env.example` with generated secrets when none exists, `dockerd` when no daemon +answers, `docker compose up -d`, `bun install`, `migrate deploy` and the seed. +It does nothing on a local machine. The `.env` it writes sets `ALLOWED_SIGN_IN` +to `localhost`, so sign-in stays closed; `bun run --filter=api dev:session` +mints a session without a provider. + +The container has no terminal UI, so `bun run dev` refuses the interactive +agent task. Run `turbo run dev --filter=app --filter=api` and +`turbo run dev:headless --filter=agent` instead. diff --git a/package.json b/package.json index 534ec9d00..f3590e38a 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "typescript": "5.9.2" }, "engines": { - "node": ">=22" + "node": ">=24" }, "packageManager": "bun@1.3.12", "devEngines": { diff --git a/packages/auth/src/auth.ts b/packages/auth/src/auth.ts index c59c98254..32d3d3cdb 100644 --- a/packages/auth/src/auth.ts +++ b/packages/auth/src/auth.ts @@ -78,7 +78,7 @@ export const auth = betterAuth({ }), emailAndPassword: { - enabled: false, + enabled: env.passwordSignIn, }, socialProviders, diff --git a/packages/auth/src/env.ts b/packages/auth/src/env.ts index 9813d7a44..4875a0962 100644 --- a/packages/auth/src/env.ts +++ b/packages/auth/src/env.ts @@ -62,6 +62,7 @@ export const env = { google: googleCredentials(), microsoft: microsoftCredentials(), slack: slackCredentials(), + passwordSignIn: optional("PASSWORD_SIGN_IN") === "true", cookieDomain: optional("AUTH_COOKIE_DOMAIN"), trustedOrigins: [...new Set([...appUrls, apiUrl])], isProduction: process.env.NODE_ENV === "production", @@ -79,4 +80,8 @@ export function isSlackConfigured(): boolean { return env.slack !== undefined; } +export function isPasswordSignInEnabled(): boolean { + return env.passwordSignIn; +} + export { apiUrl, appUrl }; diff --git a/packages/auth/src/index.ts b/packages/auth/src/index.ts index 31fbcbaad..cb09172eb 100644 --- a/packages/auth/src/index.ts +++ b/packages/auth/src/index.ts @@ -11,6 +11,7 @@ export { appUrl, isGoogleConfigured, isMicrosoftConfigured, + isPasswordSignInEnabled, isSlackConfigured, } from "./env"; export { diff --git a/packages/telemetry/src/allowlist.ts b/packages/telemetry/src/allowlist.ts index 5880dc2bd..93f64aa53 100644 --- a/packages/telemetry/src/allowlist.ts +++ b/packages/telemetry/src/allowlist.ts @@ -114,13 +114,19 @@ export function permitted( } export const AGENT_TOOLS = [ + "add_company", + "add_contact", "agent", "archive_field", "enrich_company", "fetch_contact_photo", + "find_contact_details", "find_contact_socials", "get_contact_work_history", "get_linkedin_profile", + "gleif_get_entity", + "gleif_list_subsidiaries", + "gleif_search_entities", "identify_contact", "list_deals", "list_fields", @@ -136,6 +142,14 @@ export const AGENT_TOOLS = [ "resolve_linkedin_profile", "schedule_recheck", "search_crm", + "sec_compare_compensation", + "sec_get_company", + "sec_get_proxy", + "sec_list_filings", + "sec_list_insiders", + "sec_list_owners", + "sec_search_companies", + "sec_search_filings", "set_chat_title", "set_contact_socials", "set_field_value", diff --git a/packages/validation/package.json b/packages/validation/package.json index d6fb2c36a..843e97626 100644 --- a/packages/validation/package.json +++ b/packages/validation/package.json @@ -9,6 +9,7 @@ "./agent-events": "./src/agent-events.ts", "./agent-manifest": "./src/agent-manifest.ts", "./builder-question": "./src/builder-question.ts", + "./edgar": "./src/edgar.ts", "./enrichment-queue": "./src/enrichment-queue.ts", "./eve-stream": "./src/eve-stream.ts", "./eve-tool": "./src/eve-tool.ts", diff --git a/packages/validation/src/edgar.ts b/packages/validation/src/edgar.ts new file mode 100644 index 000000000..6467078af --- /dev/null +++ b/packages/validation/src/edgar.ts @@ -0,0 +1,202 @@ +import { z } from "zod"; + +const cik = z + .string() + .trim() + .regex(/^\d{1,10}$/); +const day = z + .string() + .trim() + .regex(/^\d{4}-\d{2}-\d{2}$/); +const money = z.number().nullable(); + +export const edgarHealth = z.object({ + ok: z.literal(true), + version: z.string(), + edgartools: z.string(), + identitySet: z.boolean(), +}); + +export const edgarCompanyMatch = z.object({ + cik, + name: z.string().trim().min(1), + ticker: z.string().trim().nullable(), + exchange: z.string().trim().nullable(), +}); + +export const edgarCompanySearch = z.object({ + companies: z.array(edgarCompanyMatch), +}); + +export const edgarAddress = z.object({ + street: z.string().nullable(), + city: z.string().nullable(), + state: z.string().nullable(), + zip: z.string().nullable(), +}); + +export const edgarCompany = z.object({ + cik, + name: z.string().trim().min(1), + tickers: z.array(z.string()), + exchanges: z.array(z.string()), + sic: z.string().nullable(), + sicDescription: z.string().nullable(), + stateOfIncorporation: z.string().nullable(), + fiscalYearEnd: z.string().nullable(), + category: z.string().nullable(), + businessAddress: edgarAddress.nullable(), + website: z.string().nullable(), + formerNames: z.array(z.string()), + url: z.string().url(), +}); + +export const edgarFiling = z.object({ + accession: z.string().trim().min(1), + form: z.string().trim().min(1), + filedAt: day, + reportDate: day.nullable(), + description: z.string().nullable(), + url: z.string().url(), +}); + +export const edgarFilings = z.object({ + filings: z.array(edgarFiling), + truncated: z.boolean(), +}); + +export const edgarFilingHit = edgarFiling.extend({ + company: z.object({ cik, name: z.string().trim().min(1) }), +}); + +export const edgarFilingSearch = z.object({ + filings: z.array(edgarFilingHit), + total: z.number().int().nonnegative(), +}); + +export const edgarOwner = z.object({ + filer: z.string().trim().min(1), + form: z.string().trim().min(1), + filedAt: day, + shares: money, + percent: money, + soleVoting: money, + sharedVoting: money, + purpose: z.string().nullable(), + url: z.string().url(), +}); + +export const edgarOwners = z.object({ + owners: z.array(edgarOwner), + filingsRead: z.number().int().nonnegative(), +}); + +export const edgarInsiderTransaction = z.object({ + insider: z.string().trim().min(1), + title: z.string().nullable(), + form: z.string().trim().min(1), + filedAt: day, + kind: z.string().nullable(), + shares: money, + price: money, + url: z.string().url(), +}); + +export const edgarInsiders = z.object({ + transactions: z.array(edgarInsiderTransaction), +}); + +export const edgarCompensationYear = z.object({ + fiscalYearEnd: day.nullable(), + peoTotalComp: money, + peoActuallyPaidComp: money, + neoAverageTotalComp: money, + neoAverageActuallyPaidComp: money, +}); + +export const edgarPerformanceYear = z.object({ + fiscalYearEnd: day.nullable(), + peoActuallyPaidComp: money, + neoAverageActuallyPaidComp: money, + tsr: money, + peerTsr: money, + netIncome: money, + selectedMeasureValue: money, +}); + +export const edgarExecutivePay = z.object({ + name: z.string().trim().min(1), + title: z.string().nullable(), + year: z.number().int().nullable(), + salary: money, + bonus: money, + stockAwards: money, + optionAwards: money, + nonEquityIncentive: money, + otherCompensation: money, + total: money, +}); + +export const edgarProxyHolder = z.object({ + name: z.string().trim().min(1), + percentOfClass: money, + shares: money, +}); + +export const edgarProposal = z.object({ + number: z.number().int().nullable(), + description: z.string(), + type: z.string().nullable(), +}); + +export const edgarProxy = z.object({ + accession: z.string().trim().min(1), + filedAt: day, + url: z.string().url(), + peo: z.object({ + name: z.string().nullable(), + totalComp: money, + actuallyPaidComp: money, + }), + neoAverage: z.object({ totalComp: money, actuallyPaidComp: money }), + compensationByYear: z.array(edgarCompensationYear), + payVsPerformance: z.array(edgarPerformanceYear), + executives: z.array(edgarExecutivePay), + holders: z.array(edgarProxyHolder), + proposals: z.array(edgarProposal), + performanceMeasures: z.array(z.string()), + selectedMeasureName: z.string().nullable(), + ceoPayRatio: z + .object({ ceo: money, medianEmployee: money, ratio: money }) + .nullable(), + insiderTradingPolicyAdopted: z.boolean().nullable(), +}); + +export const edgarCompensationComparison = z.object({ + rows: z.array( + z.object({ + ticker: z.string(), + cik: cik.nullable(), + name: z.string().nullable(), + fiscalYearEnd: day.nullable(), + peoName: z.string().nullable(), + peoTotalComp: money, + peoActuallyPaidComp: money, + tsr: money, + netIncome: money, + reason: z.string().nullable(), + }), + ), +}); + +export type EdgarHealth = z.infer; +export type EdgarCompanyMatch = z.infer; +export type EdgarCompany = z.infer; +export type EdgarFiling = z.infer; +export type EdgarFilingHit = z.infer; +export type EdgarOwner = z.infer; +export type EdgarInsiderTransaction = z.infer; +export type EdgarProxy = z.infer; +export type EdgarCompensationComparison = z.infer< + typeof edgarCompensationComparison +>; diff --git a/services/edgar/.dockerignore b/services/edgar/.dockerignore new file mode 100644 index 000000000..1dec89c76 --- /dev/null +++ b/services/edgar/.dockerignore @@ -0,0 +1,6 @@ +.venv +__pycache__ +*.pyc +tests +colab.ipynb +README.md diff --git a/services/edgar/Dockerfile b/services/edgar/Dockerfile new file mode 100644 index 000000000..da4e92ab2 --- /dev/null +++ b/services/edgar/Dockerfile @@ -0,0 +1,26 @@ +FROM python:3.12-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + EDGAR_PORT=2100 \ + EDGAR_DATA_DIR=/data \ + EDGAR_LOCAL_DATA_DIR=/data + +RUN apt-get update \ + && apt-get install -y --no-install-recommends curl \ + && rm -rf /var/lib/apt/lists/* \ + && useradd --create-home --uid 1000 edgar \ + && mkdir -p /data && chown edgar:edgar /data + +WORKDIR /app +COPY requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt +COPY edgar_service ./edgar_service + +USER edgar +VOLUME ["/data"] +EXPOSE 2100 + +HEALTHCHECK --interval=30s --timeout=5s --retries=3 CMD curl -fsS http://127.0.0.1:2100/health || exit 1 + +CMD ["sh", "-c", "uvicorn edgar_service.app:app --host 0.0.0.0 --port ${EDGAR_PORT}"] diff --git a/services/edgar/README.md b/services/edgar/README.md new file mode 100644 index 000000000..488e6ea89 --- /dev/null +++ b/services/edgar/README.md @@ -0,0 +1,98 @@ +# edgar — SEC EDGAR research for the CRM + +A small HTTP service on [edgartools](https://edgartools.readthedocs.io) that the +agent's `sec_*` tools and the app's SEC page call. It reads SEC EDGAR (free, no +API key) and answers JSON: company profile, filings, full-text search, 5%+ +holders from Schedule 13D/13G, insider transactions from Forms 3/4/5, the latest +proxy statement (DEF 14A) with its executives and their pay, and a CEO-pay +comparison across tickers. + +The SEC requires every automated client to identify itself, so `EDGAR_IDENTITY` +(a name and a real email) is mandatory. + +## The contract, which any external service can reuse + +| Piece | Here | +| --- | --- | +| Base URL | `EDGAR_URL` in the CRM's `.env`, e.g. `http://127.0.0.1:2100` | +| Auth | `authorization: Bearer `; optional on loopback | +| Liveness | `GET /health` → `{ ok, version, edgartools, identitySet }` | +| Answers | JSON, HTTP 200; `404 { reason }` for a missing record; `502 { reason }` when the SEC misbehaves; `401 { reason }` for a bad secret | +| CRM side | every response parsed with Zod in `packages/validation/src/edgar.ts`; a missing `EDGAR_URL` turns the capability off | + +Only the routes are SEC-specific. A service for another source keeps the same +shape. + +## Routes + +| Route | Answers | +| --- | --- | +| `GET /companies/search?q=&limit=` | companies matching a name, ticker or CIK | +| `GET /companies/{cik or ticker}` | profile: tickers, exchanges, SIC, state, fiscal year end, address, former names | +| `GET /companies/{key}/filings?form=&from=&to=&limit=` | filings, newest first | +| `GET /filings/search?q=&form=&from=&to=&limit=` | full-text search across all filers | +| `GET /companies/{key}/owners?minPercent=&form=13D\|13G\|all&limit=` | 5%+ holders, one row per holder from their newest filing | +| `GET /companies/{key}/insiders?limit=` | Forms 3/4/5, one row per filing | +| `GET /companies/{key}/proxy?years=` | latest DEF 14A: executives, CEO pay, pay vs performance, holders, proposals, pay ratio | +| `GET /compensation/compare?tickers=A,B&years=` | CEO pay rows per ticker and fiscal year | + +## Run it with Docker Compose (default) + +```sh +# in the repo's .env +EDGAR_IDENTITY="Jane Doe jane@example.com" +EDGAR_URL="http://127.0.0.1:2100" + +docker compose up -d edgar +curl http://127.0.0.1:2100/health +``` + +The edgartools cache lives in the `crm-edgar-cache` volume, so repeat lookups +are fast and the SEC rate limit (10 requests a second) is respected by +edgartools itself. + +## Run it on another machine + +One command does everything below — build or venv, secret, service, tunnel — +and prints the two variables to paste into the CRM: + +```sh +EDGAR_IDENTITY="Jane Doe jane@example.com" services/edgar/tunnel.sh +``` + +It needs Docker (or Python 3.11+) and +[cloudflared](https://developers.cloudflare.com/cloudflare-one/connections/connect-networks/downloads/). +`EDGAR_RUNTIME=python` forces the venv path when Docker is present. By hand: + +```sh +cd services/edgar +python -m venv .venv && . .venv/bin/activate +pip install -r requirements.txt +EDGAR_IDENTITY="Jane Doe jane@example.com" EDGAR_SECRET="$(openssl rand -hex 24)" \ + uvicorn edgar_service.app:app --host 0.0.0.0 --port 2100 +``` + +Expose it with a tunnel when the CRM runs elsewhere: + +```sh +cloudflared tunnel --url http://127.0.0.1:2100 +``` + +Then set `EDGAR_URL` to the tunnel URL and `EDGAR_SECRET` to the same secret in +the CRM's environment (Vercel → crm-agent, or the local `.env`). A quick tunnel +URL changes on every start. + +## Run it in Google Colab + +Open `colab.ipynb` in Colab and run the cells: it installs the service, sets the +identity and a generated secret, starts uvicorn and a `cloudflared` tunnel, and +prints the `EDGAR_URL` and `EDGAR_SECRET` to paste into the CRM. The service +lives as long as the notebook does. + +## Tests + +```sh +pip install -e ".[dev]" +pytest # mocked edgartools objects +EDGAR_LIVE=1 python tests/smoke_live.py # real calls on CIK 320193 +``` diff --git a/services/edgar/colab.ipynb b/services/edgar/colab.ipynb new file mode 100644 index 000000000..33a89d9f5 --- /dev/null +++ b/services/edgar/colab.ipynb @@ -0,0 +1,54 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": "# CRM \u00b7 SEC EDGAR service on Colab\n\nRuns `services/edgar` here and exposes it through a Cloudflare quick tunnel. Fill the two values in the next cell, run every cell, then paste the printed `EDGAR_URL` and `EDGAR_SECRET` into the CRM's environment. The service lives as long as this notebook runs." + }, + { + "cell_type": "code", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": "IDENTITY = \"Jane Doe jane@example.com\" # the SEC asks every automated client for a name and a real email\nREPO = \"https://github.com/teknewmcc26/crm\" # the CRM repository that holds services/edgar\n" + }, + { + "cell_type": "code", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": "import secrets, subprocess, sys\n\nSECRET = secrets.token_hex(24)\nsubprocess.run([\"git\", \"clone\", \"--depth\", \"1\", REPO, \"crm\"], check=False)\nsubprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"-r\", \"crm/services/edgar/requirements.txt\"], check=True)\nsubprocess.run([\"bash\", \"-c\", \"curl -sSL https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64 -o /usr/local/bin/cloudflared && chmod +x /usr/local/bin/cloudflared\"], check=True)\nprint(\"installed\")\n" + }, + { + "cell_type": "code", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": "import os, subprocess, time\n\nenv = dict(os.environ, EDGAR_IDENTITY=IDENTITY, EDGAR_SECRET=SECRET, EDGAR_PORT=\"2100\", EDGAR_DATA_DIR=\"/content/edgar-cache\")\nserver = subprocess.Popen([sys.executable, \"-m\", \"uvicorn\", \"edgar_service.app:app\", \"--host\", \"0.0.0.0\", \"--port\", \"2100\"], cwd=\"crm/services/edgar\", env=env)\ntime.sleep(5)\nprint(subprocess.run([\"curl\", \"-s\", \"http://127.0.0.1:2100/health\"], capture_output=True, text=True).stdout)\n" + }, + { + "cell_type": "code", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": "import re, subprocess, time\n\ntunnel = subprocess.Popen([\"cloudflared\", \"tunnel\", \"--url\", \"http://127.0.0.1:2100\", \"--no-autoupdate\"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)\nurl = None\nfor _ in range(60):\n line = tunnel.stdout.readline()\n found = re.search(r\"https://[a-z0-9-]+\\.trycloudflare\\.com\", line)\n if found:\n url = found.group(0)\n break\n time.sleep(0.5)\nprint(\"EDGAR_URL=\" + (url or \"\"))\nprint(\"EDGAR_SECRET=\" + SECRET)\n" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "Set both values in the CRM (Vercel \u2192 crm-agent \u2192 Environment Variables, or the local `.env`) and redeploy or restart the agent. `sec_search_companies` then answers from here." + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/services/edgar/edgar_service/__init__.py b/services/edgar/edgar_service/__init__.py new file mode 100644 index 000000000..1cf6267ae --- /dev/null +++ b/services/edgar/edgar_service/__init__.py @@ -0,0 +1 @@ +VERSION = "0.1.0" diff --git a/services/edgar/edgar_service/__pycache__/__init__.cpython-311.pyc b/services/edgar/edgar_service/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 000000000..61d0fd2f1 Binary files /dev/null and b/services/edgar/edgar_service/__pycache__/__init__.cpython-311.pyc differ diff --git a/services/edgar/edgar_service/__pycache__/app.cpython-311.pyc b/services/edgar/edgar_service/__pycache__/app.cpython-311.pyc new file mode 100644 index 000000000..3a30b1c08 Binary files /dev/null and b/services/edgar/edgar_service/__pycache__/app.cpython-311.pyc differ diff --git a/services/edgar/edgar_service/__pycache__/auth.cpython-311.pyc b/services/edgar/edgar_service/__pycache__/auth.cpython-311.pyc new file mode 100644 index 000000000..325bacb7e Binary files /dev/null and b/services/edgar/edgar_service/__pycache__/auth.cpython-311.pyc differ diff --git a/services/edgar/edgar_service/__pycache__/config.cpython-311.pyc b/services/edgar/edgar_service/__pycache__/config.cpython-311.pyc new file mode 100644 index 000000000..4a0cea528 Binary files /dev/null and b/services/edgar/edgar_service/__pycache__/config.cpython-311.pyc differ diff --git a/services/edgar/edgar_service/__pycache__/sec.cpython-311.pyc b/services/edgar/edgar_service/__pycache__/sec.cpython-311.pyc new file mode 100644 index 000000000..6b89e933d Binary files /dev/null and b/services/edgar/edgar_service/__pycache__/sec.cpython-311.pyc differ diff --git a/services/edgar/edgar_service/__pycache__/values.cpython-311.pyc b/services/edgar/edgar_service/__pycache__/values.cpython-311.pyc new file mode 100644 index 000000000..2ba1330bc Binary files /dev/null and b/services/edgar/edgar_service/__pycache__/values.cpython-311.pyc differ diff --git a/services/edgar/edgar_service/app.py b/services/edgar/edgar_service/app.py new file mode 100644 index 000000000..bc57190ff --- /dev/null +++ b/services/edgar/edgar_service/app.py @@ -0,0 +1,117 @@ +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager + +from fastapi import Depends, FastAPI, HTTPException, Query, Request +from fastapi.responses import JSONResponse + +from . import VERSION, sec +from .auth import require_secret +from .config import settings + + +@asynccontextmanager +async def lifespan(_: FastAPI) -> AsyncIterator[None]: + sec.configure(settings()) + yield + + +app = FastAPI(title="CRM EDGAR service", version=VERSION, docs_url=None, redoc_url=None, lifespan=lifespan) + + +@app.exception_handler(HTTPException) +async def http_error(_: Request, error: HTTPException) -> JSONResponse: + detail = error.detail if isinstance(error.detail, dict) else {"reason": str(error.detail)} + return JSONResponse(status_code=error.status_code, content=detail) + + +@app.exception_handler(sec.NotFound) +async def not_found(_: Request, error: sec.NotFound) -> JSONResponse: + return JSONResponse(status_code=404, content={"reason": str(error)}) + + +@app.exception_handler(sec.Upstream) +async def upstream(_: Request, error: sec.Upstream) -> JSONResponse: + return JSONResponse(status_code=502, content={"reason": str(error)}) + + +@app.exception_handler(Exception) +async def unexpected(_: Request, error: Exception) -> JSONResponse: + return JSONResponse(status_code=502, content={"reason": f"SEC lookup failed: {error}"}) + + +@app.get("/health") +def health() -> dict: + return { + "ok": True, + "version": VERSION, + "edgartools": sec.edgartools_version(), + "identitySet": sec.identity_set(), + } + + +guarded = [Depends(require_secret)] + + +@app.get("/companies/search", dependencies=guarded) +def companies_search(q: str = Query(min_length=1), limit: int = Query(10, ge=1, le=settings().max_search)) -> dict: + return {"companies": sec.search_companies(q, limit)} + + +@app.get("/companies/{key}", dependencies=guarded) +def company(key: str) -> dict: + return sec.company(key) + + +@app.get("/companies/{key}/filings", dependencies=guarded) +def company_filings( + key: str, + form: str | None = None, + from_: str | None = Query(None, alias="from"), + to: str | None = None, + limit: int = Query(20, ge=1, le=settings().max_filings), +) -> dict: + return sec.filings(key, form, from_, to, limit) + + +@app.get("/filings/search", dependencies=guarded) +def filings_search( + q: str = Query(min_length=2), + form: str | None = None, + from_: str | None = Query(None, alias="from"), + to: str | None = None, + limit: int = Query(20, ge=1, le=settings().max_filings), +) -> dict: + return sec.search_filings(q, form, from_, to, limit) + + +@app.get("/companies/{key}/owners", dependencies=guarded) +def company_owners( + key: str, + minPercent: float = Query(5, ge=0, le=100), + form: str = Query("all", pattern="^(13D|13G|all)$"), + limit: int = Query(20, ge=1, le=settings().max_owners), +) -> dict: + return sec.owners(key, minPercent, form, limit) + + +@app.get("/companies/{key}/insiders", dependencies=guarded) +def company_insiders(key: str, limit: int = Query(20, ge=1, le=settings().max_insiders)) -> dict: + return sec.insiders(key, limit) + + +@app.get("/companies/{key}/proxy", dependencies=guarded) +def company_proxy(key: str, years: int = Query(3, ge=1, le=settings().max_years)) -> dict: + return sec.proxy(key, years) + + +@app.get("/compensation/compare", dependencies=guarded) +def compensation_compare( + tickers: str = Query(min_length=1), + years: int = Query(3, ge=1, le=settings().max_years), +) -> dict: + names = [t.strip().upper() for t in tickers.split(",") if t.strip()] + if not names: + raise HTTPException(status_code=422, detail={"reason": "Give at least one ticker."}) + if len(names) > settings().max_tickers: + raise HTTPException(status_code=422, detail={"reason": f"At most {settings().max_tickers} tickers."}) + return {"rows": sec.compare(names, years)} diff --git a/services/edgar/edgar_service/auth.py b/services/edgar/edgar_service/auth.py new file mode 100644 index 000000000..33be65e1c --- /dev/null +++ b/services/edgar/edgar_service/auth.py @@ -0,0 +1,24 @@ +import hmac + +from fastapi import HTTPException, Request + +from .config import settings + +LOOPBACK = {"127.0.0.1", "::1", "localhost"} + + +def require_secret(request: Request) -> None: + secret = settings().secret + if not secret: + host = request.client.host if request.client else "" + if host in LOOPBACK: + return + raise HTTPException( + status_code=401, + detail={"reason": "EDGAR_SECRET is not set, so only loopback callers are accepted."}, + ) + + header = request.headers.get("authorization", "") + given = header[7:].strip() if header.lower().startswith("bearer ") else "" + if not given or not hmac.compare_digest(given, secret): + raise HTTPException(status_code=401, detail={"reason": "Bad or missing bearer secret."}) diff --git a/services/edgar/edgar_service/config.py b/services/edgar/edgar_service/config.py new file mode 100644 index 000000000..79a864b04 --- /dev/null +++ b/services/edgar/edgar_service/config.py @@ -0,0 +1,33 @@ +import os +from dataclasses import dataclass + + +@dataclass(frozen=True) +class Settings: + identity: str + secret: str + port: int + data_dir: str + max_search: int + max_filings: int + max_owner_filings: int + max_owners: int + max_insiders: int + max_years: int + max_tickers: int + + +def settings() -> Settings: + return Settings( + identity=os.environ.get("EDGAR_IDENTITY", "").strip(), + secret=os.environ.get("EDGAR_SECRET", "").strip(), + port=int(os.environ.get("EDGAR_PORT", "2100")), + data_dir=os.environ.get("EDGAR_DATA_DIR", "").strip(), + max_search=25, + max_filings=100, + max_owner_filings=60, + max_owners=50, + max_insiders=100, + max_years=5, + max_tickers=10, + ) diff --git a/services/edgar/edgar_service/sec.py b/services/edgar/edgar_service/sec.py new file mode 100644 index 000000000..d9aa56cab --- /dev/null +++ b/services/edgar/edgar_service/sec.py @@ -0,0 +1,460 @@ +import os +import re +from datetime import date +from importlib import metadata +from typing import Any + +import edgar +from edgar import Company, CompanyNotFoundError, find_company, get_identity, set_identity +from edgar import search_filings as efts_search + +from .config import Settings +from .values import cik_text, day, number, rows, text, whole + +BROWSE_URL = "https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK=" +OWNER_FORMS = { + "13D": ["SCHEDULE 13D", "SC 13D"], + "13G": ["SCHEDULE 13G", "SC 13G"], +} +INSIDER_FORMS = ["3", "4", "5"] +PROXY_FORM = "DEF 14A" +EFTS_FIRST_DAY = "2001-01-01" +DISPLAY_NAME = re.compile(r"^(.*?)\s+\((?:[A-Z.\-]+\)\s+\()?CIK") +HOLDER_SHARE_KEYS = ("shares", "amount", "shares_beneficially_owned", "amount_beneficially_owned") + +_settings: Settings | None = None + + +class NotFound(Exception): + pass + + +class Upstream(Exception): + pass + + +def configure(settings: Settings) -> None: + global _settings + _settings = settings + if settings.data_dir: + os.environ.setdefault("EDGAR_LOCAL_DATA_DIR", settings.data_dir) + if settings.identity: + set_identity(settings.identity) + + +def edgartools_version() -> str: + try: + return metadata.version("edgartools") + except metadata.PackageNotFoundError: + return getattr(edgar, "__version__", "unknown") + + +def identity_set() -> bool: + try: + return bool(get_identity()) + except Exception: + return False + + +def _limits() -> Settings: + if _settings is None: + raise Upstream("The service is not configured.") + return _settings + + +def _company(key: str) -> Any: + cleaned = key.strip() + try: + company = Company(int(cleaned)) if cleaned.isdigit() else Company(cleaned.upper()) + except CompanyNotFoundError as error: + raise NotFound(f"No SEC filer matches {cleaned}.") from error + except Exception as error: + if "not found" in str(error).lower(): + raise NotFound(f"No SEC filer matches {cleaned}.") from error + raise Upstream(f"SEC lookup failed for {cleaned}: {error}") from error + if company is None or getattr(company, "not_found", False): + raise NotFound(f"No SEC filer matches {cleaned}.") + return company + + +def _match(company: Any) -> dict[str, Any]: + data = company.data + tickers = list(getattr(data, "tickers", None) or []) + exchanges = list(getattr(data, "exchanges", None) or []) + return { + "cik": cik_text(company.cik), + "name": text(company.name) or f"CIK {cik_text(company.cik)}", + "ticker": tickers[0] if tickers else None, + "exchange": exchanges[0] if exchanges else None, + } + + +def search_companies(query: str, limit: int) -> list[dict[str, Any]]: + found: dict[str, dict[str, Any]] = {} + cleaned = query.strip() + + if cleaned.isdigit() or (cleaned.isalpha() and len(cleaned) <= 6): + try: + match = _match(_company(cleaned)) + found[match["cik"]] = match + except (NotFound, Upstream): + pass + + try: + results = find_company(cleaned, top_n=limit) + for index in range(len(results)): + match = _match(results[index]) + found.setdefault(match["cik"], match) + except Exception as error: + if not found: + raise Upstream(f"SEC company search failed: {error}") from error + + return list(found.values())[:limit] + + +def _address(company: Any) -> dict[str, Any] | None: + try: + address = company.business_address() + except Exception: + return None + if address is None or getattr(address, "empty", False): + return None + street = " ".join(part for part in [text(address.street1), text(address.street2)] if part) + return { + "street": street or None, + "city": text(address.city), + "state": text(address.state_or_country), + "zip": text(address.zipcode), + } + + +def company(key: str) -> dict[str, Any]: + entity = _company(key) + data = entity.data + former = getattr(data, "former_names", None) or [] + return { + "cik": cik_text(entity.cik), + "name": text(entity.name) or f"CIK {cik_text(entity.cik)}", + "tickers": [t for t in (getattr(data, "tickers", None) or []) if t], + "exchanges": [e for e in (getattr(data, "exchanges", None) or []) if e], + "sic": text(getattr(data, "sic", None)), + "sicDescription": text(getattr(data, "sic_description", None)), + "stateOfIncorporation": text(getattr(data, "state_of_incorporation", None)), + "fiscalYearEnd": text(getattr(data, "fiscal_year_end", None)), + "category": text(getattr(data, "category", None)), + "businessAddress": _address(entity), + "website": text(getattr(data, "website", None)), + "formerNames": [name for name in (text(item.get("name")) for item in former if isinstance(item, dict)) if name], + "url": f"{BROWSE_URL}{cik_text(entity.cik)}", + } + + +def _filing_row(filing: Any) -> dict[str, Any]: + return { + "accession": text(filing.accession_no) or "", + "form": text(filing.form) or "", + "filedAt": day(filing.filing_date) or "", + "reportDate": day(getattr(filing, "report_date", None)), + "description": text(getattr(filing, "primary_doc_description", None)), + "url": filing.homepage_url, + } + + +def _entity_filings(entity: Any, form: Any, start: str | None, end: str | None) -> Any: + try: + filings = entity.get_filings(form=form) if form else entity.get_filings() + if filings is not None and (start or end): + filings = filings.filter(filing_date=f"{start or ''}:{end or ''}") + except Exception as error: + raise Upstream(f"SEC filings lookup failed: {error}") from error + return filings + + +def filings(key: str, form: str | None, start: str | None, end: str | None, limit: int) -> dict[str, Any]: + entity = _company(key) + listing = _entity_filings(entity, form.strip() if form else None, start, end) + if listing is None or getattr(listing, "empty", False): + return {"filings": [], "truncated": False} + page = list(listing.head(limit + 1)) + return { + "filings": [_filing_row(filing) for filing in page[:limit]], + "truncated": len(page) > limit, + } + + +def _hit_company(hit: Any) -> dict[str, Any]: + raw = text(getattr(hit, "company", None)) or "" + match = DISPLAY_NAME.match(raw) + return {"cik": cik_text(hit.cik), "name": (match.group(1) if match else raw) or f"CIK {cik_text(hit.cik)}"} + + +def search_filings(query: str, form: str | None, start: str | None, end: str | None, limit: int) -> dict[str, Any]: + if start or end: + start = start or EFTS_FIRST_DAY + end = end or date.today().isoformat() + try: + results = efts_search( + query.strip(), + forms=form.strip() if form else None, + start_date=start, + end_date=end, + limit=limit, + ) + hits = list(iter(results.sort_by("filed")))[:limit] + total = whole(getattr(results, "total", None)) or len(hits) + except Exception as error: + raise Upstream(f"SEC full-text search failed: {error}") from error + + out = [] + for hit in hits: + accession = text(hit.accession_number) or "" + cik = cik_text(hit.cik) + out.append( + { + "accession": accession, + "form": text(hit.form) or "", + "filedAt": day(hit.filed) or "", + "reportDate": day(getattr(hit, "period", None)), + "description": text(getattr(hit, "file_description", None)), + "url": f"https://www.sec.gov/Archives/edgar/data/{cik}/{accession.replace('-', '')}/{accession}-index.html", + "company": _hit_company(hit), + } + ) + return {"filings": out, "total": total} + + +def owners(key: str, min_percent: float, form: str, limit: int) -> dict[str, Any]: + entity = _company(key) + forms = OWNER_FORMS["13D"] + OWNER_FORMS["13G"] if form == "all" else OWNER_FORMS[form] + listing = _entity_filings(entity, forms, None, None) + if listing is None or getattr(listing, "empty", False): + return {"owners": [], "filingsRead": 0} + + seen: set[str] = set() + out: list[dict[str, Any]] = [] + read = 0 + for filing in listing.head(_limits().max_owner_filings): + read += 1 + try: + schedule = filing.obj() + except Exception: + continue + persons = getattr(schedule, "reporting_persons", None) or [] + items = getattr(schedule, "items", None) + purpose = text(getattr(items, "item4_purpose_of_transaction", None)) if "13D" in str(filing.form) else None + for person in persons: + name = text(getattr(person, "name", None)) + if not name or name.lower() in seen: + continue + seen.add(name.lower()) + percent = number(getattr(person, "percent_of_class", None)) + if percent is None or percent < min_percent: + continue + out.append( + { + "filer": name, + "form": text(filing.form) or "", + "filedAt": day(filing.filing_date) or "", + "shares": number(getattr(person, "aggregate_amount", None)), + "percent": percent, + "soleVoting": number(getattr(person, "sole_voting_power", None)), + "sharedVoting": number(getattr(person, "shared_voting_power", None)), + "purpose": purpose, + "url": filing.homepage_url, + } + ) + if len(out) >= limit: + return {"owners": out, "filingsRead": read} + return {"owners": out, "filingsRead": read} + + +def insiders(key: str, limit: int) -> dict[str, Any]: + entity = _company(key) + listing = _entity_filings(entity, INSIDER_FORMS, None, None) + if listing is None or getattr(listing, "empty", False): + return {"transactions": []} + + out = [] + for filing in listing.head(limit): + try: + form = filing.obj() + summary = form.get_ownership_summary() + except Exception: + continue + activities = list(getattr(summary, "transactions", None) or []) + primary = activities[0] if activities else None + insider = text(getattr(form, "insider_name", None)) + if not insider: + continue + out.append( + { + "insider": insider, + "title": text(getattr(form, "position", None)), + "form": text(filing.form) or "", + "filedAt": day(filing.filing_date) or "", + "kind": text(getattr(primary, "transaction_type", None)) or text(getattr(summary, "primary_activity", None)), + "shares": number(getattr(primary, "shares", None)), + "price": number(getattr(primary, "price_per_share", None)), + "url": filing.homepage_url, + } + ) + return {"transactions": out} + + +def _latest_proxy(entity: Any) -> tuple[Any, Any]: + listing = _entity_filings(entity, PROXY_FORM, None, None) + if listing is None or getattr(listing, "empty", False): + raise NotFound(f"No {PROXY_FORM} on file for {text(entity.name)}.") + filing = list(listing.head(1))[0] + try: + statement = filing.obj() + except Exception as error: + raise Upstream(f"The proxy statement could not be parsed: {error}") from error + if statement is None or not hasattr(statement, "peo_name"): + raise Upstream("The proxy statement could not be parsed.") + return filing, statement + + +def _by_year(frame: Any, years: int) -> list[dict[str, Any]]: + items = rows(frame) + items.sort(key=lambda row: str(row.get("fiscal_year_end") or ""), reverse=True) + return items[:years] + + +def _holder_shares(row: dict[str, Any]) -> float | None: + for key in HOLDER_SHARE_KEYS: + if key in row: + return number(row[key]) + return None + + +def proxy(key: str, years: int) -> dict[str, Any]: + entity = _company(key) + filing, statement = _latest_proxy(entity) + ratio = getattr(statement, "ceo_pay_ratio", None) + executives = [] + for row in rows(getattr(statement, "summary_compensation_table", None)): + name = text(row.get("name")) + if not name: + continue + executives.append( + { + "name": name, + "title": text(row.get("title")), + "year": whole(row.get("year")), + "salary": number(row.get("salary")), + "bonus": number(row.get("bonus")), + "stockAwards": number(row.get("stock_awards")), + "optionAwards": number(row.get("option_awards")), + "nonEquityIncentive": number(row.get("non_equity_incentive")), + "otherCompensation": number(row.get("other_compensation")), + "total": number(row.get("total")), + } + ) + holders = [] + for row in rows(getattr(statement, "beneficial_ownership", None)): + name = text(row.get("holder_name") or row.get("name")) + if not name: + continue + holders.append({"name": name, "percentOfClass": number(row.get("percent_of_class")), "shares": _holder_shares(row)}) + proposals = [ + { + "number": whole(getattr(item, "number", None)), + "description": text(getattr(item, "description", None)) or "", + "type": text(getattr(item, "proposal_type", None)), + } + for item in (getattr(statement, "voting_proposals", None) or []) + ] + return { + "accession": text(filing.accession_no) or "", + "filedAt": day(filing.filing_date) or "", + "url": filing.homepage_url, + "peo": { + "name": text(getattr(statement, "peo_name", None)), + "totalComp": number(getattr(statement, "peo_total_comp", None)), + "actuallyPaidComp": number(getattr(statement, "peo_actually_paid_comp", None)), + }, + "neoAverage": { + "totalComp": number(getattr(statement, "neo_avg_total_comp", None)), + "actuallyPaidComp": number(getattr(statement, "neo_avg_actually_paid_comp", None)), + }, + "compensationByYear": [ + { + "fiscalYearEnd": day(row.get("fiscal_year_end")), + "peoTotalComp": number(row.get("peo_total_comp")), + "peoActuallyPaidComp": number(row.get("peo_actually_paid_comp")), + "neoAverageTotalComp": number(row.get("neo_avg_total_comp")), + "neoAverageActuallyPaidComp": number(row.get("neo_avg_actually_paid_comp")), + } + for row in _by_year(getattr(statement, "executive_compensation", None), years) + ], + "payVsPerformance": [ + { + "fiscalYearEnd": day(row.get("fiscal_year_end")), + "peoActuallyPaidComp": number(row.get("peo_actually_paid_comp")), + "neoAverageActuallyPaidComp": number(row.get("neo_avg_actually_paid_comp")), + "tsr": number(row.get("total_shareholder_return")), + "peerTsr": number(row.get("peer_group_tsr")), + "netIncome": number(row.get("net_income")), + "selectedMeasureValue": number(row.get("company_selected_measure_value")), + } + for row in _by_year(getattr(statement, "pay_vs_performance", None), years) + ], + "executives": executives, + "holders": holders, + "proposals": proposals, + "performanceMeasures": [m for m in (text(item) for item in (getattr(statement, "performance_measures", None) or [])) if m], + "selectedMeasureName": text(getattr(statement, "company_selected_measure", None)), + "ceoPayRatio": None + if ratio is None + else { + "ceo": number(getattr(ratio, "ceo_compensation", None)), + "medianEmployee": number(getattr(ratio, "median_employee_compensation", None)), + "ratio": number(getattr(ratio, "ratio", None)), + }, + "insiderTradingPolicyAdopted": getattr(statement, "insider_trading_policy_adopted", None) + if isinstance(getattr(statement, "insider_trading_policy_adopted", None), bool) + else None, + } + + +def compare(tickers: list[str], years: int) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + for ticker in tickers: + try: + entity = _company(ticker) + statement = proxy(ticker, years) + except (NotFound, Upstream) as error: + out.append( + { + "ticker": ticker, + "cik": None, + "name": None, + "fiscalYearEnd": None, + "peoName": None, + "peoTotalComp": None, + "peoActuallyPaidComp": None, + "tsr": None, + "netIncome": None, + "reason": str(error), + } + ) + continue + performance = {row["fiscalYearEnd"]: row for row in statement["payVsPerformance"]} + for row in statement["compensationByYear"]: + year = performance.get(row["fiscalYearEnd"], {}) + out.append( + { + "ticker": ticker, + "cik": cik_text(entity.cik), + "name": text(entity.name), + "fiscalYearEnd": row["fiscalYearEnd"], + "peoName": statement["peo"]["name"], + "peoTotalComp": row["peoTotalComp"], + "peoActuallyPaidComp": row["peoActuallyPaidComp"], + "tsr": year.get("tsr"), + "netIncome": year.get("netIncome"), + "reason": None, + } + ) + return out diff --git a/services/edgar/edgar_service/values.py b/services/edgar/edgar_service/values.py new file mode 100644 index 000000000..f61ad5ebe --- /dev/null +++ b/services/edgar/edgar_service/values.py @@ -0,0 +1,62 @@ +import math +from datetime import date, datetime +from decimal import Decimal +from typing import Any + + +def number(value: Any) -> float | None: + if value is None or isinstance(value, bool): + return None + if isinstance(value, Decimal): + value = float(value) + if isinstance(value, (int, float)): + return None if isinstance(value, float) and math.isnan(value) else float(value) + try: + parsed = float(str(value).replace(",", "").replace("$", "").strip()) + except ValueError: + return None + return None if math.isnan(parsed) else parsed + + +def whole(value: Any) -> int | None: + parsed = number(value) + return None if parsed is None else int(parsed) + + +def text(value: Any) -> str | None: + if value is None: + return None + if isinstance(value, float) and math.isnan(value): + return None + cleaned = " ".join(str(value).split()) + return cleaned or None + + +def day(value: Any) -> str | None: + if value is None: + return None + if isinstance(value, datetime): + return value.date().isoformat() + if isinstance(value, date): + return value.isoformat() + cleaned = str(value).strip()[:10] + try: + return date.fromisoformat(cleaned).isoformat() + except ValueError: + return None + + +def rows(frame: Any) -> list[dict[str, Any]]: + if frame is None: + return [] + to_dict = getattr(frame, "to_dict", None) + if to_dict is None: + return [] + try: + return list(to_dict("records")) + except (TypeError, ValueError): + return [] + + +def cik_text(value: Any) -> str: + return str(value).strip().lstrip("0") or "0" diff --git a/services/edgar/pyproject.toml b/services/edgar/pyproject.toml new file mode 100644 index 000000000..302d1825c --- /dev/null +++ b/services/edgar/pyproject.toml @@ -0,0 +1,24 @@ +[project] +name = "edgar-service" +version = "0.1.0" +description = "SEC EDGAR research for the CRM, on edgartools" +requires-python = ">=3.11" +dependencies = [ + "edgartools==5.56.0", + "fastapi==0.141.1", + "uvicorn[standard]==0.41.0", + "pydantic>=2.9", +] + +[project.optional-dependencies] +dev = ["pytest>=8.3", "httpx>=0.28"] + +[build-system] +requires = ["setuptools>=70"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +include = ["edgar_service*"] + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/services/edgar/requirements.txt b/services/edgar/requirements.txt new file mode 100644 index 000000000..d8f7ac2d2 --- /dev/null +++ b/services/edgar/requirements.txt @@ -0,0 +1,4 @@ +edgartools==5.56.0 +fastapi==0.141.1 +uvicorn[standard]==0.41.0 +pydantic>=2.9 diff --git a/services/edgar/tests/__init__.py b/services/edgar/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/services/edgar/tests/__pycache__/__init__.cpython-311.pyc b/services/edgar/tests/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 000000000..d6e5dc804 Binary files /dev/null and b/services/edgar/tests/__pycache__/__init__.cpython-311.pyc differ diff --git a/services/edgar/tests/__pycache__/conftest.cpython-311-pytest-9.1.1.pyc b/services/edgar/tests/__pycache__/conftest.cpython-311-pytest-9.1.1.pyc new file mode 100644 index 000000000..492e8e845 Binary files /dev/null and b/services/edgar/tests/__pycache__/conftest.cpython-311-pytest-9.1.1.pyc differ diff --git a/services/edgar/tests/__pycache__/test_app.cpython-311-pytest-9.1.1.pyc b/services/edgar/tests/__pycache__/test_app.cpython-311-pytest-9.1.1.pyc new file mode 100644 index 000000000..f86423c29 Binary files /dev/null and b/services/edgar/tests/__pycache__/test_app.cpython-311-pytest-9.1.1.pyc differ diff --git a/services/edgar/tests/__pycache__/test_sec.cpython-311-pytest-9.1.1.pyc b/services/edgar/tests/__pycache__/test_sec.cpython-311-pytest-9.1.1.pyc new file mode 100644 index 000000000..d4fbc0e98 Binary files /dev/null and b/services/edgar/tests/__pycache__/test_sec.cpython-311-pytest-9.1.1.pyc differ diff --git a/services/edgar/tests/conftest.py b/services/edgar/tests/conftest.py new file mode 100644 index 000000000..4fc0d0a01 --- /dev/null +++ b/services/edgar/tests/conftest.py @@ -0,0 +1,21 @@ +import os + +import pytest +from fastapi.testclient import TestClient + +os.environ.setdefault("EDGAR_IDENTITY", "Test Runner test@example.test") + + +@pytest.fixture +def client(monkeypatch): + monkeypatch.setenv("EDGAR_SECRET", "s3cret") + from edgar_service import sec + from edgar_service.app import app + + monkeypatch.setattr(sec, "identity_set", lambda: True) + monkeypatch.setattr(sec, "edgartools_version", lambda: "5.56.0") + with TestClient(app, raise_server_exceptions=False) as test_client: + yield test_client + + +AUTH = {"authorization": "Bearer s3cret"} diff --git a/services/edgar/tests/smoke_live.py b/services/edgar/tests/smoke_live.py new file mode 100644 index 000000000..92e49e338 --- /dev/null +++ b/services/edgar/tests/smoke_live.py @@ -0,0 +1,37 @@ +import json +import os +import sys + +if os.environ.get("EDGAR_LIVE") != "1": + print("Set EDGAR_LIVE=1 to run the live smoke test.") + sys.exit(0) + +from edgar_service import sec +from edgar_service.config import settings + +sec.configure(settings()) +if not sec.identity_set(): + print("EDGAR_IDENTITY is required.") + sys.exit(1) + +checks = { + "search": lambda: sec.search_companies("Apple", 3), + "company": lambda: sec.company("320193"), + "filings": lambda: sec.filings("320193", "DEF 14A", None, None, 3), + "search_filings": lambda: sec.search_filings("Apple", "DEF 14A", "2025-01-01", None, 3), + "owners": lambda: sec.owners("320193", 5, "all", 5), + "insiders": lambda: sec.insiders("320193", 3), + "proxy": lambda: sec.proxy("320193", 2), + "compare": lambda: sec.compare(["AAPL", "MSFT"], 1), +} + +failed = 0 +for name, run in checks.items(): + try: + result = run() + print(f"ok {name}: {json.dumps(result, default=str)[:160]}") + except Exception as error: + failed += 1 + print(f"FAIL {name}: {error}") + +sys.exit(1 if failed else 0) diff --git a/services/edgar/tests/test_app.py b/services/edgar/tests/test_app.py new file mode 100644 index 000000000..86b1fc584 --- /dev/null +++ b/services/edgar/tests/test_app.py @@ -0,0 +1,67 @@ +from edgar_service import sec +from tests.conftest import AUTH + + +def test_health_is_open(client): + response = client.get("/health") + assert response.status_code == 200 + assert response.json() == {"ok": True, "version": "0.1.0", "edgartools": "5.56.0", "identitySet": True} + + +def test_routes_need_the_secret(client): + assert client.get("/companies/search?q=apple").status_code == 401 + assert client.get("/companies/search?q=apple", headers={"authorization": "Bearer wrong"}).status_code == 401 + + +def test_search_returns_matches(client, monkeypatch): + monkeypatch.setattr(sec, "search_companies", lambda q, limit: [{"cik": "320193", "name": "Apple Inc.", "ticker": "AAPL", "exchange": "Nasdaq"}]) + response = client.get("/companies/search?q=apple&limit=3", headers=AUTH) + assert response.status_code == 200 + assert response.json()["companies"][0]["ticker"] == "AAPL" + + +def test_not_found_is_404_with_a_reason(client, monkeypatch): + def missing(key): + raise sec.NotFound(f"No SEC filer matches {key}.") + + monkeypatch.setattr(sec, "company", missing) + response = client.get("/companies/ZZZZ", headers=AUTH) + assert response.status_code == 404 + assert response.json() == {"reason": "No SEC filer matches ZZZZ."} + + +def test_upstream_failure_is_502(client, monkeypatch): + def broken(key, form, start, end, limit): + raise sec.Upstream("SEC filings lookup failed: boom") + + monkeypatch.setattr(sec, "filings", broken) + response = client.get("/companies/320193/filings?form=10-K", headers=AUTH) + assert response.status_code == 502 + assert "boom" in response.json()["reason"] + + +def test_compare_splits_and_bounds_tickers(client, monkeypatch): + seen = {} + + def compare(tickers, years): + seen["tickers"] = tickers + seen["years"] = years + return [] + + monkeypatch.setattr(sec, "compare", compare) + response = client.get("/compensation/compare?tickers=aapl,%20msft&years=2", headers=AUTH) + assert response.status_code == 200 + assert seen == {"tickers": ["AAPL", "MSFT"], "years": 2} + + too_many = ",".join(f"T{i}" for i in range(11)) + assert client.get(f"/compensation/compare?tickers={too_many}", headers=AUTH).status_code == 422 + + +def test_any_other_failure_is_502(client, monkeypatch): + def broken(key): + raise RuntimeError("connection refused") + + monkeypatch.setattr(sec, "company", broken) + response = client.get("/companies/320193", headers=AUTH) + assert response.status_code == 502 + assert "connection refused" in response.json()["reason"] diff --git a/services/edgar/tests/test_sec.py b/services/edgar/tests/test_sec.py new file mode 100644 index 000000000..51e2a7859 --- /dev/null +++ b/services/edgar/tests/test_sec.py @@ -0,0 +1,163 @@ +from dataclasses import dataclass, field +from decimal import Decimal + +import pandas as pd + +from edgar_service import sec +from edgar_service.config import Settings +from edgar_service.values import day, number, text + + +@dataclass +class Person: + name: str + percent_of_class: float | None + aggregate_amount: int | None = None + sole_voting_power: int | None = None + shared_voting_power: int | None = None + + +@dataclass +class Items: + item4_purpose_of_transaction: str | None = None + + +@dataclass +class Schedule: + reporting_persons: list + items: Items = field(default_factory=Items) + + +@dataclass +class Filing: + form: str + filing_date: str + accession_no: str + homepage_url: str + schedule: Schedule + report_date: str | None = None + primary_doc_description: str | None = None + + def obj(self): + return self.schedule + + +class Listing: + def __init__(self, filings): + self.filings = filings + self.empty = not filings + + def head(self, n): + return self.filings[:n] + + def filter(self, **_): + return self + + +class Entity: + cik = 320193 + name = "Apple Inc." + + def __init__(self, filings): + self.filings = filings + + def get_filings(self, form=None): + return Listing(self.filings) + + +def configure(): + sec.configure(Settings("Test test@example.test", "", 2100, "", 25, 100, 60, 50, 100, 5, 10)) + + +def test_values_fold_decimals_nan_and_dates(): + assert number(Decimal("74294811.0")) == 74294811.0 + assert number(float("nan")) is None + assert number("[F1]") is None + assert number("1,000") == 1000.0 + assert text("Net Sales") == "Net Sales" + assert day("2026-01-08T00:00:00") == "2026-01-08" + assert day("bad") is None + + +def test_owners_keep_the_newest_filing_per_holder_and_apply_the_threshold(monkeypatch): + configure() + newest = Filing("SCHEDULE 13G/A", "2026-02-10", "a-1", "https://sec.test/a-1", Schedule([Person("Vanguard", 4.9, 1)])) + older = Filing("SCHEDULE 13G", "2025-02-10", "a-0", "https://sec.test/a-0", Schedule([Person("Vanguard", 7.5, 2), Person("BlackRock", 6.1, 3, 10, 0)])) + activist = Filing("SCHEDULE 13D", "2024-06-01", "d-1", "https://sec.test/d-1", Schedule([Person("Elliott", 5.5, 4)], Items("Seek board seats"))) + monkeypatch.setattr(sec, "_company", lambda key: Entity([newest, older, activist])) + + result = sec.owners("320193", 5, "all", 20) + + assert [o["filer"] for o in result["owners"]] == ["BlackRock", "Elliott"] + assert result["owners"][1]["purpose"] == "Seek board seats" + assert result["owners"][0]["purpose"] is None + assert result["filingsRead"] == 3 + + +def test_proxy_reads_the_tables(monkeypatch): + configure() + + class Ratio: + ceo_compensation = 74294811 + median_employee_compensation = 139483 + ratio = 533 + + class Proposal: + number = 1 + description = "Election of Directors" + proposal_type = "director_election" + + class Statement: + peo_name = "Mr. Cook" + peo_total_comp = Decimal("74294811.0") + peo_actually_paid_comp = Decimal("108423733.0") + neo_avg_total_comp = Decimal("23812358.0") + neo_avg_actually_paid_comp = Decimal("34125743.0") + executive_compensation = pd.DataFrame( + [ + {"fiscal_year_end": "2024-09-28", "peo_total_comp": 74609802.0, "peo_actually_paid_comp": 168980568.0, "neo_avg_total_comp": 27178896.0, "neo_avg_actually_paid_comp": 58633525.0}, + {"fiscal_year_end": "2025-09-27", "peo_total_comp": 74294811.0, "peo_actually_paid_comp": 108423733.0, "neo_avg_total_comp": 23812358.0, "neo_avg_actually_paid_comp": 34125743.0}, + ] + ) + pay_vs_performance = pd.DataFrame( + [{"fiscal_year_end": "2025-09-27", "peo_actually_paid_comp": 108423733.0, "neo_avg_actually_paid_comp": 34125743.0, "total_shareholder_return": 233.88, "peer_group_tsr": 279.51, "net_income": 112010000000.0, "company_selected_measure_value": 416161000000.0}] + ) + summary_compensation_table = pd.DataFrame( + [{"name": "Tim Cook", "title": "CEO", "year": 2025, "salary": 3000000, "bonus": None, "stock_awards": 57535293, "option_awards": None, "non_equity_incentive": 12000000, "pension_change": None, "other_compensation": 1759518, "total": 74294811}] + ) + beneficial_ownership = pd.DataFrame([{"holder_name": "The Vanguard Group", "percent_of_class": 9.63}]) + voting_proposals = [Proposal()] + performance_measures = ["Net Sales", "Operating Income"] + company_selected_measure = "Net Sales" + ceo_pay_ratio = Ratio() + insider_trading_policy_adopted = True + + class ProxyFiling(Filing): + def obj(self): + return Statement() + + filing = ProxyFiling("DEF 14A", "2026-01-08", "0001308179-26-000008", "https://sec.test/proxy", Schedule([])) + monkeypatch.setattr(sec, "_company", lambda key: Entity([filing])) + + result = sec.proxy("320193", 1) + + assert result["peo"] == {"name": "Mr. Cook", "totalComp": 74294811.0, "actuallyPaidComp": 108423733.0} + assert [row["fiscalYearEnd"] for row in result["compensationByYear"]] == ["2025-09-27"] + assert result["executives"][0]["name"] == "Tim Cook" + assert result["executives"][0]["bonus"] is None + assert result["holders"] == [{"name": "The Vanguard Group", "percentOfClass": 9.63, "shares": None}] + assert result["proposals"][0]["type"] == "director_election" + assert result["performanceMeasures"] == ["Net Sales", "Operating Income"] + assert result["ceoPayRatio"]["ratio"] == 533.0 + assert result["insiderTradingPolicyAdopted"] is True + + +def test_proxy_without_a_filing_is_not_found(monkeypatch): + configure() + monkeypatch.setattr(sec, "_company", lambda key: Entity([])) + try: + sec.proxy("320193", 1) + except sec.NotFound as error: + assert "No DEF 14A" in str(error) + else: + raise AssertionError("expected NotFound") diff --git a/services/edgar/tunnel.sh b/services/edgar/tunnel.sh new file mode 100755 index 000000000..219453afb --- /dev/null +++ b/services/edgar/tunnel.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env sh +set -eu + +HERE=$(cd "$(dirname "$0")" && pwd) +PORT=${EDGAR_PORT:-2100} +IDENTITY=${EDGAR_IDENTITY:-} +SECRET=${EDGAR_SECRET:-} +DATA_DIR=${EDGAR_DATA_DIR:-"$HERE/.cache"} +RUNTIME=${EDGAR_RUNTIME:-auto} + +if [ -z "$IDENTITY" ]; then + echo "EDGAR_IDENTITY is required: the SEC asks every automated client for a name and a real email." >&2 + echo " EDGAR_IDENTITY=\"Jane Doe jane@example.com\" $0" >&2 + exit 1 +fi + +if [ -z "$SECRET" ]; then + SECRET=$(openssl rand -hex 24 2>/dev/null || head -c 24 /dev/urandom | od -An -tx1 | tr -d ' \n') +fi + +mkdir -p "$DATA_DIR" + +if [ "$RUNTIME" = auto ]; then + if command -v docker >/dev/null 2>&1; then RUNTIME=docker; else RUNTIME=python; fi +fi + +cleanup() { + [ -n "${SERVER_PID:-}" ] && kill "$SERVER_PID" 2>/dev/null || true + [ "$RUNTIME" = docker ] && docker rm -f crm-edgar-tunnel >/dev/null 2>&1 || true + [ -n "${TUNNEL_PID:-}" ] && kill "$TUNNEL_PID" 2>/dev/null || true +} +trap cleanup EXIT INT TERM + +case "$RUNTIME" in + docker) + docker build -q -t crm-edgar "$HERE" >/dev/null + docker rm -f crm-edgar-tunnel >/dev/null 2>&1 || true + docker run -d --name crm-edgar-tunnel -p "127.0.0.1:$PORT:2100" \ + -e EDGAR_IDENTITY="$IDENTITY" -e EDGAR_SECRET="$SECRET" \ + -v "$DATA_DIR:/data" crm-edgar >/dev/null + ;; + python) + if [ ! -x "$HERE/.venv/bin/python" ]; then + python3 -m venv "$HERE/.venv" + "$HERE/.venv/bin/pip" install -q -r "$HERE/requirements.txt" + fi + PYTHONPATH="$HERE" EDGAR_IDENTITY="$IDENTITY" EDGAR_SECRET="$SECRET" \ + EDGAR_DATA_DIR="$DATA_DIR" EDGAR_LOCAL_DATA_DIR="$DATA_DIR" \ + "$HERE/.venv/bin/python" -m uvicorn edgar_service.app:app --host 127.0.0.1 --port "$PORT" >"$DATA_DIR/uvicorn.log" 2>&1 & + SERVER_PID=$! + ;; + *) + echo "EDGAR_RUNTIME must be docker or python." >&2 + exit 1 + ;; +esac + +i=0 +until curl -fsS "http://127.0.0.1:$PORT/health" >/dev/null 2>&1; do + i=$((i + 1)) + if [ "$i" -gt 60 ]; then + echo "The service did not answer on port $PORT." >&2 + exit 1 + fi + sleep 1 +done + +if ! command -v cloudflared >/dev/null 2>&1; then + echo "cloudflared is required: https://developers.cloudflare.com/cloudflare-one/connections/connect-networks/downloads/" >&2 + exit 1 +fi + +cloudflared tunnel --url "http://127.0.0.1:$PORT" --no-autoupdate >"$DATA_DIR/tunnel.log" 2>&1 & +TUNNEL_PID=$! + +URL="" +i=0 +while [ -z "$URL" ]; do + i=$((i + 1)) + if [ "$i" -gt 60 ]; then + echo "The tunnel did not come up. See $DATA_DIR/tunnel.log." >&2 + exit 1 + fi + sleep 1 + URL=$(grep -oE 'https://[a-z0-9-]+\.trycloudflare\.com' "$DATA_DIR/tunnel.log" | head -1 || true) +done + +echo +echo "Set these on the CRM (Vercel → crm-agent and crm-app, or the local .env), then redeploy:" +echo "EDGAR_URL=$URL" +echo "EDGAR_SECRET=$SECRET" +echo +echo "The service and the tunnel live as long as this shell does. Ctrl-C stops both." +wait "$TUNNEL_PID" diff --git a/turbo.json b/turbo.json index 110f9bd94..1e450a3d6 100644 --- a/turbo.json +++ b/turbo.json @@ -17,6 +17,7 @@ "SLACK_CLIENT_ID", "SLACK_CLIENT_SECRET", "MICROSOFT_TENANT_ID", + "PASSWORD_SIGN_IN", "AUTH_COOKIE_DOMAIN", "CRON_SECRET", "REDIS_URL", @@ -24,6 +25,15 @@ "PORT", "PRISMA_LOG_QUERIES", "PERPLEXITY_API_KEY", + "HUNTER_API_KEY", + "EDGAR_URL", + "EDGAR_SECRET", + "EDGAR_IDENTITY", + "APOLLO_API_KEY", + "LUSHA_API_KEY", + "DROPCONTACT_API_KEY", + "ZOOMINFO_USERNAME", + "ZOOMINFO_PASSWORD", "GITHUB_TOKEN", "BLOB_READ_WRITE_TOKEN", "AI_GATEWAY_API_KEY", @@ -33,7 +43,12 @@ "CRM_TELEMETRY_DISABLED", "DO_NOT_TRACK", "VERCEL", - "VERCEL_GIT_COMMIT_SHA" + "VERCEL_GIT_COMMIT_SHA", + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + "NODE_EXTRA_CA_CERTS", + "SSL_CERT_FILE" ], "tasks": {