fetch.js

import { loadConfig } from './lib/config.js'
import { discoverPullRequests } from './lib/discover.js'
import { fetchPullRequestDetails } from './lib/detail.js'
import { diffSnapshots, resolveDepartedPullRequests } from './lib/diff.js'
import { loadFamiliarityIndex, scoreFamiliarity } from './lib/familiarity.js'
import { assertGhReady, fetchAuthenticatedLogin } from './lib/gh.js'
import { assignBucket, buildFacts, loadBucketDefinitions } from './lib/buckets.js'
import { buildReasonBits, estimateEffort, scoreSignals } from './lib/score.js'
import { computeSignals, extractRawFields } from './lib/signals.js'
import { PREVIOUS_SNAPSHOT_PATH, SNAPSHOT_PATH } from './lib/paths.js'
import { readJsonFile, writeJsonFileAtomically } from './lib/store.js'

const SNAPSHOT_VERSION = 1

async function main() {
  const isDryRun = process.argv.includes('--dry-run')
  const config = loadConfig()

  await assertGhReady()
  const authenticatedLogin = await fetchAuthenticatedLogin()
  if (authenticatedLogin !== config.user) {
    console.warn(
      `gh is authenticated as ${authenticatedLogin} but reference/config.json says ${config.user}.`
    )
  }

  const discovery = await discoverPullRequests(config)
  console.log(`Discovered ${discovery.refs.length} pull requests after dedup`)
  for (const [source, count] of Object.entries(discovery.sourceCounts)) {
    console.log(`  ${source}: ${count}`)
  }
  if (discovery.droppedOwnPullRequests > 0) {
    console.log(`  dropped ${discovery.droppedOwnPullRequests} authored by ${config.user}`)
  }

  if (isDryRun) {
    console.log(`Discovery API cost: ${discovery.apiCost}`)
    return
  }

  const { detailsByNodeId, apiCost, unfetchable } = await fetchPullRequestDetails(discovery.refs)
  const now = new Date()

  const familiarity = await buildFamiliarity({ config, refs: discovery.refs, detailsByNodeId, now })

  const prs = discovery.refs
    .filter((ref) => detailsByNodeId.has(ref.nodeId))
    .map((ref) => buildSnapshotEntry(ref, detailsByNodeId.get(ref.nodeId), config, now, familiarity.index))
    .sort((left, right) => right.score - left.score)

  const previous = readJsonFile(SNAPSHOT_PATH)
  const snapshot = {
    version: SNAPSHOT_VERSION,
    generatedAt: now.toISOString(),
    user: config.user,
    org: config.org,
    sourceCounts: discovery.sourceCounts,
    familiarityIndexAt: familiarity.index?.builtAt ?? null,
    apiCost: discovery.apiCost + apiCost + familiarity.apiCost,
    unfetchable: unfetchable.map((ref) => ({ key: ref.key, url: ref.url, title: ref.title })),
    prs,
  }

  const { deltasByKey, departedNodeIds } = diffSnapshots(previous, snapshot)
  for (const pr of snapshot.prs) pr.deltas = deltasByKey.get(pr.key) ?? []

  const departed = await resolveDepartedPullRequests(departedNodeIds)
  snapshot.departed = departed.entries
  snapshot.apiCost += departed.apiCost
  snapshot.changedCount = [...deltasByKey.values()].length

  if (previous) writeJsonFileAtomically(PREVIOUS_SNAPSHOT_PATH, previous)
  writeJsonFileAtomically(SNAPSHOT_PATH, snapshot)

  const { buckets, problems } = loadBucketDefinitions()
  for (const problem of problems) console.warn(`Bucket definitions: ${problem}`)

  const bucketCounts = countByBucket(prs, buckets)
  console.log(`Wrote ${prs.length} pull requests to ${SNAPSHOT_PATH}`)
  for (const [bucket, count] of Object.entries(bucketCounts)) console.log(`  ${bucket}: ${count}`)
  if (unfetchable.length > 0) console.log(`  could not fetch: ${unfetchable.length}`)
  if (snapshot.changedCount > 0) console.log(`Changed since last refresh: ${snapshot.changedCount}`)
  if (departed.entries.length > 0) console.log(`Left the queue: ${departed.entries.length}`)
  console.log(`Total API cost: ${snapshot.apiCost}`)
}

async function buildFamiliarity({ config, refs, detailsByNodeId, now }) {
  if (process.argv.includes('--no-familiarity')) return { index: null, apiCost: 0 }

  const repos = [...new Set(refs.filter((ref) => detailsByNodeId.has(ref.nodeId)).map((ref) => ref.repo))]

  try {
    const result = await loadFamiliarityIndex({ config, repos, now })
    if (result.wasRebuilt) console.log(`Rebuilt familiarity index for ${repos.length} repositories`)
    return result
  } catch (error) {
    console.warn(`Familiarity index unavailable, scoring without it: ${error.message}`)
    return { index: null, apiCost: 0 }
  }
}

function buildSnapshotEntry(ref, detail, config, now, familiarityIndex) {
  const changedFiles = (detail.files?.nodes ?? []).map((file) => file.path)
  const familiarity = familiarityIndex ? scoreFamiliarity(familiarityIndex, ref.repo, changedFiles) : null
  const signals = computeSignals({ ref, detail, config, now, familiarity })
  const { score, contributions } = scoreSignals(signals)

  return {
    key: ref.key,
    nodeId: ref.nodeId,
    repo: ref.repo,
    repoName: ref.repoName,
    number: ref.number,
    url: ref.url,
    title: detail.title ?? ref.title,
    author: detail.author?.login ?? ref.author,
    createdAt: detail.createdAt ?? ref.createdAt,
    updatedAt: detail.updatedAt ?? ref.updatedAt,
    sources: ref.sources,
    raw: extractRawFields(detail),
    signals,
    score,
    contributions,
    reasonBits: buildReasonBits(signals),
    effort: estimateEffort(signals),
    deltas: [],
  }
}

function countByBucket(prs, buckets) {
  const counts = {}
  for (const pr of prs) {
    const bucket = assignBucket(buildFacts(pr), buckets)
    counts[bucket] = (counts[bucket] ?? 0) + 1
  }
  return counts
}

main().catch((error) => {
  console.error(error.message)
  process.exit(1)
})