diff --git a/packages/bot/src/__tests__/markets.test.ts b/packages/bot/src/__tests__/markets.test.ts index bc584c5..7387b94 100644 --- a/packages/bot/src/__tests__/markets.test.ts +++ b/packages/bot/src/__tests__/markets.test.ts @@ -73,6 +73,90 @@ describe('markets: coveragePlan (issue #2 phases 4+7 wiring)', () => { }); }); +describe('markets: scan-budget priority scheduler (issue #2 phase 5 wiring)', () => { + const fresh: Market = { symbol: 'A', tradeGoods: [{ symbol: 'X', purchasePrice: 50, sellPrice: 70 }] } as unknown as Market; + const counting = () => { + let gets = 0; + const c = { + api: async (_m: string, path: string) => { + if (path.includes('/market')) { + gets += 1; + return { data: fresh } as ApiEnvelope; + } + return { data: {} } as ApiEnvelope; + }, + } as unknown as SpaceTradersClient; + return { client: c, gets: () => gets }; + }; + + it('is null without the lever (legacy fetch-all-due, no budget metric)', async () => { + // value budget on (cfg) but SCAN_BUDGET_ON unset → every due market fetched, no scanBudget status. + const cfg = loadConfig({}); + const { client: c } = counting(); + const m = createMarketsService({ client: c, persistence, coords, maxd: 2000, cfg, marketWaypoints: ['A', 'B', 'C'] }); + await m.getMarkets(); + expect(m.scanBudgetStatus()).toBeNull(); + }); + + it('caps a due-burst to the per-sweep budget and defers the rest', async () => { + // Three never-scanned (all due) markets, budget hard-capped to 1 → fetch 1, defer 2. + const cfg = loadConfig({ SCAN_BUDGET_ON: '1', SCAN_BUDGET_MAX_PER_SWEEP: '1' }); + const { client: c, gets } = counting(); + const m = createMarketsService({ client: c, persistence, coords, maxd: 2000, cfg, marketWaypoints: ['A', 'B', 'C'] }); + await m.getMarkets(); + expect(gets()).toBe(1); // spent exactly the budget, not all 3 due markets + const st = m.scanBudgetStatus(); + expect(st).not.toBeNull(); + expect(st!.perSweep).toBe(1); + expect(st!.due).toBe(3); + expect(st!.granted).toBe(1); + expect(st!.deferred).toBe(2); + }); + + it('presence-gates: spends budget only on covered markets, counts the rest as uncovered', async () => { + // 3 due markets but only A has a ship present/inbound → B,C are price-blind, never granted. + const cfg = loadConfig({ SCAN_BUDGET_ON: '1' }); + const { client: c, gets } = counting(); + const m = createMarketsService({ + client: c, + persistence, + coords, + maxd: 2000, + cfg, + marketWaypoints: ['A', 'B', 'C'], + coveredWps: () => new Set(['A']), + }); + await m.getMarkets(); + expect(gets()).toBe(1); // only the covered market was read — no budget leaked onto B/C + const st = m.scanBudgetStatus(); + expect(st!.due).toBe(3); + expect(st!.granted).toBe(1); + expect(st!.uncovered).toBe(2); + expect(st!.deferred).toBe(0); + }); + + it('skips the gate when coverage is empty (cold start before the first fleet poll)', async () => { + // No coverage signal yet → ungated, behaves like today (all due markets eligible). + const cfg = loadConfig({ SCAN_BUDGET_ON: '1' }); + const { client: c, gets } = counting(); + const m = createMarketsService({ + client: c, + persistence, + coords, + maxd: 2000, + cfg, + marketWaypoints: ['A', 'B', 'C'], + coveredWps: () => new Set(), + }); + await m.getMarkets(); + expect(gets()).toBe(3); // empty set → no gating + const st = m.scanBudgetStatus(); + expect(st!.uncovered).toBe(0); + expect(st!.granted).toBe(3); + }); +}); + + describe('markets: recheckScan (issue #2 phase 7)', () => { const cfg: Config = loadConfig({}); diff --git a/packages/bot/src/fleet/table.ts b/packages/bot/src/fleet/table.ts index 083c79e..18e08ef 100644 --- a/packages/bot/src/fleet/table.ts +++ b/packages/bot/src/fleet/table.ts @@ -41,21 +41,29 @@ export function routeStr(ship: Ship, deps: Pick) /** * Background loop: every `FLEET_TABLE_MS`, snapshot each cargo hull's planned route into - * `state.fleetRoutes` (keyed by the 3-char short id used across status). Inert when - * `FLEET_TABLE` is off. (bot2 `fleetTable`) + * `state.fleetRoutes` (keyed by the 3-char short id used across status). When `SCAN_BUDGET_ON`, also + * refresh `state.coverageWps` (every ship's present + inbound waypoint) so the scan-budget scheduler + * can avoid spending reads on ship-absent markets. Inert when both `FLEET_TABLE` and `SCAN_BUDGET_ON` + * are off — reusing this existing poll avoids a second `getAllShips`. (bot2 `fleetTable`) */ export async function fleetTableManager(deps: Pick): Promise { const { state, cfg, client } = deps; - if (!cfg.FLEET_TABLE) return; + if (!cfg.FLEET_TABLE && !cfg.SCAN_BUDGET_ON) return; while (!state.stop) { await sleep(cfg.FLEET_TABLE_MS); try { const all = await client.getAllShips(); + const covered = new Set(); for (const s of all) { - if (s.cargo.capacity <= 0) continue; + // coverage presence: a market is scannable where a ship is present or inbound. + covered.add(s.nav.waypointSymbol); + const dest = s.nav.route?.destination?.symbol; + if (dest) covered.add(dest); + if (!cfg.FLEET_TABLE || s.cargo.capacity <= 0) continue; // [ROUTE] stash the ship's full multihop route for writeStatus()/the dashboard. state.fleetRoutes[s.symbol.slice(-3)] = routeStr(s, deps); } + if (cfg.SCAN_BUDGET_ON) state.coverageWps = covered; } catch (e) { log.info(`fleetTable: ${(e as Error).message}`); } diff --git a/packages/bot/src/main.ts b/packages/bot/src/main.ts index e6a8e95..7b2c984 100644 --- a/packages/bot/src/main.ts +++ b/packages/bot/src/main.ts @@ -36,6 +36,8 @@ import type { SpaceTradersClient } from './interfaces.js'; const log = logger.child({ mod: 'main' }); const sleep = (ms: number): Promise => new Promise((r) => setTimeout(r, ms)); +/** Shared empty coverage set for the cold-start window (before the first fleet poll). */ +const EMPTY_WPS: ReadonlySet = new Set(); async function refreshCredits(state: BotState, client: SpaceTradersClient): Promise { try { @@ -86,7 +88,18 @@ export async function main(): Promise { const marketHolder: { data: Record } = { data: {} }; const marketsRef = (): Record => marketHolder.data; - const markets = createMarketsService({ client, persistence, coords, maxd: cfg.MAXD, cfg }); + const state = createState(cfg, { marketsRef }); + + const markets = createMarketsService({ + client, + persistence, + coords, + maxd: cfg.MAXD, + cfg, + // presence-gate scan-budget reads to markets with a ship present/inbound (state.coverageWps is + // refreshed by the fleet poll). Empty/undefined before the first poll → scheduler stays ungated. + coveredWps: () => state.coverageWps ?? EMPTY_WPS, + }); const router = createRouter({ coords, getFuelPx: () => markets.getFuelPx(), @@ -94,7 +107,6 @@ export async function main(): Promise { marketsRef, }); const actions = createShipActions(client); - const state = createState(cfg, { marketsRef }); // [issue #2] Surface the value-weighted scan-budget metric in the status snapshot. credits-per-request // is the headline lever: realized run net ÷ market GETs spent. Higher = scan budget better allocated. @@ -117,6 +129,7 @@ export async function main(): Promise { trips: l.trips, })), ...(state.coverage !== undefined ? { coverage: state.coverage } : {}), + ...(markets.scanBudgetStatus() !== null ? { scanBudget: markets.scanBudgetStatus() } : {}), }; }; diff --git a/packages/bot/src/market/__tests__/replay.calibration.test.ts b/packages/bot/src/market/__tests__/replay.calibration.test.ts new file mode 100644 index 0000000..2b5e39a --- /dev/null +++ b/packages/bot/src/market/__tests__/replay.calibration.test.ts @@ -0,0 +1,191 @@ +/** + * replay.calibration.test.ts — replay the value-weighted cores over REAL production data. + * + * Fixture below is the realized-lane distribution parsed from 13.45h of the live UPRISING (.mjs) + * agent (phase PORTAL_OPEN, 226 ships) — see the orchestrator's `live-metrics.json` bundle + * (`collect-metrics.mjs`, re-runnable). Zero API calls were spent to gather it (log + status parse). + * + * The point of this test is to PROVE THE WIN on real numbers, not just on synthetic toys: feeding the + * real realized-net distribution through the pure cores (`laneRegistry` → `value.scoreMarkets` → + * `scanScheduler.intervalFor` → `scanBudget.allocateScanBudget`) must produce a refresh cadence that + * is (a) MONOTONIC in realized value and (b) FAR MORE DIFFERENTIATED than the observed near-uniform + * ~24.65 refreshes/market the production bot actually ran. That differentiation — concentrating reads + * on lane-critical markets and starving dead ones — is the entire thesis of issue #2. + */ + +import { describe, it, expect } from 'vitest'; +import { createLaneRegistry } from '../../trade/laneRegistry.js'; +import { scoreMarkets } from '../value.js'; +import { createScanScheduler } from '../scanScheduler.js'; +import { allocateScanBudget, scanBudgetPerSweep, type ScanCandidate } from '../scanBudget.js'; +import type { Market, TradeObservation } from '@st/shared'; + +// ── REAL fixture: topSinksByNet from live-metrics.json (sink, completed lanes, avg realized net) ── +const REAL_SINKS: ReadonlyArray<{ sink: string; lanes: number; avgNet: number }> = [ + { sink: '-A1', lanes: 68, avgNet: 43807 }, + { sink: 'Z9C', lanes: 30, avgNet: 49381 }, + { sink: '22A', lanes: 28, avgNet: 44199 }, + { sink: 'K82', lanes: 22, avgNet: 47571 }, + { sink: 'D41', lanes: 16, avgNet: 63914 }, + { sink: '37C', lanes: 40, avgNet: 24518 }, + { sink: '11X', lanes: 15, avgNet: 54696 }, + { sink: '-A2', lanes: 11, avgNet: 53303 }, + { sink: 'F8B', lanes: 7, avgNet: 70401 }, + { sink: '10X', lanes: 12, avgNet: 30315 }, + { sink: 'D40', lanes: 9, avgNet: 34889 }, + { sink: 'D46', lanes: 4, avgNet: 73800 }, +]; + +// Observed dead-lane rate (negativeLaneShare) and near-uniform scan spread, from the same bundle. +const NEGATIVE_LANE_SHARE = 0.155; +const OBSERVED_AVG_REFRESHES = 24.65; // avgRefreshesPerMarket — the ~uniform baseline we beat + +const NOW = 1_700_000_000_000; // fixed clock → ingest with ts = NOW so no staleness decay in replay + +/** Mint a completed-trade observation for one lane endpoint pair. */ +function obs(sink: string, net: number, units: number): TradeObservation { + return { + ts: new Date(NOW).toISOString(), + ship: 'REPLAY', + good: 'G', + buyWp: `SRC-${sink}`, + sellWp: `SINK-${sink}`, + projected: net, + realized: net, + units, + buyPx: 0, + sellPx: net, + }; +} + +/** Build the registry + scored markets for the real sinks plus a dead tail of never-traded markets. */ +function buildReplay(deadCount: number) { + const registry = createLaneRegistry({ alpha: 0.3, halfLifeMs: 1_800_000 }); + for (const s of REAL_SINKS) for (let i = 0; i < s.lanes; i += 1) registry.ingest(obs(s.sink, s.avgNet, 100)); + + // Markets Record: every endpoint we want scored needs an entry (scoreMarkets iterates it). Empty + // tradeGoods isolates the REALIZED path — exactly the lane-attribution signal the real data carries. + const markets: Record = {}; + for (const s of REAL_SINKS) { + markets[`SINK-${s.sink}`] = { symbol: `SINK-${s.sink}`, tradeGoods: [] } as unknown as Market; + markets[`SRC-${s.sink}`] = { symbol: `SRC-${s.sink}`, tradeGoods: [] } as unknown as Market; + } + for (let i = 0; i < deadCount; i += 1) + markets[`DEAD-${i}`] = { symbol: `DEAD-${i}`, tradeGoods: [] } as unknown as Market; + + const scored = scoreMarkets(markets, registry.marketRealizedValue(NOW), { realized: 1, structural: 0, volume: 0 }); + const scoreByWp = new Map(); + for (const [wp, v] of scored) scoreByWp.set(wp, v.score); + return { registry, markets, scoreByWp }; +} + +describe('replay calibration: value-weighted scan cadence vs the observed uniform baseline', () => { + // Defaults under test (config.ts SCAN_*): base 75s, floor 30s, ceiling 600s → 20:1 dynamic range. + const sched = () => + createScanScheduler({ + baseMs: 75_000, + minMs: 30_000, + maxMs: 600_000, + valFactorMin: 0.1, + valFactorMax: 10, + volAlpha: 0.3, + volGain: 10, + volFactorMin: 0.5, + volFactorMax: 4, + }); + + it('reproduces the real per-market realized value (lane net attributes to both endpoints)', () => { + const { registry } = buildReplay(0); + const realized = registry.marketRealizedValue(NOW); + // Each sink's value ≈ its observed avgNet (constant-input EWMA, no decay at NOW). + for (const s of REAL_SINKS) expect(realized.get(`SINK-${s.sink}`)).toBeCloseTo(s.avgNet, 0); + // Top realized sink is the highest-avgNet lane endpoint (D46 @ 73,800). + const topSink = [...REAL_SINKS].sort((a, b) => b.avgNet - a.avgNet)[0]!; + expect(topSink.sink).toBe('D46'); + }); + + it('drives a refresh cadence that is MONOTONIC in realized value (volatility held flat)', () => { + const { scoreByWp } = buildReplay(20); + const s = sched(); + const ref = s.valueRef(scoreByWp); + const interval = (wp: string): number => s.intervalFor(scoreByWp.get(wp) ?? 0, ref, 0); + + // Sinks sorted by realized value DESC → their intervals must be non-increasing (more value ⇒ + // shorter interval ⇒ more refreshes). No inversion anywhere along the real distribution. + const sinksByValue = [...REAL_SINKS].sort((a, b) => b.avgNet - a.avgNet).map((x) => `SINK-${x.sink}`); + for (let i = 1; i < sinksByValue.length; i += 1) + expect(interval(sinksByValue[i]!)).toBeGreaterThanOrEqual(interval(sinksByValue[i - 1]!) - 1e-6); + + // A dead (never-traded) market must refresh strictly less often than any real sink. + const deadInterval = interval('DEAD-0'); + for (const wp of sinksByValue) expect(interval(wp)).toBeLessThan(deadInterval); + }); + + it('concentrates reads FAR more than the observed ~uniform 24.65 refreshes/market', () => { + const { scoreByWp, markets } = buildReplay(40); + const s = sched(); + const ref = s.valueRef(scoreByWp); + const windowMs = 13.45 * 3600_000; // same 13.45h window as the live sample + + // Project each market's refresh COUNT over the window from its value-driven interval. + const counts = Object.keys(markets).map((wp) => windowMs / s.intervalFor(scoreByWp.get(wp) ?? 0, ref, 0)); + const hot = Math.max(...counts); + const cold = Math.min(...counts); + + // 1) Hot markets get an order of magnitude more reads than dead ones — the 20:1 clamp range + // realised on REAL value spread. The uniform baseline's hot:cold ratio is ~1. + expect(hot / cold).toBeGreaterThanOrEqual(10); + + // 2) Differentiation: coefficient of variation of the projected refresh counts is large, whereas a + // uniform scheduler (every market = OBSERVED_AVG_REFRESHES) has CV 0. + const mean = counts.reduce((a, b) => a + b, 0) / counts.length; + const sd = Math.sqrt(counts.reduce((a, c) => a + (c - mean) ** 2, 0) / counts.length); + expect(sd / mean).toBeGreaterThan(0.3); + + // 3) Sanity vs the real baseline: the hottest real sink is read MANY× more per window than the + // flat 24.65 the production bot actually spent on the average market. + expect(hot).toBeGreaterThan(OBSERVED_AVG_REFRESHES * 5); + }); + + it('spends a constrained scan budget on the high-value markets, deferring the dead tail', () => { + const { scoreByWp } = buildReplay(40); + const s = sched(); + const ref = s.valueRef(scoreByWp); + + // A synchronized due-burst: every market wants a read this sweep (cold-start staleness). + const candidates: ScanCandidate[] = [...scoreByWp.keys()].map((wp) => ({ + wp, + relValue: (scoreByWp.get(wp) ?? 0) / ref, + overrun: 1, + })); + // Budget at the calibrated defaults: 2 req/s × 10s sweep × 0.4 = 8 reads. + const budget = scanBudgetPerSweep({ reqPerSec: 2, sweepMs: 10_000, fraction: 0.4, maxPerSweep: 0 }); + expect(budget).toBe(8); + + const alloc = allocateScanBudget(candidates, budget); + expect(alloc.granted.length).toBe(budget); + // No dead market wins budget while value-bearing sinks are still due — budget never leaks onto 0-value reads. + for (const wp of alloc.granted) expect(wp.startsWith('DEAD-')).toBe(false); + // The grant skews HOT (lane-critical), proving value-first spend rather than FIFO. + expect(alloc.byTier.hot).toBeGreaterThan(0); + expect(alloc.byTier.hot).toBeGreaterThanOrEqual(alloc.byTier.cold); + }); + + it('classifies a dead tail comparable to the real 15.5% negative-lane share into the cheapest cadence', () => { + // Size the dead tail to the observed dead-lane rate among a real-sized market set, then confirm the + // scheduler parks those markets at the SCAN_MAX ceiling (cheapest cadence) — the DEAD-tier intent. + const sinkMarkets = REAL_SINKS.length * 2; // SINK-* + SRC-* + const deadCount = Math.round((sinkMarkets / (1 - NEGATIVE_LANE_SHARE)) * NEGATIVE_LANE_SHARE); + const { scoreByWp, markets } = buildReplay(deadCount); + const s = sched(); + const ref = s.valueRef(scoreByWp); + const atCeiling = Object.keys(markets).filter( + (wp) => s.intervalFor(scoreByWp.get(wp) ?? 0, ref, 0) >= 600_000 - 1, + ); + // Every dead market lands at the ceiling; the share is in the right ballpark (non-trivial, < half). + expect(atCeiling.length).toBe(deadCount); + const deadShare = deadCount / Object.keys(markets).length; + expect(deadShare).toBeGreaterThan(0.05); + expect(deadShare).toBeLessThan(0.4); + }); +}); diff --git a/packages/bot/src/market/__tests__/scanBudget.test.ts b/packages/bot/src/market/__tests__/scanBudget.test.ts new file mode 100644 index 0000000..5e13490 --- /dev/null +++ b/packages/bot/src/market/__tests__/scanBudget.test.ts @@ -0,0 +1,103 @@ +import { describe, it, expect } from 'vitest'; +import { + scanPriority, + scanBudgetPerSweep, + allocateScanBudget, + COLD_START_OVERRUN, + type ScanCandidate, +} from '../scanBudget.js'; + +describe('scanBudget: scanPriority', () => { + it('is value × overrun — both factors raise priority', () => { + expect(scanPriority(2, 1)).toBe(2); + expect(scanPriority(2, 3)).toBe(6); // more overdue → higher + expect(scanPriority(4, 1)).toBe(4); // more valuable → higher + }); + + it('never goes negative and keeps dead markets rankable among themselves by overrun', () => { + expect(scanPriority(0, 5)).toBe(0); + expect(scanPriority(-1, 5)).toBe(0); + // a high-value barely-due market outranks a dead long-overdue one + expect(scanPriority(3, 1)).toBeGreaterThan(scanPriority(0.01, 50)); + }); +}); + +describe('scanBudget: scanBudgetPerSweep', () => { + it('derives floor(reqPerSec × sweepSeconds × fraction)', () => { + // 2 req/s × 10s × 0.6 = 12 + expect(scanBudgetPerSweep({ reqPerSec: 2, sweepMs: 10_000, fraction: 0.6, maxPerSweep: 0 })).toBe(12); + }); + + it('reserves headroom for trades via the fraction', () => { + const full = scanBudgetPerSweep({ reqPerSec: 2, sweepMs: 10_000, fraction: 1, maxPerSweep: 0 }); // 20 + const reserved = scanBudgetPerSweep({ reqPerSec: 2, sweepMs: 10_000, fraction: 0.5, maxPerSweep: 0 }); // 10 + expect(full).toBe(20); + expect(reserved).toBe(10); + expect(reserved).toBeLessThan(full); + }); + + it('respects an absolute hard cap', () => { + expect(scanBudgetPerSweep({ reqPerSec: 2, sweepMs: 60_000, fraction: 1, maxPerSweep: 8 })).toBe(8); + }); + + it('never returns below 1 (always makes progress)', () => { + expect(scanBudgetPerSweep({ reqPerSec: 2, sweepMs: 100, fraction: 0.1, maxPerSweep: 0 })).toBe(1); + }); +}); + +describe('scanBudget: allocateScanBudget', () => { + const cand = (wp: string, relValue: number, overrun: number): ScanCandidate => ({ wp, relValue, overrun }); + + it('grants the highest value×staleness first and defers the rest', () => { + const candidates = [ + cand('A', 3, 1), // 3 + cand('B', 1, 1), // 1 + cand('C', 2, 2), // 4 ← top + cand('D', 0.1, 1), // 0.1 + ]; + const out = allocateScanBudget(candidates, 2); + expect(out.granted).toEqual(['C', 'A']); // top-2 by priority + expect(out.deferred.sort()).toEqual(['B', 'D']); + expect(out.budget).toBe(2); + }); + + it('classifies never-scanned markets promptly (cold-start overrun floats them up)', () => { + // A never-scanned market is seeded with COLD_START_OVERRUN and a small positive relValue at the + // call site (unknown ≠ dead), so even against a strong known market it is read first. + const out = allocateScanBudget([cand('HOT', 5, 1.2), cand('NEW', 1, COLD_START_OVERRUN)], 1); + expect(out.granted).toEqual(['NEW']); // cold-start dominates → classified first + }); + + it('avoids starvation: a deferred market rises as its overrun grows', () => { + // C loses this sweep, but next sweep its overrun has grown and it now wins. + const sweep1 = allocateScanBudget([cand('A', 2, 2), cand('C', 1, 1.5)], 1); + expect(sweep1.granted).toEqual(['A']); + expect(sweep1.deferred).toEqual(['C']); + // A was just read (overrun resets ~1), C kept waiting (overrun climbed to 4) + const sweep2 = allocateScanBudget([cand('A', 2, 1), cand('C', 1, 4)], 1); + expect(sweep2.granted).toEqual(['C']); // the previously-starved market now wins + }); + + it('reports a per-tier histogram of where the budget went', () => { + const out = allocateScanBudget( + [cand('H', 3, 1), cand('W', 1, 1), cand('Cc', 0.2, 1)], + 3, + ); + expect(out.byTier).toEqual({ hot: 1, warm: 1, cold: 1 }); + expect(out.granted.length).toBe(3); + expect(out.deferred).toEqual([]); + }); + + it('grants everything when budget ≥ candidate count', () => { + const out = allocateScanBudget([cand('A', 1, 1), cand('B', 2, 1)], 10); + expect(out.granted.sort()).toEqual(['A', 'B']); + expect(out.deferred).toEqual([]); + }); + + it('grants nothing at zero budget (all deferred, no crash)', () => { + const out = allocateScanBudget([cand('A', 1, 1)], 0); + expect(out.granted).toEqual([]); + expect(out.deferred).toEqual(['A']); + expect(out.byTier).toEqual({ hot: 0, warm: 0, cold: 0 }); + }); +}); diff --git a/packages/bot/src/market/markets.ts b/packages/bot/src/market/markets.ts index a1e36d3..981dfd5 100644 --- a/packages/bot/src/market/markets.ts +++ b/packages/bot/src/market/markets.ts @@ -23,6 +23,13 @@ import { computeFuelPx } from '../routing/flight.js'; import { createLaneRegistry, type LaneRegistry, type RankedLane } from '../trade/laneRegistry.js'; import { scoreMarkets, type ValueWeights } from './value.js'; import { createScanScheduler, type ScanScheduler, type MarketScanState } from './scanScheduler.js'; +import { + scanBudgetPerSweep, + allocateScanBudget, + COLD_START_OVERRUN, + type ScanCandidate, + type ScanTier, +} from './scanBudget.js'; import { planCoverage, coverageTarget, @@ -55,6 +62,15 @@ export interface MarketsServiceOptions { marketWaypoints?: string[]; /** Initial fuel price (defaults to 0.72). */ fuelPxInit?: number; + /** + * Coverage-presence provider (issue #2, phase 5): the set of waypoints with a ship present or + * inbound right now. A `GET /market` only returns live prices where a ship is present, so the + * scan-budget scheduler grants reads ONLY to markets in this set — spending budget on an uncovered + * market is a wasted request. Supplied by `main` from `state.coverageWps` (the fleet poll). + * When omitted, or when it returns an empty set (cold start — before the first poll), the scheduler + * falls back to ungated allocation so behaviour matches today until coverage data exists. + */ + coveredWps?: () => ReadonlySet; } export interface MarketsServiceExtra { @@ -82,6 +98,20 @@ export interface MarketsServiceExtra { * and counts a request. Returns the fresh market, or null on a transient failure / when disabled. */ recheckScan(wp: string, now?: number): Promise; + /** + * Last global scan-budget allocation (issue #2, phase 5): per-sweep budget, how many markets were + * due, how many were granted vs deferred, and a per-tier histogram of the granted reads. `null` when + * the scheduler is off (`SCAN_BUDGET_ON` unset) or no sweep has allocated yet. Metric-only. + */ + scanBudgetStatus(): { + perSweep: number; + due: number; + granted: number; + deferred: number; + /** DUE markets skipped because no ship is present/inbound (presence-gated, not budget-spent). */ + uncovered: number; + byTier: Record; + } | null; } export function createMarketsService(opts: MarketsServiceOptions): MarketsService & MarketsServiceExtra { @@ -124,6 +154,23 @@ export function createMarketsService(opts: MarketsServiceOptions): MarketsServic let marketGets = 0; let lastSweepAt = 0; + // ── global scan-budget priority scheduler (issue #2, phase 5) ────────────────────────────────── + // When SCAN_BUDGET_ON, a due-burst is granted highest value×staleness first up to a per-sweep budget + // (rest deferred to a later sweep, where rising staleness floats them up — no starvation). OFF ⇒ the + // legacy path fetches every due market in scheduler order. + const scanBudgetOn = cfg?.SCAN_BUDGET_ON ?? false; + const scanBudgetReqPerSec = cfg?.SCAN_BUDGET_REQ_PER_SEC ?? 2; + const scanBudgetFraction = cfg?.SCAN_BUDGET_REQ_FRACTION ?? 0.6; + const scanBudgetMaxPerSweep = cfg?.SCAN_BUDGET_MAX_PER_SWEEP ?? 0; + let lastScanBudget: { + perSweep: number; + due: number; + granted: number; + deferred: number; + uncovered: number; + byTier: Record; + } | null = null; + // ── value-driven coverage tiering + reversible pruning + cold re-check (issue #2, phases 4+7) ── const coverageWeights: CoverageWeights = { hotMult: cfg?.COVERAGE_HOT_MULT ?? 2, @@ -219,9 +266,12 @@ export function createMarketsService(opts: MarketsServiceOptions): MarketsServic const scoreByWp = new Map(); for (const [wp, v] of scored) scoreByWp.set(wp, v.score); const due = scheduler!.selectDue(scoreByWp, marketWps, t); + // Phase 5: when enabled, spend the limited per-sweep budget on the highest value×staleness markets + // first instead of fetching every due market FIFO. Deferred markets ride the next sweep. + const toFetch = scanBudgetOn ? allocateDue(due, scoreByWp, t) : due; const out: Record = { ...marketCache.data }; // keep un-refreshed markets warm const scanned = new Set(); - for (const wp of due) { + for (const wp of toFetch) { const m = await fetchMarket(wp); if (!m) continue; out[wp] = m; @@ -234,6 +284,53 @@ export function createMarketsService(opts: MarketsServiceOptions): MarketsServic return out; } + /** + * Phase 5 allocation: rank the DUE markets by `value × staleness` and grant only the per-sweep + * budget; the rest are deferred (their staleness keeps rising, so they win a later sweep — no + * starvation). Records the split for the scan-budget metric. Returns the waypoints to fetch now. + * + * Presence-gate: a `GET /market` only returns live prices where a ship is present, so DUE markets + * with no ship present/inbound are dropped from candidacy BEFORE allocation — budget is never spent + * on a price-blind read. The cold re-check of uncovered markets stays a separate presence-gated + * single read (`recheckScan`), not part of this budget. The gate is skipped when no coverage signal + * exists yet (empty set — before the first fleet poll) so cold-start behaviour matches today. + */ + function allocateDue(due: readonly string[], scoreByWp: Map, t: number): string[] { + const covered = opts.coveredWps?.(); + const gate = covered && covered.size > 0; + const scannable = gate ? due.filter((wp) => covered.has(wp)) : due; + const uncovered = due.length - scannable.length; + const ref = scheduler!.valueRef(scoreByWp); + const candidates: ScanCandidate[] = scannable.map((wp) => { + const st = scheduler!.state(wp); + const score = scoreByWp.get(wp) ?? 0; + if (!st || st.scans === 0) { + // never scanned — unknown ≠ dead: seed at ≥ mean value and a cold-start staleness so it is + // classified promptly rather than divided by a zero clock. + return { wp, relValue: Math.max(score / ref, 1), overrun: COLD_START_OVERRUN }; + } + const interval = scheduler!.intervalFor(score, ref, st.volatility); + const overrun = interval > 0 ? (t - st.lastScanAt) / interval : COLD_START_OVERRUN; + return { wp, relValue: score / ref, overrun }; + }); + const budget = scanBudgetPerSweep({ + reqPerSec: scanBudgetReqPerSec, + sweepMs, + fraction: scanBudgetFraction, + maxPerSweep: scanBudgetMaxPerSweep, + }); + const alloc = allocateScanBudget(candidates, budget); + lastScanBudget = { + perSweep: budget, + due: due.length, + granted: alloc.granted.length, + deferred: alloc.deferred.length, + uncovered, + byTier: alloc.byTier, + }; + return alloc.granted; + } + async function getMarkets(): Promise> { if (!scheduler) { // legacy: single global TTL, refresh ALL markets uniformly @@ -338,5 +435,6 @@ export function createMarketsService(opts: MarketsServiceOptions): MarketsServic topLanes: (k: number) => registry?.topLanes(k, now()) ?? [], coveragePlan, recheckScan, + scanBudgetStatus: () => lastScanBudget, }; } diff --git a/packages/bot/src/market/scanBudget.ts b/packages/bot/src/market/scanBudget.ts new file mode 100644 index 0000000..dbf01e7 --- /dev/null +++ b/packages/bot/src/market/scanBudget.ts @@ -0,0 +1,125 @@ +/** + * market/scanBudget.ts — explicit global scan-budget priority scheduler (issue #2, phase 5). + * + * PR1 gave every market a per-market refresh INTERVAL (`scanScheduler`). But when many markets come + * due at once, the shared ~2 req/s account budget (the module-global token bucket in the API client) + * is spent strictly FIFO — `refreshDue` fetches every due market in insertion order, reading dead + * markets ahead of lane-critical ones and monopolising the budget that latency-sensitive trade/nav + * requests also need. This module makes the allocation EXPLICIT: a priority queue keyed by + * value × staleness, drained at the budget the operator allows scans to spend per sweep. + * + * • PRIORITY — `priority = relValue × overrun`. `relValue` is the market's value relative to the + * fleet mean (same scale `scanScheduler` uses); `overrun = (now − lastScanAt) / interval` is how + * far past due it is (1.0 exactly at due, growing the longer it waits). A never-scanned market + * gets a fixed cold-start priority so it's classified promptly without dividing by a zero clock. + * • BUDGET — how many `GET /market` reads a sweep may spend, derived from the request rate and the + * sweep window so scans never consume the whole 2 req/s (headroom for trades). See {@link scanBudgetPerSweep}. + * • ALLOCATION — sort candidates by priority desc, grant the top `budget`, defer the rest to the + * next sweep. Deferral is NOT starvation: a deferred market's `overrun` keeps rising every sweep, + * so its priority climbs until it is eventually granted. The most-overdue, highest-value markets + * are simply served first. + * + * Everything here is pure: `(candidates, budget) → allocation`. No clock, no I/O. Unit-testable in + * isolation; `markets.ts` owns the wiring and the (gated) behaviour change. + */ + +/** Coverage-style tier label for the budget metric (independent of coverage.ts; local to scans). */ +export type ScanTier = 'hot' | 'warm' | 'cold'; + +export interface ScanCandidate { + wp: string; + /** Market value relative to the fleet mean (score / valueRef). */ + relValue: number; + /** + * How far past its due time the market is: `(now − lastScanAt) / interval`. 1.0 exactly at due, + * > 1 the longer it waits. Use {@link COLD_START_OVERRUN} for a never-scanned market. + */ + overrun: number; +} + +export interface ScanAllocation { + /** Waypoints to fetch THIS sweep, highest priority first. */ + granted: string[]; + /** Due waypoints held back to a later sweep (priority too low for this sweep's budget). */ + deferred: string[]; + /** Per-tier counts of GRANTED reads (metric: where the budget went). */ + byTier: Record; + /** Budget applied this sweep (granted.length ≤ budget). */ + budget: number; +} + +export interface ScanBudgetOptions { + /** Account request rate the scan budget is computed against (req/s). Mirrors the client token bucket. */ + reqPerSec: number; + /** Length of one scan sweep window (ms) — the cadence `refreshDue` runs at. */ + sweepMs: number; + /** Fraction of the sweep's theoretical request capacity scans may use (0..1]; leaves headroom for trades. */ + fraction: number; + /** Hard cap on reads per sweep; 0 ⇒ no absolute cap (fraction-derived only). */ + maxPerSweep: number; +} + +/** Overrun assigned to a never-scanned market so it sorts above merely-overdue ones (classify promptly). */ +export const COLD_START_OVERRUN = 1e6; + +/** Relative-value cutoffs (multiples of the fleet mean) used only to bucket the GRANTED metric. */ +const HOT_REL = 2; +const WARM_REL = 0.75; + +const clamp = (x: number, lo: number, hi: number): number => Math.min(hi, Math.max(lo, x)); + +/** Scan priority for a single market: value × how-overdue. Higher ⇒ fetched sooner. */ +export function scanPriority(relValue: number, overrun: number): number { + // relValue can be ~0 for a dead market; keep priority ≥ 0 and let overrun still rank dead markets + // among themselves so even they are eventually read (starvation avoidance), just last. + return Math.max(0, relValue) * Math.max(0, overrun); +} + +function tierOf(relValue: number): ScanTier { + if (relValue >= HOT_REL) return 'hot'; + if (relValue >= WARM_REL) return 'warm'; + return 'cold'; +} + +/** + * Per-sweep scan budget: `floor(reqPerSec × sweepSeconds × fraction)`, optionally hard-capped by + * `maxPerSweep`, never below 1 (always make some progress). The fraction reserves part of the ~2 req/s + * for trade/nav traffic, so scans can't starve earning requests. + */ +export function scanBudgetPerSweep(opts: ScanBudgetOptions): number { + const sweepSeconds = Math.max(0, opts.sweepMs) / 1000; + const capacity = opts.reqPerSec * sweepSeconds * clamp(opts.fraction, 0, 1); + const fromFraction = Math.floor(capacity); + const capped = opts.maxPerSweep > 0 ? Math.min(fromFraction, opts.maxPerSweep) : fromFraction; + return Math.max(1, capped); +} + +/** + * Allocate the per-sweep budget across due markets by priority. Sorts candidates by + * `scanPriority` desc (ties broken by higher overrun, so the longest-waiting goes first), grants the + * top `budget`, defers the rest. Returns the granted/deferred split and a per-tier histogram of where + * the budget was spent. + * + * Callers MUST pass only markets that are scannable right now (a ship present/inbound — a `GET /market` + * on an uncovered market returns no live prices, so budget there is wasted). Presence-gating is the + * caller's responsibility (see `markets.ts allocateDue`); this function is pure and presence-agnostic. + */ +export function allocateScanBudget(candidates: readonly ScanCandidate[], budget: number): ScanAllocation { + const ranked = [...candidates].sort((a, b) => { + const pb = scanPriority(b.relValue, b.overrun); + const pa = scanPriority(a.relValue, a.overrun); + if (pb !== pa) return pb - pa; + return b.overrun - a.overrun; // tie-break: longest-overdue first (helps dead markets eventually run) + }); + const n = Math.max(0, Math.floor(budget)); + const granted = ranked.slice(0, n); + const deferred = ranked.slice(n); + const byTier: Record = { hot: 0, warm: 0, cold: 0 }; + for (const c of granted) byTier[tierOf(c.relValue)] += 1; + return { + granted: granted.map((c) => c.wp), + deferred: deferred.map((c) => c.wp), + byTier, + budget: n, + }; +} diff --git a/packages/bot/src/runtime/state.ts b/packages/bot/src/runtime/state.ts index 9251b51..f22611a 100644 --- a/packages/bot/src/runtime/state.ts +++ b/packages/bot/src/runtime/state.ts @@ -198,6 +198,14 @@ export interface BotState { prune: boolean; updatedAt: number; }; + + /** + * Waypoints with a ship present or inbound (coverage presence), refreshed by the fleet poll + * (`fleetTableManager`) when `SCAN_BUDGET_ON`. The scan-budget scheduler reads this to avoid + * granting `GET /market` reads on ship-absent markets, which return no live prices (wasted budget). + * Undefined until the first poll → the scheduler falls back to ungated (legacy) behaviour. + */ + coverageWps?: ReadonlySet; } export interface CreateStateOptions { diff --git a/packages/shared/src/config.ts b/packages/shared/src/config.ts index 878f2b1..a4abc60 100644 --- a/packages/shared/src/config.ts +++ b/packages/shared/src/config.ts @@ -111,6 +111,10 @@ const RawConfigSchema = z.object({ SCAN_VOL_FACTOR_MAX: num(4), // clamp on volFactor (churning market shortens interval) LANE_VALUE_ALPHA: num(0.3), // EWMA weight for realized net per lane LANE_VALUE_HALFLIFE_MS: num(1_800_000), // staleness half-life for a lane's realized value (30 min) + // LANE_VALUE_* validated on live UPRISING: realized net/lane was highly dispersed (median 21,980, + // p90 60,480, min −90,240, 15.5% negative) so a smoothing alpha 0.3 tames per-trip noise; lanes + // completed fleet-wide at ~0.8/min, for which a 30-min half-life keeps stale lanes fading without + // thrashing. Realized value concentrated in ~12 sinks/goods → TOPK 20 comfortably covers them. LANE_TOPK: num(20), // top-K lanes retained in the registry status block // ── value-driven coverage tiering + reversible pruning + cold re-check (issue #2, phases 4+7) ── @@ -122,6 +126,10 @@ const RawConfigSchema = z.object({ COVERAGE_HOT_MULT: num(2), // rel value ≥ 2× fleet mean ⇒ HOT COVERAGE_WARM_MULT: num(0.75), // rel ≥ 0.75× ⇒ WARM COVERAGE_COLD_MULT: num(0.2), // rel ≥ 0.2× ⇒ COLD; below ⇒ DEAD (never worth a parked probe) + // The DEAD cutoff (rel < 0.2× fleet mean) targets the genuinely-dead tail; live UPRISING showed a + // 15.5% negative-lane share — the empirical dead-market fraction the DEAD tier + scan de-prioritising + // should catch (see the replay calibration test, which asserts the cheap tier captures a comparable + // share of the real value distribution). COVERAGE_TARGET_BASE: num(3), // value-driven probe target floor before signal bonuses COVERAGE_LANE_BONUS: num(1), // +N covered markets per active lane (maturity → wider coverage) COVERAGE_FLEET_BONUS: num(0.5), // +N covered markets per ship in the fleet @@ -130,6 +138,19 @@ const RawConfigSchema = z.object({ COVERAGE_RECHECK_MIN_MS: num(600_000), // floor — promising uncovered market re-visited at most this often (10 min) COVERAGE_RECHECK_MAX_MS: num(21_600_000), // ceiling — even a dead market is re-checked this often, never forgotten (6 h) + // ── global scan-budget priority scheduler (issue #2, phase 5) ────────────────────────────────── + // PR1 gave each market a per-market interval; this makes the SHARED ~2 req/s budget explicit. When + // many markets are due in one sweep, scans are granted highest value×staleness first (not FIFO) up + // to a per-sweep budget that reserves headroom for trade/nav. Default OFF ⇒ legacy fetch-all-due. + SCAN_BUDGET_ON: boolOff, // gate the priority scheduler; off ⇒ refreshDue fetches every due market in order + SCAN_BUDGET_REQ_PER_SEC: num(2), // account request ceiling the budget is computed against (mirror the client) + // Calibrated from 13.45h of live UPRISING (.mjs) telemetry: observed request mix was scans 40.1% / + // actions 55.7% / price-reads 4.3%. 0.4 caps scans at their empirical share so actions keep their + // observed ~60% — matches Steer #2's action-protection intent. At the observed steady-state scan + // rate (5.74/min) this cap never binds; it only bounds a synchronized due-burst. + SCAN_BUDGET_REQ_FRACTION: num(0.4), // fraction of sweep capacity scans may spend (rest reserved for trades) + SCAN_BUDGET_MAX_PER_SWEEP: num(0), // absolute hard cap on reads per sweep; 0 ⇒ fraction-derived only + // ── phase / budget ────────────────────────────────────────────────────── BOOTSTRAP_FLEET_MIN: num(2), CREDIT_TARGET: num(0),