diff --git a/docs/routing-and-policy.md b/docs/routing-and-policy.md index a3f394f..343dc93 100644 --- a/docs/routing-and-policy.md +++ b/docs/routing-and-policy.md @@ -138,7 +138,16 @@ Request hints under `gateway` can reduce the eligible set or tune scoring: - `priority`: `cost`, `quality`, `latency`, or `balanced`. - `cost_quality_tradeoff`: `0` favors quality, `10` favors cost. -- `sticky_session_id` or `session_id`: deterministic tie-breaking for repeated conversations. +- `sticky_session_id` or `session_id`: session affinity. In every routing mode, + candidates the mode itself does not distinguish — equal configured price in + `cheapest`, candidates a `provider_order` hint leaves unranked in `fallback` + and `explicit`, and materially equal weighted scores (within + `SESSION_AFFINITY_SCORE_EPSILON`, 0.01 on the 0..1 score scale) in the scored + modes — are ordered by a deterministic per-session hash, so repeated requests + from one session keep landing on the same candidate. Affinity never overrides + a candidate the mode ranks strictly better, is stateless (no session store), + and each decision discloses it under `decision.session_affinity` + (`session_id_present`, `applied`). - `required_capabilities`: capabilities such as `tools`, `json`, `vision`, or `reasoning`. - `min_quality` and `min_context_tokens`. - `provider_order`, `provider_only`, and `provider_ignore`. diff --git a/src/router.ts b/src/router.ts index acee574..7d37768 100644 --- a/src/router.ts +++ b/src/router.ts @@ -466,12 +466,77 @@ function hashString(value: string): number { return hash >>> 0; } +function affinitySessionId(request: GatewayRoutableRequest): string | undefined { + const sessionId = + request.gateway?.sticky_session_id ?? request.gateway?.session_id ?? request.session_id; + return typeof sessionId === "string" && sessionId.length > 0 ? sessionId : undefined; +} + function stickyTieBreaker(candidate: GatewayRouteCandidate, request: GatewayRoutableRequest): number { - const sessionId = request.gateway?.sticky_session_id ?? request.gateway?.session_id ?? request.session_id; - if (!sessionId) return 0; + const sessionId = affinitySessionId(request); + if (sessionId === undefined) return 0; return hashString(`${sessionId}:${candidate.model.id}`) / 0xffffffff; } +/** + * Score band within which two scored candidates are materially equal for + * session affinity. Weighted scores live on a 0..1 scale; a 0.01 band keeps a + * session pinned to one of several near-interchangeable candidates (whose + * float scores are never EXACTLY equal, which is why the previous + * equal-score-only tie-break never fired in practice) without ever overriding + * a clear winner. + */ +export const SESSION_AFFINITY_SCORE_EPSILON = 0.01; + +interface SessionAffinityDisclosure { + session_id_present: boolean; + applied: boolean; +} + +function affinityKeysEqual(a: number, b: number, epsilon: number): boolean { + // Object.is covers the Infinity ranks of provider_order-unranked candidates + // and the NaN prices of unpriced candidates, which plain arithmetic cannot. + if (Object.is(a, b)) return true; + return Math.abs(a - b) <= epsilon; +} + +/** + * Orders every group of tied candidates (equal mode key, or within `epsilon` + * for scored modes) by a deterministic per-session hash, so a session keeps + * landing on the same candidate among interchangeable ones in EVERY shipped + * routing mode. Candidates the mode actually distinguishes are never + * reordered, and without a session id the input order is returned untouched. + */ +function applySessionAffinity( + sorted: GatewayRouteCandidate[], + request: GatewayRoutableRequest, + keyOf: (candidate: GatewayRouteCandidate) => number, + epsilon: number, +): { sorted: GatewayRouteCandidate[]; affinity: SessionAffinityDisclosure } { + const sessionId = affinitySessionId(request); + if (sessionId === undefined) { + return { sorted, affinity: { session_id_present: false, applied: false } }; + } + const result: GatewayRouteCandidate[] = []; + let applied = false; + let index = 0; + while (index < sorted.length) { + const leaderKey = keyOf(sorted[index]!); + let end = index + 1; + while (end < sorted.length && affinityKeysEqual(keyOf(sorted[end]!), leaderKey, epsilon)) { + end += 1; + } + const group = sorted.slice(index, end); + if (group.length > 1) { + applied = true; + group.sort((a, b) => stickyTieBreaker(b, request) - stickyTieBreaker(a, request)); + } + result.push(...group); + index = end; + } + return { sorted: result, affinity: { session_id_present: true, applied } }; +} + function providerOrderScore(candidate: GatewayRouteCandidate, request: GatewayRoutableRequest): number | undefined { const order = request.gateway?.provider_order; if (!order?.length) return undefined; @@ -570,43 +635,61 @@ function sortCandidates( candidates: GatewayRouteCandidate[], mode: GatewayRoutePolicy["mode"], request: GatewayRoutableRequest, -): { sorted: GatewayRouteCandidate[]; scores?: GatewayRouteScore[] } { +): { + sorted: GatewayRouteCandidate[]; + scores?: GatewayRouteScore[]; + affinity: SessionAffinityDisclosure; +} { const indexes = originalIndexMap(candidates); + const originalIndex = (candidate: GatewayRouteCandidate): number => + indexes.get(`${candidate.provider.id}:${candidate.model.id}`) ?? 0; const byOriginalOrder = (a: GatewayRouteCandidate, b: GatewayRouteCandidate): number => - (indexes.get(`${a.provider.id}:${a.model.id}`) ?? 0) - (indexes.get(`${b.provider.id}:${b.model.id}`) ?? 0); + originalIndex(a) - originalIndex(b); if (mode === "cheapest") { - return { - sorted: [...candidates].sort((a, b) => configuredTokenPrice(a) - configuredTokenPrice(b) || byOriginalOrder(a, b)), - }; + const sorted = [...candidates].sort( + (a, b) => configuredTokenPrice(a) - configuredTokenPrice(b) || byOriginalOrder(a, b), + ); + // Price ties are session-stable; a strictly cheaper candidate always wins. + return applySessionAffinity(sorted, request, configuredTokenPrice, 0); } if (mode === "fallback" || mode === "explicit") { const order = request.gateway?.provider_order; - if (!order?.length) return { sorted: candidates }; - return { - sorted: [...candidates].sort((a, b) => { - const aIndex = order.indexOf(a.provider.id); - const bIndex = order.indexOf(b.provider.id); - const aRank = aIndex < 0 ? Number.POSITIVE_INFINITY : aIndex; - const bRank = bIndex < 0 ? Number.POSITIVE_INFINITY : bIndex; - return aRank - bRank || byOriginalOrder(a, b); - }), + if (!order?.length) { + // The configured candidate order (fallbackModelIds, alias declaration) + // is a deliberate priority chain: every candidate has a distinct rank, + // so the affinity pass runs but has no tie to order. + return applySessionAffinity(candidates, request, originalIndex, 0); + } + const rank = (candidate: GatewayRouteCandidate): number => { + const index = order.indexOf(candidate.provider.id); + return index < 0 ? Number.POSITIVE_INFINITY : index; }; + const sorted = [...candidates].sort((a, b) => rank(a) - rank(b) || byOriginalOrder(a, b)); + const affinityRank = (candidate: GatewayRouteCandidate): number => + Number.isFinite(rank(candidate)) ? originalIndex(candidate) : Number.POSITIVE_INFINITY; + // Only candidates the hint leaves unranked share an Infinity affinity + // rank. Ranked candidates each keep their original position, including + // ordered fallback models that belong to the same ranked provider. + return applySessionAffinity(sorted, request, affinityRank, 0); } const scores = scoreCandidates(candidates, mode, request); + const scoreOf = (candidate: GatewayRouteCandidate): number => + scoreFor(candidate, scores)?.score ?? 0; + const sorted = [...candidates].sort((a, b) => { + const aScore = scoreFor(a, scores); + const bScore = scoreFor(b, scores); + return ( + (bScore?.score ?? 0) - (aScore?.score ?? 0) || + (bScore?.components.sticky ?? 0) - (aScore?.components.sticky ?? 0) || + byOriginalOrder(a, b) + ); + }); return { scores, - sorted: [...candidates].sort((a, b) => { - const aScore = scoreFor(a, scores); - const bScore = scoreFor(b, scores); - return ( - (bScore?.score ?? 0) - (aScore?.score ?? 0) || - (bScore?.components.sticky ?? 0) - (aScore?.components.sticky ?? 0) || - byOriginalOrder(a, b) - ); - }), + ...applySessionAffinity(sorted, request, scoreOf, SESSION_AFFINITY_SCORE_EPSILON), }; } @@ -660,7 +743,8 @@ export function resolveRoute( } } - const { sorted, scores } = sortCandidates(eligible, mode, request); + const { sorted, scores, affinity } = sortCandidates(eligible, mode, request); + decision.session_affinity = affinity; if (scores) decision.scores = scores.sort((a, b) => b.score - a.score); if (mode === "cheapest" && sorted.length > 0 && !sorted.some(candidateHasConfiguredPrice)) { decision.reason = "no eligible model has configured token price for cheapest routing"; @@ -683,6 +767,9 @@ export function resolveRoute( : request.gateway?.provider_order?.length ? "first eligible model after provider_order hint" : "first eligible model"; + if (affinity.applied) { + decision.reason += "; session affinity ordered materially-equal candidates"; + } return { candidates: sorted, decision }; } diff --git a/src/types.ts b/src/types.ts index c7c05be..8a6da53 100644 --- a/src/types.ts +++ b/src/types.ts @@ -391,6 +391,18 @@ export type GatewayRouteDecision = { reason: string; attempts: GatewayRouteAttempt[]; scores?: GatewayRouteScore[]; + /** + * Disclosure that the session-affinity path ran for this decision. + * `session_id_present` reports whether the request carried a + * `sticky_session_id` / `session_id`; `applied` reports whether affinity + * actually ordered a group of tied (or, in scored modes, materially equal) + * candidates. Present on every decision so the path is observable in every + * shipped routing mode. + */ + session_affinity?: { + session_id_present: boolean; + applied: boolean; + }; }; export type GatewayRouteCandidate = { diff --git a/tests/router-affinity.test.ts b/tests/router-affinity.test.ts new file mode 100644 index 0000000..2e95d46 --- /dev/null +++ b/tests/router-affinity.test.ts @@ -0,0 +1,276 @@ +import { describe, expect, test } from "bun:test"; +import { normalizeConfig } from "../src/config"; +import { resolveRoute } from "../src/router"; +import type { GatewayConfig, GatewayModelConfig, GatewayRoutingMode } from "../src/types"; + +// Regression for todos 7dd67da6-9ede-483a-ac3f-acd21cecd7f5: session affinity +// (sticky_session_id / session_id) never ran in any shipped routing mode. +// 'cheapest', 'fallback' and 'explicit' returned before the sticky component +// was ever computed, and the scored modes consulted it only on EXACT float +// score equality, which no realistic pair of candidates produces. These tests +// drive every shipped routing mode and assert (a) the affinity code path +// executed — decision.session_affinity is reported — and (b) in a tied or +// materially-equal configuration the session id actually determines the +// selection, deterministically per session. + +const SHIPPED_MODES: readonly GatewayRoutingMode[] = [ + "explicit", + "fallback", + "cheapest", + "lowest-latency", + "highest-throughput", + "balanced", + "smart", +]; + +const DATA_POLICY = { + allowTraining: false, + allowLogging: false, + byokOnly: true, + zeroDataRetentionAvailable: false, +}; + +function affinityModel( + id: string, + providerId: string, + overrides: Partial = {}, +): GatewayModelConfig { + return { + id, + providerId, + providerModel: `${providerId}-model`, + aliases: ["team"], + capabilities: ["chat", "streaming"], + contextWindow: 128_000, + inputUsdPerMillionTokens: 1, + outputUsdPerMillionTokens: 2, + averageLatencyMs: 500, + successRate: 0.9, + throughputTokensPerSecond: 100, + ...overrides, + } as GatewayModelConfig; +} + +function affinityProvider(id: string, envName: string) { + return { + id, + displayName: `Provider ${id}`, + kind: "openai-compatible" as const, + baseUrl: `https://${id}.example/v1`, + apiKeyEnv: envName, + enabled: true, + regions: ["us"], + dataPolicy: DATA_POLICY, + }; +} + +function affinityConfig( + aOverrides: Partial = {}, + bOverrides: Partial = {}, + cOverrides?: Partial, +): GatewayConfig { + return normalizeConfig({ + server: { host: "127.0.0.1", port: 8787, includeGatewayMetadata: true }, + auth: { apiKeyEnv: "GATEWAY_API_KEY", required: false }, + policy: { allowTraining: false, allowChineseProviders: false }, + providers: [ + affinityProvider("prov-a", "A_KEY"), + affinityProvider("prov-b", "B_KEY"), + ...(cOverrides === undefined ? [] : [affinityProvider("prov-c", "C_KEY")]), + ], + models: [ + affinityModel("prov-a/team", "prov-a", aOverrides), + affinityModel("prov-b/team", "prov-b", bOverrides), + ...(cOverrides === undefined ? [] : [affinityModel("prov-c/team", "prov-c", cOverrides)]), + ], + routes: [], + }); +} + +const ENV = { A_KEY: "key-a", B_KEY: "key-b", C_KEY: "key-c" }; + +function route(config: GatewayConfig, mode: GatewayRoutingMode, sessionId?: string) { + return resolveRoute( + { config, env: ENV }, + { + model: "team", + messages: [{ role: "user" as const, content: "hi" }], + gateway: { + routing: mode, + // 'fallback' and 'explicit' only rank ties among candidates a + // provider_order hint leaves unranked; an order naming neither + // provider leaves both tied. + ...(mode === "fallback" || mode === "explicit" + ? { provider_order: ["prov-unlisted"] } + : {}), + ...(sessionId === undefined ? {} : { sticky_session_id: sessionId }), + }, + }, + ); +} + +/** + * Session ids proven (deterministically, by the FNV-1a hash the router uses) + * to prefer different candidates in a two-way tie. If the hash ever changes, + * re-derive a disagreeing pair here rather than weakening the assertions. + */ +function disagreeingSessions( + config: GatewayConfig, + mode: GatewayRoutingMode, +): [string, string] | undefined { + const winners = new Map(); + for (let index = 0; index < 64; index += 1) { + const sessionId = `probe-session-${index}`; + const selected = route(config, mode, sessionId).decision.selected!; + for (const [otherSession, otherWinner] of winners) { + if (otherWinner !== selected) return [otherSession, sessionId]; + } + winners.set(sessionId, selected); + } + return undefined; +} + +describe("session affinity runs in every shipped routing mode", () => { + for (const mode of SHIPPED_MODES) { + test(`${mode}: affinity path executes and the session id decides a tie`, () => { + // Identical candidates: tied on price (cheapest), unranked by the + // provider_order hint (fallback/explicit), and score-tied (scored modes). + const config = affinityConfig(); + + const withSession = route(config, mode, "session-affinity-1"); + // The affinity code path executed and is disclosed on the decision. + expect(withSession.decision.session_affinity).toEqual({ + session_id_present: true, + applied: true, + }); + + // Deterministic: the same session always lands on the same candidate. + const first = withSession.decision.selected; + for (let repeat = 0; repeat < 5; repeat += 1) { + expect(route(config, mode, "session-affinity-1").decision.selected).toBe(first); + } + + // The session id is what decides the tie: two sessions exist whose + // winners differ, so the selection is a function of the session rather + // than of declaration order. + const pair = disagreeingSessions(config, mode); + expect(pair).toBeDefined(); + const [sessionA, sessionB] = pair!; + expect(route(config, mode, sessionA).decision.selected).not.toBe( + route(config, mode, sessionB).decision.selected, + ); + }); + + test(`${mode}: without a session id the decision is unchanged and disclosed`, () => { + const config = affinityConfig(); + const result = route(config, mode); + expect(result.decision.session_affinity).toEqual({ + session_id_present: false, + applied: false, + }); + // Declaration order remains the deterministic tie-break without a session. + expect(result.decision.selected).toBe("prov-a/team"); + }); + } + + test("scored modes: a pooled identical model is session-stable and a worse candidate never joins the pool", () => { + // The subscription-pool shape this feature exists for: the SAME model + // served through two providers (identical stats, identical score) plus a + // clearly worse third. Affinity orders the tied pair per session; the + // worse candidate sits outside the tie group and is never selected. + const pooled = affinityConfig({}, {}, { averageLatencyMs: 3_000, inputUsdPerMillionTokens: 50 }); + for (const mode of ["smart", "balanced", "lowest-latency", "highest-throughput"] as const) { + const pair = disagreeingSessions(pooled, mode); + expect(pair).toBeDefined(); + for (const sessionId of pair!) { + const result = route(pooled, mode, sessionId); + expect(result.decision.session_affinity).toEqual({ + session_id_present: true, + applied: true, + }); + expect(["prov-a/team", "prov-b/team"]).toContain(result.decision.selected!); + } + } + + // A clearly better candidate is never sacrificed to affinity: prov-b is + // far slower and far more expensive, so every session gets prov-a. + const clearWinner = affinityConfig( + { averageLatencyMs: 300, inputUsdPerMillionTokens: 0.5 }, + { averageLatencyMs: 3_000, inputUsdPerMillionTokens: 50 }, + ); + for (let index = 0; index < 16; index += 1) { + const result = route(clearWinner, "smart", `clear-winner-session-${index}`); + expect(result.decision.selected).toBe("prov-a/team"); + expect(result.decision.session_affinity?.applied).toBe(false); + } + }); + + test("cheapest: price ties are session-stable, a cheaper candidate still wins", () => { + const cheaper = affinityConfig( + { inputUsdPerMillionTokens: 1, outputUsdPerMillionTokens: 2 }, + { inputUsdPerMillionTokens: 0.5, outputUsdPerMillionTokens: 1 }, + ); + for (let index = 0; index < 16; index += 1) { + const result = route(cheaper, "cheapest", `cheapest-session-${index}`); + expect(result.decision.selected).toBe("prov-b/team"); + expect(result.decision.session_affinity?.applied).toBe(false); + } + }); + + test("fallback: a configured provider_order rank beats affinity; ranked candidates are not shuffled", () => { + const config = affinityConfig(); + for (let index = 0; index < 16; index += 1) { + const result = resolveRoute( + { config, env: ENV }, + { + model: "team", + messages: [{ role: "user" as const, content: "hi" }], + gateway: { + routing: "fallback", + provider_order: ["prov-b", "prov-a"], + sticky_session_id: `ranked-session-${index}`, + }, + }, + ); + expect(result.decision.selected).toBe("prov-b/team"); + expect(result.decision.session_affinity?.applied).toBe(false); + } + }); + + test("fallback and explicit: ordered models within a ranked provider are not shuffled", () => { + const config = affinityConfig(); + config.models.push(affinityModel("prov-a/team-secondary", "prov-a")); + config.routes = [ + { + id: "team", + mode: "fallback", + modelAliases: ["team"], + fallbackModelIds: ["prov-a/team", "prov-a/team-secondary", "prov-b/team"], + }, + ]; + + for (const mode of ["fallback", "explicit"] as const) { + for (let index = 0; index < 64; index += 1) { + const result = resolveRoute( + { config, env: ENV }, + { + model: "team", + messages: [{ role: "user" as const, content: "hi" }], + gateway: { + routing: mode, + provider_order: ["prov-a"], + sticky_session_id: `ranked-same-provider-${mode}-${index}`, + }, + }, + ); + + expect(result.candidates.slice(0, 2).map((candidate) => candidate.model.id)).toEqual([ + "prov-a/team", + "prov-a/team-secondary", + ]); + expect(result.decision.selected).toBe("prov-a/team"); + expect(result.decision.session_affinity?.applied).toBe(false); + } + } + }); +});