diff --git a/.changelog/next/changed-issue-4161.md b/.changelog/next/changed-issue-4161.md new file mode 100644 index 0000000000..6f269ee1cf --- /dev/null +++ b/.changelog/next/changed-issue-4161.md @@ -0,0 +1 @@ +- Factored the repeated 'bump the seeded Claude provider tier' migration into a reusable `makeSeededProviderTierMigration` factory (plus a shared test runner), so the next model-tier default bump is a small data table instead of another ~200-line hand-copy diff --git a/scripts/migrations/_lib.js b/scripts/migrations/_lib.js index 9caaa07cc9..64f59405f9 100644 --- a/scripts/migrations/_lib.js +++ b/scripts/migrations/_lib.js @@ -17,6 +17,14 @@ * `writeMediaRegistry` collapse the strict-read → absent/unreadable skip → * bucket-array guard shell shared by every migration that patches * `data/media-models.json` (244, 247, …). + * 7. Seeded-provider-tier bumps — `makeSeededProviderTierMigration` collapses + * the exact-match → rewrite-models → swap-retired-pointers shell that + * 032 / 058 / 153 / 206 each hand-copied. Those four stay frozen; the + * factory is for the next bump. + * + * Families 5 and 7 both target `data/providers.json` and share its + * read → parse → shape-guard preamble via `readProvidersDoc`; each still owns + * its own log copy and result shape. * * The runner (`scripts/run-migrations.js`) explicitly skips `_`-prefixed * files so this module is never imported as a migration. @@ -748,6 +756,43 @@ export function makeBrainSeedMigration({ logTag, entityType, seedIds, seedLabel, const PROVIDERS_REL_PATH = 'data/providers.json'; +/** + * Read + parse + shape-guard `data/providers.json` for the two provider + * migration families below. Returns a discriminated result: + * + * - `{ ok: false, reason: 'no-file' | 'unreadable' | 'bad-shape', path }` — + * absent (a fresh install seeds from data.reference), unparseable, or missing + * its `providers` map. In every case the caller leaves the file untouched: a + * migration must never clobber a user's stored apiKeys to "fix" a shape. + * - `{ ok: true, config, providers, path }` — mutate `providers` in place, then + * persist the whole `config` with `writeJsonAtomic(path, config)`. + * + * Deliberately silent: each family owns its own log copy (they say different + * things about what the skip costs the user), so the wording stays per-family + * while the read shell is shared. `err` carries the parse failure for the + * `'unreadable'` message. + */ +async function readProvidersDoc({ rootDir }) { + const path = join(rootDir, PROVIDERS_REL_PATH); + const raw = await readFile(path, 'utf-8').catch((err) => { + if (err.code === 'ENOENT') return null; + throw err; + }); + if (raw == null) return { ok: false, reason: 'no-file', path }; + + let config; + try { + config = JSON.parse(raw); + } catch (err) { + return { ok: false, reason: 'unreadable', path, err }; + } + + const providers = config?.providers; + if (!providers || typeof providers !== 'object') return { ok: false, reason: 'bad-shape', path }; + + return { ok: true, config, providers, path }; +} + /** * Build a provider-seed migration's `up()`. Returns `{ up }`, so a migration is * `export default makeProviderSeedMigration({ label, defs })`. @@ -772,30 +817,15 @@ export function makeProviderSeedMigration({ label, defs }) { const noun = defs.length === 1 ? 'provider' : 'providers'; async function up({ rootDir }) { - const providersPath = join(rootDir, PROVIDERS_REL_PATH); - const raw = await readFile(providersPath, 'utf-8').catch((err) => { - if (err.code === 'ENOENT') return null; - throw err; - }); - if (raw == null) { - console.log(`📄 ${PROVIDERS_REL_PATH} not present — skipping (fresh install seeds ${label} from data.reference)`); - return { ok: false, reason: 'no-file', added: 0 }; - } - - let config; - try { - config = JSON.parse(raw); - } catch (err) { - console.log(`⚠️ ${PROVIDERS_REL_PATH}: invalid JSON, skipping (${err.message})`); - return { ok: false, reason: 'unreadable', added: 0 }; - } - - if (!config || typeof config !== 'object' || !config.providers || typeof config.providers !== 'object') { - console.log(`⚠️ ${PROVIDERS_REL_PATH}: unexpected shape, skipping`); - return { ok: false, reason: 'bad-shape', added: 0 }; + const doc = await readProvidersDoc({ rootDir }); + if (!doc.ok) { + if (doc.reason === 'no-file') console.log(`📄 ${PROVIDERS_REL_PATH} not present — skipping (fresh install seeds ${label} from data.reference)`); + else if (doc.reason === 'unreadable') console.log(`⚠️ ${PROVIDERS_REL_PATH}: invalid JSON, skipping (${doc.err.message})`); + else console.log(`⚠️ ${PROVIDERS_REL_PATH}: unexpected shape, skipping`); + return { ok: false, reason: doc.reason, added: 0 }; } - const providers = config.providers; + const { config, providers, path: providersPath } = doc; let added = 0; for (const def of defs) { @@ -828,6 +858,169 @@ export function makeProviderSeedMigration({ label, defs }) { return { up }; } +// ---- seeded-provider-tier bump migration family ---- +// +// Migrations 032 / 058 / 153 / 206 each bump ONE model tier of the seeded +// Claude provider entries (`claude-code`, `claude-code-tui`, and their +// `-bedrock` twins) from a retired model id to its replacement. `setup-data.js` +// merges *missing* provider entries but never updates existing ones, so an +// existing install only picks a new default up when a migration rewrites its +// `data/providers.json`. +// +// All four are the same shell with different data, and it is a deliberately +// conservative shell: +// - `models` is rewritten only when it matches the prior seeded list EXACTLY +// (order-sensitive). A curated list — reordered, trimmed, extended — is left +// alone rather than silently reset to the shipped default. +// - On a rewrite, every retired id is swapped to its mapped replacement +// wherever it appears (the `models` array and any tier pointer). Pointers +// parked on still-current models are preserved. +// - Bedrock ids map like-for-like, so a long-context `…[1m]` pin lands on the +// new `…[1m]` id instead of silently dropping to the standard-context id. +// - The "already-new models but stale pointer" case is repaired: an install +// freshly seeded from the new data.reference can still carry a tier pointer +// at a now-absent id, which would leave it requesting a model it no longer +// lists. +// +// The four shipped copies stay FROZEN and do not consume this factory. A +// migration is the historical record of what it did to an install; rewriting an +// applied one to route through shared code would change that record and risk +// changing its behavior for anyone who has not run it yet. This factory is the +// shell the NEXT tier bump uses, and `_testHelpers.js#runSeededProviderTierMigrationTests` +// is its companion test runner. + +// The four tier pointers a provider entry can park on a model id. A bump must +// consider all of them, not just `defaultModel` — an install that pinned +// `heavyModel` to the retired id would otherwise be left pointing at a model +// that is no longer in its `models` list. +const TIER_POINTER_KEYS = ['defaultModel', 'lightModel', 'mediumModel', 'heavyModel']; + +/** + * The post-bump `models` array for one target, derived from its prior seeded + * list plus its id map — so the two can never drift apart in a caller's data + * table. Exported for the shared test runner (and for a migration that wants to + * assert the shape it is about to ship). + */ +export const seededProviderTierModels = ({ oldModels, idMap }) => + oldModels.map((id) => (Object.hasOwn(idMap, id) ? idMap[id] : id)); + +// Order-sensitive equality. Reordering the seeded list counts as customization +// (skipped) — that is the "left alone" promise 032/058/153/206 all made. +const sameModelList = (a, b) => + Array.isArray(a) && Array.isArray(b) && a.length === b.length && a.every((v, i) => v === b[i]); + +// Swap any tier pointer still referencing a retired id to its replacement. +// `Object.hasOwn` before the lookup: a bare `idMap[provider[key]]` would inherit +// an Object.prototype member for a pointer literally named `constructor` / +// `toString`. Unreachable via the UI or any seed, but the guard costs nothing. +// Mutates in place; returns true if any pointer changed. +const swapTierPointers = (provider, idMap) => { + let changed = false; + for (const key of TIER_POINTER_KEYS) { + const mapped = Object.hasOwn(idMap, provider[key]) ? idMap[provider[key]] : null; + if (mapped) { + provider[key] = mapped; + changed = true; + } + } + return changed; +}; + +/** + * Build a seeded-provider-tier bump migration's `up()`. Returns `{ up }`, so a + * migration collapses to `export default makeSeededProviderTierMigration({…})` + * over a small data table. + * + * - `targets` — `{ [providerId]: { oldModels, idMap } }`. + * - `oldModels` — the EXACT prior seeded `models` array, in order. Only an + * exact match is rewritten. + * - `idMap` — retired id → replacement id, for every id this bump + * retires. Bedrock targets list the plain and `[1m]` ids separately so + * each maps like-for-like. Ids absent from the map are still-current + * tiers and are carried through untouched. + * Sibling providers that ship identical lists (the CLI/TUI pair, the two + * Bedrock entries) should share one spec object. + * - `tierLabel` — the human phrase for the tier being bumped, used in the log + * lines (e.g. `'opus tier claude-opus-5'`). + * + * The summary log reports each touched provider's resulting `defaultModel` — + * the "what will this install actually run now" value — regardless of which + * tier was bumped. + * + * Resolves to `{ ok, reason: 'no-file' | 'unreadable' | 'bad-shape' | + * 'no-change' | 'bumped', touched, alreadyCurrent, customized }`, where the + * three arrays hold provider ids. + */ +export function makeSeededProviderTierMigration({ targets, tierLabel }) { + // Derive each target's post-bump list once, at build time. + const plans = Object.entries(targets).map(([id, target]) => ({ + id, + idMap: target.idMap, + oldModels: target.oldModels, + newModels: seededProviderTierModels(target), + })); + + async function up({ rootDir }) { + const doc = await readProvidersDoc({ rootDir }); + if (!doc.ok) { + if (doc.reason === 'no-file') console.log(`📄 ${PROVIDERS_REL_PATH} not present — skipping (fresh install seeds from data.reference with the new defaults)`); + else if (doc.reason === 'unreadable') console.log(`⚠️ ${PROVIDERS_REL_PATH}: invalid JSON, skipping (${doc.err.message})`); + else console.log(`⚠️ ${PROVIDERS_REL_PATH}: no providers map — skipping`); + return { ok: false, reason: doc.reason, touched: [], alreadyCurrent: [], customized: [] }; + } + + const { config, providers, path: providersPath } = doc; + + const touched = []; + const alreadyCurrent = []; + const customized = []; + + for (const plan of plans) { + // `Object.hasOwn` rather than a bare `providers[plan.id]` probe: every + // plain object inherits `constructor` / `toString`, so a target id + // colliding with one would read as present and get "bumped" on the + // prototype. No seeded provider id is one of those, but the probe stays + // honest for free. + if (!Object.hasOwn(providers, plan.id)) continue; + const provider = providers[plan.id]; + if (!provider || typeof provider !== 'object') continue; + + if (sameModelList(provider.models, plan.oldModels)) { + // Prior seeded list → rewrite models + swap retired pointers. + provider.models = [...plan.newModels]; + swapTierPointers(provider, plan.idMap); + touched.push(plan.id); + continue; + } + + if (sameModelList(provider.models, plan.newModels)) { + // Models already current — only act if a tier pointer is still orphaned + // at a now-absent retired id. + if (swapTierPointers(provider, plan.idMap)) touched.push(plan.id); + else alreadyCurrent.push(plan.id); + continue; + } + + customized.push(plan.id); + } + + if (touched.length === 0) { + const notes = []; + if (alreadyCurrent.length > 0) notes.push(`already current: ${alreadyCurrent.join(', ')}`); + if (customized.length > 0) notes.push(`customized: ${customized.join(', ')}`); + console.log(`✅ ${PROVIDERS_REL_PATH}: nothing to bump for ${tierLabel}${notes.length ? ` (${notes.join('; ')})` : ''}`); + return { ok: true, reason: 'no-change', touched, alreadyCurrent, customized }; + } + + await writeJsonAtomic(providersPath, config); + const summary = touched.map((id) => `${id} (default: ${providers[id].defaultModel})`).join(', '); + console.log(`📝 ${PROVIDERS_REL_PATH}: updated ${summary} → ${tierLabel}`); + return { ok: true, reason: 'bumped', touched, alreadyCurrent, customized }; + } + + return { up }; +} + /** * Build the per-subdir prompt-drift tables that `scripts/setup-data.js` uses * for its "pending migration" warning by sweeping every numbered migration's diff --git a/scripts/migrations/_lib.test.js b/scripts/migrations/_lib.test.js index 779a9117f1..d8fe47ee45 100644 --- a/scripts/migrations/_lib.test.js +++ b/scripts/migrations/_lib.test.js @@ -9,16 +9,22 @@ * exercise the branches the helper guards behind opt-in flags. * * Also home to the shell suites for the other migration factories — - * `makeSplitMigration`'s flags and `makeProviderSeedMigration`'s whole - * read → guard → add-missing-ids → write shell, which the six provider-seed - * migrations (149/152/185/195/201/231) used to re-assert one file at a time. + * `makeSplitMigration`'s flags, `makeProviderSeedMigration`'s whole + * read → guard → add-missing-ids → write shell (which the six provider-seed + * migrations 149/152/185/195/201/231 used to re-assert one file at a time), and + * `makeSeededProviderTierMigration` — including a differential suite proving it + * reproduces the shipped 153 / 206 tier bumps byte-for-byte without those + * frozen migrations being rewritten to consume it. */ import { describe, it, expect, beforeEach, afterEach } from 'vitest'; import { mkdtempSync, rmSync, writeFileSync, readFileSync, mkdirSync, existsSync } from 'fs'; import { tmpdir } from 'os'; import { join } from 'path'; -import { applyPromptReplaceMigration, md5, readLayoutsDoc, writeLayoutsDoc, makeSplitMigration, makeProviderSeedMigration } from './_lib.js'; +import { applyPromptReplaceMigration, md5, readLayoutsDoc, writeLayoutsDoc, makeSplitMigration, makeProviderSeedMigration, makeSeededProviderTierMigration, seededProviderTierModels } from './_lib.js'; +import { runSeededProviderTierMigrationTests } from './_testHelpers.js'; +import sonnet5Migration from './153-claude-default-sonnet-5.js'; +import opus5Migration from './206-claude-default-opus-5.js'; const FILENAME = 'pipeline-fake.md'; const BODY_OLD = '# OLD\n'; @@ -425,3 +431,279 @@ describe('makeProviderSeedMigration', () => { expect({}.name).toBeUndefined(); }); }); + +// ---- makeSeededProviderTierMigration ---- +// +// The factory has to earn the right to be used by the NEXT tier bump, and the +// only evidence that counts is the shipped ones: migrations 153 and 206 stay +// frozen and are re-expressed here as `targets` tables, then run head-to-head +// against factory-built equivalents over the same fixtures. If the factory ever +// diverges from what those two actually do to a `data/providers.json`, the +// differential suite below fails. + +const SONNET_5_BARE = { + oldModels: ['claude-haiku-4-5', 'claude-sonnet-4-6', 'claude-opus-4-8'], + idMap: { 'claude-sonnet-4-6': 'claude-sonnet-5' }, +}; +const SONNET_5_BEDROCK = { + oldModels: [ + 'us.anthropic.claude-haiku-4-5', + 'us.anthropic.claude-sonnet-4-6', + 'global.anthropic.claude-opus-4-8', + 'global.anthropic.claude-opus-4-8[1m]', + ], + idMap: { 'us.anthropic.claude-sonnet-4-6': 'us.anthropic.claude-sonnet-5' }, +}; +const SONNET_5_TARGETS = { + 'claude-code': SONNET_5_BARE, + 'claude-code-tui': SONNET_5_BARE, + 'claude-code-bedrock': SONNET_5_BEDROCK, + 'claude-code-tui-bedrock': SONNET_5_BEDROCK, +}; + +const OPUS_5_BARE = { + oldModels: ['claude-haiku-4-5', 'claude-sonnet-5', 'claude-opus-4-8'], + idMap: { 'claude-opus-4-8': 'claude-opus-5' }, +}; +const OPUS_5_BEDROCK = { + oldModels: [ + 'us.anthropic.claude-haiku-4-5', + 'us.anthropic.claude-sonnet-5', + 'global.anthropic.claude-opus-4-8', + 'global.anthropic.claude-opus-4-8[1m]', + ], + idMap: { + 'global.anthropic.claude-opus-4-8': 'global.anthropic.claude-opus-5', + 'global.anthropic.claude-opus-4-8[1m]': 'global.anthropic.claude-opus-5[1m]', + }, +}; +const OPUS_5_TARGETS = { + 'claude-code': OPUS_5_BARE, + 'claude-code-tui': OPUS_5_BARE, + 'claude-code-bedrock': OPUS_5_BEDROCK, + 'claude-code-tui-bedrock': OPUS_5_BEDROCK, +}; + +/** + * Every `data/providers.json` shape a tier bump has to have an opinion about, + * derived from a `targets` table so both parameterizations get the same + * coverage: the prior seeded shape, a survivor pin, an orphan pointer, the + * post-bump shape, a trimmed list, a reordered list, one doc per retired id, + * and a doc carrying an unrelated provider plus a top-level key. + */ +const tierScenarioDocs = (targets) => { + const specs = Object.entries(targets).map(([id, target]) => ({ + id, + target, + retired: target.oldModels.filter((m) => Object.hasOwn(target.idMap, m)), + surviving: target.oldModels.filter((m) => !Object.hasOwn(target.idMap, m)), + newModels: seededProviderTierModels(target), + })); + + const entry = (s, overrides = {}) => ({ + id: s.id, + models: [...s.target.oldModels], + defaultModel: s.retired.at(-1), + lightModel: s.surviving[0], + mediumModel: s.surviving.at(-1), + heavyModel: s.retired.at(-1), + ...overrides, + }); + const doc = (build) => ({ providers: Object.fromEntries(specs.map((s) => [s.id, build(s)])) }); + const bumped = (s, overrides = {}) => entry(s, { + models: [...s.newModels], + defaultModel: s.target.idMap[s.retired.at(-1)], + heavyModel: s.target.idMap[s.retired.at(-1)], + ...overrides, + }); + + const scenarios = { + seeded: doc((s) => entry(s)), + survivorPin: doc((s) => entry(s, { defaultModel: s.surviving[0] })), + orphanPointer: doc((s) => bumped(s, { defaultModel: s.retired.at(-1) })), + alreadyCurrent: doc(bumped), + trimmedList: doc((s) => entry(s, { models: s.target.oldModels.slice(1) })), + reorderedList: doc((s) => entry(s, { models: [s.target.oldModels.at(-1), ...s.target.oldModels.slice(0, -1)] })), + missingPointers: doc((s) => ({ id: s.id, models: [...s.target.oldModels] })), + unrelatedNeighbor: { + activeProvider: 'claude-code', + providers: { + ...Object.fromEntries(specs.map((s) => [s.id, entry(s)])), + 'unrelated-provider': { id: 'unrelated-provider', models: ['some-configured-default'] }, + }, + }, + }; + + // One doc per retired id, so the Bedrock plain-vs-`[1m]` split is exercised + // in both directions rather than only on whichever id the fixture defaults to. + for (const s of specs) { + for (const retiredId of s.retired) { + scenarios[`pin:${s.id}:${retiredId}`] = { + providers: { [s.id]: entry(s, { defaultModel: retiredId, heavyModel: retiredId }) }, + }; + } + } + + return scenarios; +}; + +describe.each([ + ['153 (sonnet tier → claude-sonnet-5)', sonnet5Migration, SONNET_5_TARGETS, 'sonnet tier claude-sonnet-5'], + ['206 (opus tier → claude-opus-5)', opus5Migration, OPUS_5_TARGETS, 'opus tier claude-opus-5'], +])('makeSeededProviderTierMigration reproduces shipped migration %s', (_label, shipped, targets, tierLabel) => { + const factory = makeSeededProviderTierMigration({ targets, tierLabel }); + + let shippedRoot; + let factoryRoot; + + beforeEach(() => { + shippedRoot = mkdtempSync(join(tmpdir(), 'tier-shipped-')); + factoryRoot = mkdtempSync(join(tmpdir(), 'tier-factory-')); + for (const root of [shippedRoot, factoryRoot]) mkdirSync(join(root, 'data'), { recursive: true }); + }); + + afterEach(() => { + for (const root of [shippedRoot, factoryRoot]) rmSync(root, { recursive: true, force: true }); + }); + + const providersFile = (root) => join(root, 'data', 'providers.json'); + + // The fixtures the family deliberately declines to touch. Naming them keeps + // the equality assertion below honest: without this split, a factory that + // wrote nothing at all would "match" the shipped migration on every fixture. + const NO_OP_SCENARIOS = new Set(['alreadyCurrent', 'trimmedList', 'reorderedList']); + + it.each(Object.keys(tierScenarioDocs(targets)))('produces an identical data/providers.json for the %s fixture', async (scenario) => { + const input = JSON.stringify(tierScenarioDocs(targets)[scenario], null, 2) + '\n'; + for (const root of [shippedRoot, factoryRoot]) writeFileSync(providersFile(root), input); + + await shipped.up({ rootDir: shippedRoot }); + await factory.up({ rootDir: factoryRoot }); + + const shippedOut = readFileSync(providersFile(shippedRoot), 'utf-8'); + expect(readFileSync(providersFile(factoryRoot), 'utf-8')).toBe(shippedOut); + if (NO_OP_SCENARIOS.has(scenario)) expect(shippedOut).toBe(input); + else expect(shippedOut).not.toBe(input); + }); + + it('matches the shipped no-op behaviour on an absent file', async () => { + await shipped.up({ rootDir: shippedRoot }); + await factory.up({ rootDir: factoryRoot }); + + expect(existsSync(providersFile(shippedRoot))).toBe(false); + expect(existsSync(providersFile(factoryRoot))).toBe(false); + }); + + it('matches the shipped skip on an unparseable file and on a missing providers map', async () => { + for (const input of ['{ not valid json', '{}\n', '{ "providers": null }\n']) { + for (const root of [shippedRoot, factoryRoot]) writeFileSync(providersFile(root), input); + + await shipped.up({ rootDir: shippedRoot }); + await factory.up({ rootDir: factoryRoot }); + + expect(readFileSync(providersFile(shippedRoot), 'utf-8')).toBe(input); + expect(readFileSync(providersFile(factoryRoot), 'utf-8')).toBe(input); + } + }); +}); + +describe('makeSeededProviderTierMigration — shared contract (206 parameterization)', () => { + runSeededProviderTierMigrationTests({ + migration: makeSeededProviderTierMigration({ targets: OPUS_5_TARGETS, tierLabel: 'opus tier claude-opus-5' }), + targets: OPUS_5_TARGETS, + prefix: 'tier-contract-opus-', + }); +}); + +describe('makeSeededProviderTierMigration — shared contract (153 parameterization)', () => { + runSeededProviderTierMigrationTests({ + migration: makeSeededProviderTierMigration({ targets: SONNET_5_TARGETS, tierLabel: 'sonnet tier claude-sonnet-5' }), + targets: SONNET_5_TARGETS, + prefix: 'tier-contract-sonnet-', + }); +}); + +describe('makeSeededProviderTierMigration — factory-specific guards', () => { + let rootDir; + let providersPath; + + beforeEach(() => { + rootDir = mkdtempSync(join(tmpdir(), 'tier-guards-')); + mkdirSync(join(rootDir, 'data'), { recursive: true }); + providersPath = join(rootDir, 'data/providers.json'); + }); + + afterEach(() => rmSync(rootDir, { recursive: true, force: true })); + + const write = (value) => writeFileSync(providersPath, JSON.stringify(value, null, 2) + '\n'); + const read = () => JSON.parse(readFileSync(providersPath, 'utf-8')); + + it('never hands a target its own copy of the derived models array', async () => { + // Two providers sharing one spec object must not end up sharing one array — + // an in-memory mutation of either would otherwise show up in both. + const mig = makeSeededProviderTierMigration({ targets: OPUS_5_TARGETS, tierLabel: 'opus tier claude-opus-5' }); + write({ + providers: { + 'claude-code': { id: 'claude-code', models: [...OPUS_5_BARE.oldModels], defaultModel: 'claude-opus-4-8' }, + 'claude-code-tui': { id: 'claude-code-tui', models: [...OPUS_5_BARE.oldModels], defaultModel: 'claude-opus-4-8' }, + }, + }); + + await mig.up({ rootDir }); + + const out = read().providers; + out['claude-code'].models.push('mutated'); + expect(out['claude-code-tui'].models).toEqual(['claude-haiku-4-5', 'claude-sonnet-5', 'claude-opus-5']); + expect(OPUS_5_BARE.oldModels).toEqual(['claude-haiku-4-5', 'claude-sonnet-5', 'claude-opus-4-8']); + }); + + it('does not treat an inherited prototype key as a present provider', async () => { + // `providers['constructor']` is truthy on ANY plain object; a bare presence + // probe would "bump" the prototype instead of skipping a target that the + // install simply does not have. + const mig = makeSeededProviderTierMigration({ + targets: { constructor: OPUS_5_BARE }, + tierLabel: 'opus tier claude-opus-5', + }); + write({ providers: {} }); + const before = readFileSync(providersPath, 'utf-8'); + + expect(await mig.up({ rootDir })).toMatchObject({ ok: true, reason: 'no-change', touched: [] }); + + expect(readFileSync(providersPath, 'utf-8')).toBe(before); + expect({}.models).toBeUndefined(); + }); + + it('skips a target whose entry is not an object rather than throwing', async () => { + const mig = makeSeededProviderTierMigration({ targets: { 'claude-code': OPUS_5_BARE }, tierLabel: 'opus tier claude-opus-5' }); + write({ providers: { 'claude-code': 'not-an-object' } }); + const before = readFileSync(providersPath, 'utf-8'); + + expect(await mig.up({ rootDir })).toMatchObject({ ok: true, reason: 'no-change' }); + expect(readFileSync(providersPath, 'utf-8')).toBe(before); + }); + + it('leaves a pointer at an id this bump does not retire alone', async () => { + const mig = makeSeededProviderTierMigration({ targets: { 'claude-code': OPUS_5_BARE }, tierLabel: 'opus tier claude-opus-5' }); + write({ + providers: { + 'claude-code': { + id: 'claude-code', + models: [...OPUS_5_BARE.oldModels], + defaultModel: 'claude-haiku-4-5', + lightModel: 'claude-haiku-4-5', + mediumModel: 'claude-sonnet-5', + heavyModel: 'claude-opus-4-8', + }, + }, + }); + + await mig.up({ rootDir }); + + const after = read().providers['claude-code']; + expect(after.defaultModel).toBe('claude-haiku-4-5'); + expect(after.lightModel).toBe('claude-haiku-4-5'); + expect(after.mediumModel).toBe('claude-sonnet-5'); + expect(after.heavyModel).toBe('claude-opus-5'); + }); +}); diff --git a/scripts/migrations/_testHelpers.js b/scripts/migrations/_testHelpers.js index 834eb78c60..8fd1dbff98 100644 --- a/scripts/migrations/_testHelpers.js +++ b/scripts/migrations/_testHelpers.js @@ -1,10 +1,15 @@ /** - * Shared test scaffolding for hash-driven prompt-replace migrations. - * Companion to `./_lib.js`. The runner skips `_`-prefixed files. + * Shared test scaffolding for the migration families in `./_lib.js`. The runner + * skips `_`-prefixed files, so nothing here is ever executed as a migration. * - * Per-migration `*.test.js` collapses to a `describe` + a single - * `runPromptMigrationTests({ migration, applyMigration, ACCEPTED_OLD_MD5, - * NEW_SHIPPED_MD5, prefix })` call — six standard cases fire inside it. + * - `runPromptMigrationTests` — hash-driven prompt-replace migrations. A + * per-migration `*.test.js` collapses to a `describe` + a single + * `runPromptMigrationTests({ migration, applyMigration, ACCEPTED_OLD_MD5, + * NEW_SHIPPED_MD5, prefix })` call; six standard cases fire inside it. + * - `runSeededProviderTierMigrationTests` — seeded-provider-tier bumps built + * with `makeSeededProviderTierMigration`. Every fixture is derived from the + * migration's own `targets` table, so a bump's test is a `describe` + one + * call and still asserts the full conservative contract. */ import { it, expect, beforeEach, afterEach } from 'vitest'; @@ -13,7 +18,7 @@ import { tmpdir } from 'os'; import { join, dirname } from 'path'; import { fileURLToPath } from 'url'; -import { md5 } from './_lib.js'; +import { md5, seededProviderTierModels } from './_lib.js'; export { md5 }; @@ -141,3 +146,235 @@ export function runPromptMigrationTests({ } }); } + +// ---- seeded-provider-tier bump migrations ---- + +/** + * Expand a migration's `targets` table into everything the fixtures need: which + * ids this bump retires, which survive it, and the post-bump `models` array. + * Derived rather than hand-listed so a caller's data table is the only place a + * model id is written down. + */ +const tierFixtures = (targets) => + Object.entries(targets).map(([id, target]) => ({ + id, + target, + retired: target.oldModels.filter((m) => Object.hasOwn(target.idMap, m)), + surviving: target.oldModels.filter((m) => !Object.hasOwn(target.idMap, m)), + newModels: seededProviderTierModels(target), + })); + +// A provider entry in its prior seeded shape: the old models list, with +// default/heavy parked on the last retired id (the Bedrock `[1m]` variant, when +// there is one) and light/medium on ids this bump does not touch. +const seededTierEntry = (f, overrides = {}) => ({ + id: f.id, + models: [...f.target.oldModels], + defaultModel: f.retired.at(-1), + lightModel: f.surviving[0] ?? f.retired[0], + mediumModel: f.surviving.at(-1) ?? f.retired[0], + heavyModel: f.retired.at(-1), + ...overrides, +}); + +// The same entry after a correct bump. +const bumpedTierEntry = (f, overrides = {}) => ({ + ...seededTierEntry(f), + models: [...f.newModels], + defaultModel: f.target.idMap[f.retired.at(-1)], + heavyModel: f.target.idMap[f.retired.at(-1)], + ...overrides, +}); + +/** + * Standard suite for a migration built with `makeSeededProviderTierMigration`. + * A bump's `*.test.js` collapses to a `describe` + a single + * `runSeededProviderTierMigrationTests({ migration, targets, prefix })` call. + * + * - `migration` — the migration's default export (`{ up }`). + * - `targets` — the SAME `targets` table passed to the factory. Every + * fixture is derived from it, so the suite covers each provider and each + * retired id the bump actually ships. + * - `prefix` — `mkdtempSync` directory name (`'migration-207-'`), so a + * debugger leaves a recognizable sandbox in the temp dir. + * + * The cases assert the family's conservative contract: exact-match-only + * rewrites, like-for-like id mapping (a `[1m]` long-context pin must not drop a + * context tier), still-current pointers preserved, orphan pointers repaired, + * and a byte-for-byte no-op on anything customized or already current. + */ +export function runSeededProviderTierMigrationTests({ migration, targets, prefix }) { + const fixtures = tierFixtures(targets); + const ids = fixtures.map((f) => f.id); + + let rootDir; + let providersPath; + + beforeEach(() => { + rootDir = mkdtempSync(join(tmpdir(), prefix)); + mkdirSync(join(rootDir, 'data'), { recursive: true }); + providersPath = join(rootDir, 'data', 'providers.json'); + }); + + afterEach(() => { + rmSync(rootDir, { recursive: true, force: true }); + }); + + const write = (providers) => writeFileSync(providersPath, JSON.stringify({ providers }, null, 2) + '\n'); + const read = () => JSON.parse(readFileSync(providersPath, 'utf-8')).providers; + const raw = () => readFileSync(providersPath, 'utf-8'); + const entriesFrom = (build) => Object.fromEntries(fixtures.map((f) => [f.id, build(f)])); + + it('declares at least one retired id per target (the table actually bumps something)', () => { + expect(fixtures.length).toBeGreaterThan(0); + for (const f of fixtures) { + expect(f.retired.length).toBeGreaterThan(0); + expect(f.newModels).not.toEqual(f.target.oldModels); + // Every replacement must land in the post-bump list, or a pointer swap + // would leave the provider pinned to a model it does not offer. + for (const replacement of Object.values(f.target.idMap)) { + expect(f.newModels).toContain(replacement); + } + } + }); + + it('rewrites the seeded models list and swaps the retired tier pointers', async () => { + write(entriesFrom(seededTierEntry)); + + const result = await migration.up({ rootDir }); + + expect(result).toMatchObject({ ok: true, reason: 'bumped', customized: [] }); + expect(result.touched.sort()).toEqual([...ids].sort()); + const out = read(); + for (const f of fixtures) { + expect(out[f.id]).toEqual(bumpedTierEntry(f)); + expect(out[f.id].models).toContain(out[f.id].defaultModel); + } + }); + + it('maps every retired id like-for-like (a [1m] long-context pin keeps its context tier)', async () => { + for (const f of fixtures) { + for (const retiredId of f.retired) { + write({ [f.id]: seededTierEntry(f, { defaultModel: retiredId, heavyModel: retiredId }) }); + + await migration.up({ rootDir }); + + const after = read()[f.id]; + expect(after.defaultModel).toBe(f.target.idMap[retiredId]); + expect(after.heavyModel).toBe(f.target.idMap[retiredId]); + expect(after.models).toContain(after.defaultModel); + } + } + }); + + it('preserves a tier pointer parked on a still-current model', async () => { + const withSurvivors = fixtures.filter((f) => f.surviving.length > 0); + expect(withSurvivors.length).toBeGreaterThan(0); + + for (const f of withSurvivors) { + const pinned = f.surviving[0]; + write({ [f.id]: seededTierEntry(f, { defaultModel: pinned }) }); + + await migration.up({ rootDir }); + + const after = read()[f.id]; + expect(after.defaultModel).toBe(pinned); + expect(after.models).toEqual(f.newModels); + expect(after.heavyModel).toBe(f.target.idMap[f.retired.at(-1)]); + } + }); + + it('repairs an orphan retired pointer when the models list is already current', async () => { + // A fresh seed from the new data.reference can still carry a pointer at a + // now-absent id — left alone it would request a model the install no longer + // lists. + write(entriesFrom((f) => bumpedTierEntry(f, { defaultModel: f.retired.at(-1) }))); + + const result = await migration.up({ rootDir }); + + expect(result.touched.sort()).toEqual([...ids].sort()); + const out = read(); + for (const f of fixtures) { + expect(out[f.id].defaultModel).toBe(f.target.idMap[f.retired.at(-1)]); + expect(out[f.id].models).toContain(out[f.id].defaultModel); + } + }); + + it('is a byte-for-byte no-op once models and pointers are current', async () => { + write(entriesFrom(bumpedTierEntry)); + const before = raw(); + + const result = await migration.up({ rootDir }); + + expect(result).toMatchObject({ ok: true, reason: 'no-change', touched: [] }); + expect(result.alreadyCurrent.sort()).toEqual([...ids].sort()); + expect(raw()).toBe(before); + }); + + it('is idempotent — a second run rewrites nothing', async () => { + write(entriesFrom(seededTierEntry)); + await migration.up({ rootDir }); + const afterFirst = raw(); + + await migration.up({ rootDir }); + + expect(raw()).toBe(afterFirst); + }); + + it('skips a curated models list instead of resetting it to the shipped default', async () => { + write(entriesFrom((f) => seededTierEntry(f, { models: f.target.oldModels.slice(1) }))); + const before = raw(); + + const result = await migration.up({ rootDir }); + + expect(result).toMatchObject({ ok: true, reason: 'no-change' }); + expect(result.customized.sort()).toEqual([...ids].sort()); + expect(raw()).toBe(before); + }); + + it('treats a reordered seeded list as customization', async () => { + const reorderable = fixtures.filter((f) => f.target.oldModels.length > 1); + expect(reorderable.length).toBeGreaterThan(0); + + write(Object.fromEntries(reorderable.map((f) => [ + f.id, + seededTierEntry(f, { models: [f.target.oldModels.at(-1), ...f.target.oldModels.slice(0, -1)] }), + ]))); + const before = raw(); + + await migration.up({ rootDir }); + + expect(raw()).toBe(before); + }); + + it('leaves providers outside the target set untouched', async () => { + const unrelated = { id: 'unrelated-provider', models: ['some-configured-default'], defaultModel: 'some-configured-default' }; + write({ ...entriesFrom(seededTierEntry), 'unrelated-provider': unrelated }); + + await migration.up({ rootDir }); + + expect(read()['unrelated-provider']).toEqual(unrelated); + }); + + it('is a no-op when data/providers.json is absent (fresh install seeds from data.reference)', async () => { + expect(await migration.up({ rootDir })).toMatchObject({ ok: false, reason: 'no-file' }); + expect(existsSync(providersPath)).toBe(false); + }); + + it('leaves an unparseable file byte-identical rather than clobbering user data', async () => { + writeFileSync(providersPath, '{ not valid json'); + + expect(await migration.up({ rootDir })).toMatchObject({ ok: false, reason: 'unreadable' }); + expect(raw()).toBe('{ not valid json'); + }); + + it('leaves the file untouched when the providers map is missing or not an object', async () => { + for (const doc of [{}, { providers: null }, { providers: 'nope' }]) { + writeFileSync(providersPath, JSON.stringify(doc, null, 2) + '\n'); + const before = raw(); + + expect(await migration.up({ rootDir })).toMatchObject({ ok: false, reason: 'bad-shape' }); + expect(raw()).toBe(before); + } + }); +}