Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,8 @@ The OpenAI Codex route uses ChatGPT subscription access directly and never sends

The Grok route uses subscription access directly at xAI and never sends its OAuth token to Vercel AI Gateway or OpenAI. Its session is stored privately at `~/.fx/grok-auth.json`, refreshed when needed, and used only with the authenticated xAI catalog and Responses API.

Codex and Grok discover current stable client versions from upstream release metadata without requiring either CLI to be installed. fx caches release metadata for one minute. Opening `/model` or requesting ACP model options refreshes an expired subscription catalog. If a release lookup temporarily fails, fx uses the last successfully fetched version.

To use an AI Gateway API key instead:

```bash
Expand Down
27 changes: 26 additions & 1 deletion src/acp/server.zig
Original file line number Diff line number Diff line change
Expand Up @@ -2036,6 +2036,7 @@ fn handleSetConfigOption(state: *ServerState, alloc: Allocator, msg: *jsonrpc.Me
});
if (comptime !host_target.is_wasm) {
if (session.provider != .gateway) {
try refreshModelCatalogForOptions(state);
var model_available = false;
if (state.capability_resolver.catalogEntries()) |entries| {
for (entries) |entry| {
Expand Down Expand Up @@ -2217,7 +2218,7 @@ fn handleSetConfigOption(state: *ServerState, alloc: Allocator, msg: *jsonrpc.Me
.message = "Failed to persist session provider",
});
};
state.capability_resolver.adoptOwnedCatalog(alloc, &catalog);
state.capability_resolver.adoptOwnedCatalog(alloc, catalog_provider, access, &catalog);
if (staged_credential) |*credential| {
adoptServerCredential(state, credential);
} else {
Expand All @@ -2235,6 +2236,7 @@ fn handleSetConfigOption(state: *ServerState, alloc: Allocator, msg: *jsonrpc.Me
}
}

try refreshModelCatalogForOptions(state);
const current_model = if (state.active_session) |s| s.model else state.selected_model;
const current_mode: []const u8 = if (state.active_session) |s| s.mode else state.cfg.mode_registry.default_mode_id;

Expand All @@ -2259,6 +2261,29 @@ fn handleSetConfigOption(state: *ServerState, alloc: Allocator, msg: *jsonrpc.Me
try state.writer.writeResponse(alloc, msg.id, out.writer.buffered());
}

pub fn refreshModelCatalogForOptions(state: *ServerState) !void {
if (comptime host_target.is_wasm) return;
if (state.cfg.minimal_kernel) return;
const active = if (state.active_session) |*session| session else return;
const provider = catalogProviderFor(state, active.provider) orelse return;
std.debug.assert(state.active_prompt == null);
// Restoring the same session can leave its previous cancellation flag set.
var cancel_flag = std.atomic.Value(bool).init(false);
try state.capability_resolver.refreshIfDue(state.alloc, provider, .{
.access = if (state.cfg.auth_mode == .host_managed)
.host_managed
else
credentials.catalogAccessForCredentialAndAccount(
active.credential_source,
active.api_key,
state.gateway_team,
active.account_id,
),
.endpoint = state.cfg.gateway_models_path,
.cancel_flag = &cancel_flag,
});
}

fn commitActiveSessionProvider(
alloc: Allocator,
session: *ActiveSessionState,
Expand Down
2 changes: 2 additions & 0 deletions src/acp/sessions.zig
Original file line number Diff line number Diff line change
Expand Up @@ -301,6 +301,7 @@ fn writeNewSessionResponse(
msg: *jsonrpc.Message,
session_id: []const u8,
) !void {
try server.refreshModelCatalogForOptions(state);
var out: std.Io.Writer.Allocating = .init(alloc);
defer out.deinit();

Expand Down Expand Up @@ -845,6 +846,7 @@ fn writeLoadSessionResponse(
msg: *jsonrpc.Message,
model: []const u8,
) !void {
try server.refreshModelCatalogForOptions(state);
var out: std.Io.Writer.Allocating = .init(alloc);
defer out.deinit();
try out.writer.writeAll("{\"configOptions\":[");
Expand Down
32 changes: 27 additions & 5 deletions src/core/app/model_cache_runtime.zig
Original file line number Diff line number Diff line change
Expand Up @@ -320,7 +320,7 @@ pub const Runtime = struct {
provider: model_catalog.Provider,
access: credentials.CatalogAccess,
) void {
if (!self.beginLoad(access)) return;
if (!self.beginLoad(access, provider.refresh_interval_ms)) return;

const owned_access = OwnedCatalogAccess.init(self.alloc, access) catch {
self.markFailed(.{
Expand Down Expand Up @@ -350,7 +350,7 @@ pub const Runtime = struct {
provider: model_catalog.Provider,
access: credentials.CatalogAccess,
) void {
if (!self.beginLoad(access)) return;
if (!self.beginLoad(access, provider.refresh_interval_ms)) return;

const result = model_catalog.fetchWithPublicFallback(provider, self.alloc, .{
.access = access,
Expand Down Expand Up @@ -393,7 +393,7 @@ pub const Runtime = struct {
self.mutex.unlock(io_mod.getIo());
}

fn beginLoad(self: *Self, access: credentials.CatalogAccess) bool {
fn beginLoad(self: *Self, access: credentials.CatalogAccess, refresh_interval_ms: ?i64) bool {
self.finishThreadIfDone();

const requested_access = model_catalog.AccessMetadata.init(access);
Expand Down Expand Up @@ -424,13 +424,17 @@ pub const Runtime = struct {

const now = io_mod.milliTimestamp();
self.mutex.lockUncancelable(io_mod.getIo());
const expired = if (refresh_interval_ms) |interval|
now < self.last_attempt_ms or now - self.last_attempt_ms >= interval
else
false;
const should_load = switch (self.state) {
.idle => true,
.failed => now - self.last_attempt_ms >= 1000,
.ready => if (self.outcome.last_failure) |failed|
failed.failure.retryable and now - self.last_attempt_ms >= 1000
expired or (failed.failure.retryable and now - self.last_attempt_ms >= 1000)
else
false,
expired,
.loading => false,
};
if (!should_load) {
Expand Down Expand Up @@ -931,6 +935,24 @@ const StaleCatalog = struct {
}
};

test "model cache expires successful catalogs only when the provider requests refresh" {
for ([_]bool{ false, true }) |expires| {
var runtime = Runtime.init(std.testing.allocator, "/v1/models");
defer runtime.deinit();
var source = AuthChangeCatalog{ .model_id = "first" };
var provider = source.provider();
provider.refresh_interval_ms = if (expires) 60_000 else null;
runtime.loadCooperative(provider, .{ .public_only = .no_credential });
source.model_id = "new-release";
runtime.loadCooperative(provider, .{ .public_only = .no_credential });
try std.testing.expectEqual(@as(usize, 1), source.calls);
runtime.last_attempt_ms -= 60_000;
runtime.loadCooperative(provider, .{ .public_only = .no_credential });
try std.testing.expectEqual(@as(usize, if (expires) 2 else 1), source.calls);
try std.testing.expectEqualStrings(if (expires) "new-release" else "first", runtime.catalog.items[0].id);
}
}

test "model cache clears an old failure after a clean empty refresh" {
var runtime = Runtime.init(std.testing.allocator, "/v1/models");
defer runtime.deinit();
Expand Down
140 changes: 116 additions & 24 deletions src/core/gateway/gateway_provider.zig
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,9 @@ const std = @import("std");
const credentials = @import("../auth/credentials.zig");
const oauth_transport = @import("../auth/oauth_transport.zig");
const model_capabilities = @import("../config/model_capabilities.zig");
const model_provider = @import("../config/model_provider.zig");
const debug_trace = @import("../shared/debug_trace.zig");
const io_mod = @import("../shared/io.zig");
const output_contracts = @import("../output/output_contracts.zig");
const model_catalog = @import("model_catalog.zig");
const model_catalog_metadata = @import("model_catalog_metadata.zig");
Expand Down Expand Up @@ -149,11 +151,58 @@ const CapabilityResolverState = enum {
pub const CapabilityResolver = struct {
catalog: std.ArrayList(model_catalog.ModelCatalogEntry) = .empty,
state: CapabilityResolverState = .idle,
last_attempt_ms: i64 = 0,
requested_access: ?model_catalog.AccessMetadata = null,
provider_id: ?model_provider.ProviderId = null,

pub fn deinit(self: *CapabilityResolver, alloc: Allocator) void {
model_catalog.freeModelCatalog(alloc, &self.catalog);
}

/// The owner must exclude capability readers during refresh. Borrowed
/// catalog entries stay valid until the next exclusive refresh or adoption.
pub fn refreshIfDue(
self: *CapabilityResolver,
alloc: Allocator,
provider: model_catalog.Provider,
input: model_catalog.FetchInput,
) model_capabilities.ResolveError!void {
const now = io_mod.milliTimestamp();
const requested = model_catalog.AccessMetadata.init(input.access);
const access_changed = self.provider_id != provider.provider_id or
self.requested_access == null or !std.meta.eql(self.requested_access.?, requested);
const expired = if (provider.refresh_interval_ms) |interval|
now < self.last_attempt_ms or now - self.last_attempt_ms >= interval
else
false;
if (self.state != .idle and !access_changed and !expired) return;

const result = model_catalog.fetchWithPublicFallback(provider, alloc, input);
const loaded = switch (result) {
.loaded => |loaded| loaded,
.failed => |failed| {
if (failed.failure.category == .cancellation) return error.Cancelled;
self.last_attempt_ms = now;
self.requested_access = requested;
self.provider_id = provider.provider_id;
if (access_changed) {
if (self.catalog.items.len > 0) debug_trace.logf("gateway", "dropping model catalog after access changed entries={d}", .{self.catalog.items.len});
model_catalog.freeModelCatalog(alloc, &self.catalog);
self.catalog = .empty;
}
if (self.catalog.items.len == 0) self.state = .failed;
debug_trace.logf("gateway", "model catalog refresh failed category={t} retained={}", .{ failed.failure.category, self.state == .ready });
return;
},
};
model_catalog.freeModelCatalog(alloc, &self.catalog);
self.catalog = loaded.catalog;
self.state = .ready;
self.last_attempt_ms = now;
self.requested_access = requested;
self.provider_id = provider.provider_id;
}

pub fn resolve(
self: *CapabilityResolver,
alloc: Allocator,
Expand All @@ -163,30 +212,7 @@ pub const CapabilityResolver = struct {
fallback: model_capabilities.Capabilities,
) model_capabilities.ResolveError!model_capabilities.Capabilities {
if (self.state == .idle) {
const result = model_catalog.fetchWithPublicFallback(provider, alloc, input);
const loaded = switch (result) {
.loaded => |loaded| loaded,
.failed => |failed| {
const failure = failed.failure;
if (failure.category == .cancellation) {
debug_trace.logf(
"gateway",
"model catalog lookup outcome=cancelled model={s}",
.{model},
);
return failCapabilities(error.Cancelled);
}
self.state = .failed;
debug_trace.logf(
"gateway",
"model catalog lookup outcome=fetch_failed model={s} category={t}",
.{ model, failure.category },
);
return fallback;
},
};
self.catalog = loaded.catalog;
self.state = .ready;
try self.refreshIfDue(alloc, provider, input);
}

if (self.state == .failed) {
Expand Down Expand Up @@ -244,12 +270,17 @@ pub const CapabilityResolver = struct {
pub fn adoptOwnedCatalog(
self: *CapabilityResolver,
alloc: Allocator,
provider: model_catalog.Provider,
access: credentials.CatalogAccess,
owned_catalog: *std.ArrayList(model_catalog.ModelCatalogEntry),
) void {
model_catalog.freeModelCatalog(alloc, &self.catalog);
self.catalog = owned_catalog.*;
owned_catalog.* = .empty;
self.state = .ready;
self.last_attempt_ms = io_mod.milliTimestamp();
self.requested_access = .init(access);
self.provider_id = provider.provider_id;
}
};

Expand Down Expand Up @@ -333,6 +364,67 @@ const FakeCatalog = struct {
}
};

test "capability resolver refreshes expired snapshots at the owner boundary and retains a usable catalog" {
const alloc = std.testing.allocator;
var fake = FakeCatalog{ .outcome = .unavailable };
var provider = fake.provider();
provider.refresh_interval_ms = 60_000;
var resolver: CapabilityResolver = .{};
defer resolver.deinit(alloc);
const input = model_catalog.FetchInput{ .endpoint = "https://example.invalid" };
try resolver.refreshIfDue(alloc, provider, input);
try std.testing.expect(resolver.catalogEntries() == null);
fake.outcome = .ready;
try resolver.refreshIfDue(alloc, provider, input);
try std.testing.expectEqual(@as(usize, 1), fake.calls);
resolver.last_attempt_ms -= 60_000;
// Worker capability reads keep the established snapshot until its owner refreshes.
_ = try resolver.resolve(alloc, provider, input, "provider/model", .{});
try std.testing.expectEqual(@as(usize, 1), fake.calls);
try resolver.refreshIfDue(alloc, provider, input);
try std.testing.expectEqual(@as(usize, 2), fake.calls);
try std.testing.expectEqual(@as(usize, 1), resolver.catalogEntries().?.len);
fake.outcome = .unavailable;
resolver.last_attempt_ms -= 60_000;
try resolver.refreshIfDue(alloc, provider, input);
try std.testing.expectEqual(@as(usize, 3), fake.calls);
try std.testing.expectEqualStrings("provider/model", resolver.catalogEntries().?[0].id);
}

test "capability refresh does not retain a catalog for changed access" {
const alloc = std.testing.allocator;
var fake = FakeCatalog{ .outcome = .ready };
var resolver: CapabilityResolver = .{};
defer resolver.deinit(alloc);
try resolver.refreshIfDue(alloc, fake.provider(), .{ .endpoint = "https://example.invalid" });
fake.outcome = .unavailable;
try resolver.refreshIfDue(alloc, fake.provider(), .{
.endpoint = "https://example.invalid",
.access = .host_managed,
});
try std.testing.expect(resolver.catalogEntries() == null);
try std.testing.expectEqual(@as(usize, 2), fake.calls);
}

test "capability refresh keys host-managed catalogs by provider" {
const alloc = std.testing.allocator;
var fake = FakeCatalog{ .outcome = .ready };
var provider = fake.provider();
provider.refresh_interval_ms = 60_000;
var resolver: CapabilityResolver = .{};
defer resolver.deinit(alloc);
const input = model_catalog.FetchInput{ .endpoint = "https://example.invalid", .access = .host_managed };
try resolver.refreshIfDue(alloc, provider, input);
provider.provider_id = .codex;
try resolver.refreshIfDue(alloc, provider, input);
try std.testing.expectEqual(@as(usize, 2), fake.calls);
fake.outcome = .unavailable;
provider.provider_id = .grok;
try resolver.refreshIfDue(alloc, provider, input);
try std.testing.expectEqual(@as(usize, 3), fake.calls);
try std.testing.expect(resolver.catalogEntries() == null);
}

test "available capabilities never fetch and use a completed catalog snapshot" {
const alloc = std.testing.allocator;
var fake = FakeCatalog{ .outcome = .ready };
Expand Down
3 changes: 3 additions & 0 deletions src/core/gateway/model_catalog.zig
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
const std = @import("std");
const credentials = @import("../auth/credentials.zig");
const model_provider = @import("../config/model_provider.zig");
const collections = @import("../shared/collections.zig");
const debug_trace = @import("../shared/debug_trace.zig");
const io_mod = @import("../shared/io.zig");
Expand Down Expand Up @@ -138,6 +139,8 @@ pub const Provider = struct {
/// When set, context must remain valid until every in-flight `fetch` returns.
context: ?*anyopaque = null,
fetch_fn: FetchFn,
provider_id: model_provider.ProviderId = .gateway,
refresh_interval_ms: ?i64 = null,

/// Returns owned catalog entries; the caller frees them with `freeModelCatalog`.
pub fn fetch(self: Provider, alloc: Allocator, input: FetchInput) Allocator.Error!ProviderResult {
Expand Down
Loading
Loading