From 5604ec43cc85638b22ab9eb95f499df2b4f21313 Mon Sep 17 00:00:00 2001 From: Reza Karamad Date: Thu, 3 Sep 2026 23:07:14 +0200 Subject: [PATCH 1/9] feat: add GCP Memorystore for Valkey IAM auth and private-CA TLS support Signed-off-by: Reza Karamad --- .../cluster-topology/cluster-node-row.tsx | 10 + .../connection/ClusterConnectionGroup.tsx | 2 +- .../components/connection/ConnectionEntry.tsx | 2 +- .../src/components/ui/connection-modal.tsx | 12 +- apps/frontend/src/state/epics/valkeyEpics.ts | 9 +- .../valkey-features/cluster/clusterSlice.ts | 2 +- .../connection/connectionSlice.ts | 4 +- apps/metrics/package.json | 1 + apps/metrics/src/effects/monitor-stream.js | 20 +- apps/metrics/src/index.js | 15 ++ apps/metrics/src/utils/gcp-iam-provider.js | 24 +++ apps/metrics/src/valkey-client.js | 33 ++- apps/server/package.json | 1 + apps/server/src/__tests__/connection.test.ts | 60 ++++++ apps/server/src/actions/connection.ts | 2 +- apps/server/src/connection.ts | 27 ++- apps/server/src/gcp-iam-provider.ts | 16 ++ apps/server/src/iam-token-refresh.ts | 44 ++++ apps/server/src/metrics-orchestrator.ts | 27 ++- apps/server/src/valkey-client.ts | 20 +- docker/description.md | 17 +- .../src/content/docs/configuration/metrics.md | 5 + .../src/content/docs/configuration/server.md | 5 + .../src/content/docs/features/connections.md | 4 + .../src/content/docs/reference/limitations.md | 2 +- package-lock.json | 193 +++++++++++++++++- 26 files changed, 513 insertions(+), 44 deletions(-) create mode 100644 apps/metrics/src/utils/gcp-iam-provider.js create mode 100644 apps/server/src/gcp-iam-provider.ts create mode 100644 apps/server/src/iam-token-refresh.ts diff --git a/apps/frontend/src/components/cluster-topology/cluster-node-row.tsx b/apps/frontend/src/components/cluster-topology/cluster-node-row.tsx index 77516533..cf9f2821 100644 --- a/apps/frontend/src/components/cluster-topology/cluster-node-row.tsx +++ b/apps/frontend/src/components/cluster-topology/cluster-node-row.tsx @@ -109,6 +109,16 @@ export function ClusterNodeRow({ awsReplicationGroupId: primaryConfig.awsReplicationGroupId, }, })) + } else if (primaryConfig.authType === "gcp-iam") { + // GCP IAM: tokens are minted from ambient credentials, no password needed + dispatch(connectPending({ + connectionId, + connectionDetails: { + ...baseDetails, + username: primaryConfig.username ?? "", + authType: "gcp-iam", + }, + })) } else if (R.isNotNil(encryptedPassword)) { // Password already encrypted from existing cluster connection — do NOT re-encrypt dispatch(connectPending({ diff --git a/apps/frontend/src/components/connection/ClusterConnectionGroup.tsx b/apps/frontend/src/components/connection/ClusterConnectionGroup.tsx index 5c0cbf30..8cdf68d5 100644 --- a/apps/frontend/src/components/connection/ClusterConnectionGroup.tsx +++ b/apps/frontend/src/components/connection/ClusterConnectionGroup.tsx @@ -101,7 +101,7 @@ export const ClusterConnectionGroup = ({ clusterId, connections, highlight = "", const handleConnectLatest = () => { if (!lastOpenedNode) return const { password, authType } = lastOpenedNode.connection.connectionDetails - if (authType !== "iam" && R.isNil(password) && onPasswordRequired) { + if (authType !== "iam" && authType !== "gcp-iam" && R.isNil(password) && onPasswordRequired) { onPasswordRequired(lastOpenedNode.connectionId) return } diff --git a/apps/frontend/src/components/connection/ConnectionEntry.tsx b/apps/frontend/src/components/connection/ConnectionEntry.tsx index d1737fd8..cf38d8c8 100644 --- a/apps/frontend/src/components/connection/ConnectionEntry.tsx +++ b/apps/frontend/src/components/connection/ConnectionEntry.tsx @@ -43,7 +43,7 @@ export const ConnectionEntry = ({ const handleDisconnect = () => dispatch(closeConnection({ connectionId })) const handleConnect = () => { const { password, authType } = connection.connectionDetails - if (authType !== "iam" && R.isNil(password) && onPasswordRequired) { + if (authType !== "iam" && authType !== "gcp-iam" && R.isNil(password) && onPasswordRequired) { onPasswordRequired(connectionId) return } diff --git a/apps/frontend/src/components/ui/connection-modal.tsx b/apps/frontend/src/components/ui/connection-modal.tsx index 574c7904..1046a7e8 100644 --- a/apps/frontend/src/components/ui/connection-modal.tsx +++ b/apps/frontend/src/components/ui/connection-modal.tsx @@ -168,7 +168,7 @@ export function ConnectionModal({ - onConnectionDetailsChange({ ...connectionDetails, authType: value as "password" | "iam" }) + onConnectionDetailsChange({ ...connectionDetails, authType: value as "password" | "iam" | "gcp-iam" }) } value={connectionDetails.authType ?? "password"} > @@ -180,6 +180,10 @@ export function ConnectionModal({ +
+ + +
@@ -222,6 +226,12 @@ export function ConnectionModal({ /> + ) : connectionDetails.authType === "gcp-iam" ? ( +

+ Uses Application Default Credentials to mint and rotate a short-lived access + token. Authenticates as the default user — no username or + password required. +

) : (
diff --git a/apps/frontend/src/state/epics/valkeyEpics.ts b/apps/frontend/src/state/epics/valkeyEpics.ts index 427f111e..9257978c 100644 --- a/apps/frontend/src/state/epics/valkeyEpics.ts +++ b/apps/frontend/src/state/epics/valkeyEpics.ts @@ -57,8 +57,8 @@ export const connectionEpic = (store: Store) => select(connectPending), filter(() => !selectIsAtConnectionLimit(store.getState())), mergeMap(async (action) => { - const { password } = action.payload.connectionDetails - if (R.isNil(password) || action.payload.connectionDetails.authType === "iam") return action + const { password, authType } = action.payload.connectionDetails + if (R.isNil(password) || authType === "iam" || authType === "gcp-iam") return action // Password is dispatched as plaintext if secureStorage is unavailable const decryptedPassword = password.length > 0 && secureStorage.isAvailable() ? await secureStorage.decrypt(password) : password @@ -283,7 +283,8 @@ export const autoReconnectEpic = (store: Store) => .filter(([, connection]) => connection.status === DISCONNECTED) .filter(([, connection]) => R.isNotNil(connection.connectionDetails.password) || - connection.connectionDetails.authType === "iam") + connection.connectionDetails.authType === "iam" || + connection.connectionDetails.authType === "gcp-iam") if (disconnectedConnections.length > 0) { console.log(`Auto-reconnecting ${disconnectedConnections.length} connection(s)`) @@ -316,7 +317,7 @@ export const autoResumeEpic = (store: Store) => .filter(([, connection]) => isAutoResumeEligible(connection)) .forEach(([connectionId, connection]) => { const { password, authType } = connection.connectionDetails - if (authType === "iam" || (R.isNotNil(password) && R.isEmpty(password))) { + if (authType === "iam" || authType === "gcp-iam" || (R.isNotNil(password) && R.isEmpty(password))) { if (connection.status !== DISCONNECTED) { store.dispatch(connectPending({ connectionId, diff --git a/apps/frontend/src/state/valkey-features/cluster/clusterSlice.ts b/apps/frontend/src/state/valkey-features/cluster/clusterSlice.ts index 15ca494b..2e042818 100644 --- a/apps/frontend/src/state/valkey-features/cluster/clusterSlice.ts +++ b/apps/frontend/src/state/valkey-features/cluster/clusterSlice.ts @@ -16,7 +16,7 @@ export interface PrimaryNode { //TODO: Add handling and UI for uploading cert caCertPath?: string replicas: ReplicaNode[]; - authType?: "password" | "iam"; + authType?: "password" | "iam" | "gcp-iam"; awsRegion?: string; awsReplicationGroupId?: string; } diff --git a/apps/frontend/src/state/valkey-features/connection/connectionSlice.ts b/apps/frontend/src/state/valkey-features/connection/connectionSlice.ts index 9db74e15..8e23d7bd 100644 --- a/apps/frontend/src/state/valkey-features/connection/connectionSlice.ts +++ b/apps/frontend/src/state/valkey-features/connection/connectionSlice.ts @@ -38,7 +38,7 @@ export interface ConnectionDetails { // JSON module availability check jsonModuleAvailable?: boolean; endpointType: EndpointType - authType?: "password" | "iam" + authType?: "password" | "iam" | "gcp-iam" awsRegion?: string awsReplicationGroupId?: string /** @@ -88,7 +88,7 @@ export const isAutoResumeEligible = (connection: ConnectionState): boolean => { if (userDisconnected) return false const { password, authType } = connectionDetails - if (authType === "iam" || (R.isNotNil(password) && R.isEmpty(password))) + if (authType === "iam" || authType === "gcp-iam" || (R.isNotNil(password) && R.isEmpty(password))) return status !== DISCONNECTED return R.isNil(password) } diff --git a/apps/metrics/package.json b/apps/metrics/package.json index 72841422..d5447e4f 100644 --- a/apps/metrics/package.json +++ b/apps/metrics/package.json @@ -16,6 +16,7 @@ "@smithy/signature-v4": "^5.3.13", "@valkey/valkey-glide": "^2.4.0", "express": "^4.21.2", + "google-auth-library": "^11.0.2", "heap-js": "^2.7.1", "iovalkey": "^0.3.3", "ramda": "^0.31.3", diff --git a/apps/metrics/src/effects/monitor-stream.js b/apps/metrics/src/effects/monitor-stream.js index 73f366f0..08d1748b 100644 --- a/apps/metrics/src/effects/monitor-stream.js +++ b/apps/metrics/src/effects/monitor-stream.js @@ -1,7 +1,9 @@ import { Subject, timer, race, firstValueFrom, defer, of } from "rxjs" import { exhaustMap, catchError, map } from "rxjs" import Valkey from "iovalkey" +import { readFileSync } from "node:fs" import { ElastiCacheIAMProvider } from "../utils/elasticache-iam-provider.js" +import { GcpIAMProvider } from "../utils/gcp-iam-provider.js" function getConnectionOptions() { const host = process.env.VALKEY_HOST @@ -10,16 +12,26 @@ function getConnectionOptions() { const verifyTlsCertificate = process.env.VALKEY_VERIFY_CERT let tls = undefined if (process.env.VALKEY_TLS === "true") { - tls = verifyTlsCertificate === "false" ? { rejectUnauthorized: false } : {} + if (verifyTlsCertificate === "false") { + tls = { rejectUnauthorized: false } + } else if (process.env.VALKEY_CA_CERT_PATH) { + tls = { ca: readFileSync(process.env.VALKEY_CA_CERT_PATH) } + } else { + tls = {} + } } return { host, port, username, tls } } async function getPassword() { const username = process.env.VALKEY_USERNAME - return process.env.VALKEY_AUTH_TYPE === "iam" - ? await new ElastiCacheIAMProvider(username, process.env.VALKEY_REPLICATION_GROUP_ID, process.env.VALKEY_AWS_REGION).getCredentials() - : process.env.VALKEY_PASSWORD + if (process.env.VALKEY_AUTH_TYPE === "iam") { + return await new ElastiCacheIAMProvider(username, process.env.VALKEY_REPLICATION_GROUP_ID, process.env.VALKEY_AWS_REGION).getCredentials() + } + if (process.env.VALKEY_AUTH_TYPE === "gcp-iam") { + return await new GcpIAMProvider().getCredentials() + } + return process.env.VALKEY_PASSWORD } /** diff --git a/apps/metrics/src/index.js b/apps/metrics/src/index.js index d9321797..28c6f74d 100644 --- a/apps/metrics/src/index.js +++ b/apps/metrics/src/index.js @@ -15,6 +15,7 @@ import { bigKeysQuerySchema, cpuQuerySchema, memoryQuerySchema, parseQuery } fro import { sanitizeUrl } from "./utils/helpers.js" import { setupNdjsonCleaner, stopNdjsonCleaner } from "./effects/ndjson-cleaner.js" import { createValkeyClient } from "./valkey-client.js" +import { GcpIAMProvider } from "./utils/gcp-iam-provider.js" import { scanBigKeys } from "./analyzers/scan-big-keys.js" async function main() { @@ -31,6 +32,19 @@ async function main() { const client = await createValkeyClient(cfg) const ownNodeId = sanitizeUrl(`${process.env.VALKEY_HOST}-${process.env.VALKEY_PORT}`) + // GCP OAuth2 tokens expire ~1h; rotate the connection password before then so + // reconnects keep authenticating. AWS IAM refreshes natively inside Glide. + const gcpTokenRefresh = process.env.VALKEY_AUTH_TYPE === "gcp-iam" + ? setInterval(async () => { + try { + await client.updateConnectionPassword(await new GcpIAMProvider().getCredentials(), true) + } catch (err) { + console.error("[gcp-iam] token refresh error:", err.message) + } + }, 45 * 60 * 1000) + : undefined + gcpTokenRefresh?.unref?.() + await setupNdjsonCleaner(cfg) await setupCollectors(client, cfg) @@ -225,6 +239,7 @@ async function main() { try { await stopNdjsonCleaner() await stopCollectors() + if (gcpTokenRefresh) clearInterval(gcpTokenRefresh) if (client) { client.close() } diff --git a/apps/metrics/src/utils/gcp-iam-provider.js b/apps/metrics/src/utils/gcp-iam-provider.js new file mode 100644 index 00000000..6eef4636 --- /dev/null +++ b/apps/metrics/src/utils/gcp-iam-provider.js @@ -0,0 +1,24 @@ +import { GoogleAuth } from "google-auth-library" + +// GCP Memorystore for Valkey IAM auth uses a short-lived OAuth2 access token as the AUTH password. +// We mint the token from Application Default Credentials (Workload Identity in GKE, +// the metadata server on Google Compute Engine, or GOOGLE_APPLICATION_CREDENTIALS locally). +class GcpIAMProvider { + #auth + + constructor() { + this.#auth = new GoogleAuth({ + scopes: "https://www.googleapis.com/auth/cloud-platform", + }) + } + + async getCredentials() { + const token = await this.#auth.getAccessToken() + if (!token) { + throw new Error("Unable to mint a GCP access token from Application Default Credentials") + } + return token + } +} + +export { GcpIAMProvider } diff --git a/apps/metrics/src/valkey-client.js b/apps/metrics/src/valkey-client.js index 395d6559..f21dcdff 100644 --- a/apps/metrics/src/valkey-client.js +++ b/apps/metrics/src/valkey-client.js @@ -1,4 +1,6 @@ import { GlideClient, GlideClusterClient, ServiceType, NodeDiscoveryMode } from "@valkey/valkey-glide" +import { readFileSync } from "node:fs" +import { GcpIAMProvider } from "./utils/gcp-iam-provider.js" const SUPPORTED_VALKEY_MODES = new Set(["standalone", "cluster"]) @@ -30,22 +32,35 @@ export const createValkeyClient = async (cfg = {}) => { region: process.env.VALKEY_AWS_REGION, }, } - : process.env.VALKEY_PASSWORD ? { - username: process.env.VALKEY_USERNAME, - password: process.env.VALKEY_PASSWORD, - } : undefined + : process.env.VALKEY_AUTH_TYPE === "gcp-iam" + ? { + // "default" is the only supported username for GCP IAM authentication + // https://docs.cloud.google.com/memorystore/docs/valkey/manage-iam-auth#error-messages + username: "default", + password: await new GcpIAMProvider().getCredentials(), + } + : process.env.VALKEY_PASSWORD ? { + username: process.env.VALKEY_USERNAME, + password: process.env.VALKEY_PASSWORD, + } : undefined const useTLS = process.env.VALKEY_TLS === "true" + // Glide's TLS runs in its Rust core, so a custom CA must be passed explicitly + // via `rootCertificates` (Node's trust store / NODE_EXTRA_CA_CERTS do not apply). + const caCertPath = process.env.VALKEY_CA_CERT_PATH + const tlsAdvancedConfiguration = !useTLS + ? undefined + : process.env.VALKEY_VERIFY_CERT === "false" + ? { insecure: true } + : caCertPath + ? { rootCertificates: readFileSync(caCertPath) } + : undefined const sharedOptions = { addresses, credentials, useTLS, advancedConfiguration: { - ...(useTLS && process.env.VALKEY_VERIFY_CERT === "false" && { - tlsAdvancedConfiguration: { - insecure: true, - }, - }), + ...(tlsAdvancedConfiguration && { tlsAdvancedConfiguration }), connectionTimeout: 30000, }, requestTimeout: 5000, diff --git a/apps/server/package.json b/apps/server/package.json index c7572f13..0f863a6c 100644 --- a/apps/server/package.json +++ b/apps/server/package.json @@ -16,6 +16,7 @@ "@valkey/valkey-glide": "^2.4.0", "express": "^4.21.2", "express-rate-limit": "^8.3.1", + "google-auth-library": "^11.0.2", "helmet": "^8.2.0", "p-limit": "^6.1.0", "ramda": "^0.31.3", diff --git a/apps/server/src/__tests__/connection.test.ts b/apps/server/src/__tests__/connection.test.ts index eeaa3020..dfbb2702 100644 --- a/apps/server/src/__tests__/connection.test.ts +++ b/apps/server/src/__tests__/connection.test.ts @@ -1,8 +1,12 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ import { describe, it, mock, beforeEach, afterEach } from "node:test" import assert from "node:assert" +import fs from "node:fs" +import os from "node:os" +import path from "node:path" import { GlideClient, GlideClusterClient, InfoOptions } from "@valkey/valkey-glide" import { buildConnectionId, KEY_EVICTION_POLICY, VALKEY, toNodeId } from "valkey-common" +import { GoogleAuth } from "google-auth-library" import { _resetConnectInFlight, _resetInFlightClusterClients, @@ -347,6 +351,62 @@ describe("connectToValkey", () => { }) }) + it("should mint an access token as the password when authType is gcp-iam", async () => { + const standalone = buildStandaloneMock({ clusterEnabled: "1" }) + const cluster = buildClusterMock() + mock.method(GoogleAuth.prototype, "getAccessToken", async () => "fake-gcp-token") + + const gcpPayload = { + ...DEFAULT_PAYLOAD, + connectionDetails: { + ...DEFAULT_PAYLOAD.connectionDetails, + authType: "gcp-iam" as const, + username: "ignored@project.iam.gserviceaccount.com", + password: undefined, + }, + } + + await withMockedClients(standalone, cluster, async () => { + await connectToValkey(ctx(), mockWs, gcpPayload) + + const calls = (GlideClusterClient.createClient as any).mock.calls + assert.ok(calls.length > 0) + const config = calls[0].arguments[0] + assert.strictEqual(config.credentials.password, "fake-gcp-token") + assert.strictEqual(config.credentials.username, "default") + assert.strictEqual(config.credentials.iamConfig, undefined) + }) + }) + + it("passes the connection's caCertPath to Glide as rootCertificates", async () => { + const standalone = buildStandaloneMock({ clusterEnabled: "1" }) + const cluster = buildClusterMock() + const caPath = path.join(os.tmpdir(), `valkey-ca-${process.pid}-${Date.now()}.pem`) + fs.writeFileSync(caPath, "-----BEGIN CERTIFICATE-----\ntestca\n-----END CERTIFICATE-----\n") + + const tlsPayload = { + ...DEFAULT_PAYLOAD, + connectionDetails: { + ...DEFAULT_PAYLOAD.connectionDetails, + tls: true, + verifyTlsCertificate: true, + caCertPath: caPath, + }, + } + + try { + await withMockedClients(standalone, cluster, async () => { + await connectToValkey(ctx(), mockWs, tlsPayload) + const config = (GlideClusterClient.createClient as any).mock.calls[0].arguments[0] + const ca = config.advancedConfiguration.tlsAdvancedConfiguration.rootCertificates + assert.ok(Buffer.isBuffer(ca)) + assert.match(ca.toString(), /BEGIN CERTIFICATE/) + }) + } finally { + fs.rmSync(caPath, { force: true }) + } + }) + it("should handle connection errors", async () => { const error = new Error("Connection failed") const originalCreateClient = GlideClient.createClient diff --git a/apps/server/src/actions/connection.ts b/apps/server/src/actions/connection.ts index 25076102..1eb4739e 100644 --- a/apps/server/src/actions/connection.ts +++ b/apps/server/src/actions/connection.ts @@ -17,7 +17,7 @@ export interface ConnectionDetails { //TODO: Add handling and UI for uploading cert caCertPath?: string; endpointType: EndpointType; - authType?: "password" | "iam"; + authType?: "password" | "iam" | "gcp-iam"; awsRegion?: string; awsReplicationGroupId?: string; /** diff --git a/apps/server/src/connection.ts b/apps/server/src/connection.ts index ab606621..84d8a32a 100644 --- a/apps/server/src/connection.ts +++ b/apps/server/src/connection.ts @@ -26,6 +26,8 @@ import { subscribe } from "./node-watchers" import { clearCpuSamples } from "./node-utilization" import { createClusterValkeyClient, createStandaloneValkeyClient } from "./valkey-client" import { isConnectionAuthorized } from "./session" +import { mintGcpAccessToken } from "./gcp-iam-provider" +import { registerGcpTokenRefresh, unregisterGcpTokenRefresh } from "./iam-token-refresh" export type ConnectionContext = { clients: Map @@ -171,7 +173,7 @@ async function connectToValkeyLocked( const { host, port, username, password, tls: useTLS, - verifyTlsCertificate, authType, awsRegion, awsReplicationGroupId, + verifyTlsCertificate, caCertPath, authType, awsRegion, awsReplicationGroupId, } = payload.connectionDetails const db = payload.connectionDetails.db @@ -193,7 +195,9 @@ async function connectToValkeyLocked( region: awsRegion!, }, } - : password ? { username, password } : undefined + : authType === "gcp-iam" + ? { username: "default", password: await mintGcpAccessToken() } + : password ? { username, password } : undefined let standaloneClient: GlideClient | undefined @@ -256,6 +260,7 @@ async function connectToValkeyLocked( credentials, useTLS, verifyTlsCertificate, + caCertPath, }) // Open the registration gate: the metrics process spawned below will POST @@ -330,6 +335,7 @@ async function connectToValkeyLocked( credentials, useTLS, verifyTlsCertificate, + caCertPath, databaseId: clusterDatabaseId, }) inFlightClusterClients.set(clusterId, ownInflight) @@ -366,6 +372,7 @@ async function connectToValkeyLocked( } shouldCloseClusterClientOnError = false + if (authType === "gcp-iam") registerGcpTokenRefresh(clusterClient, `cluster ${clusterId}`) return clusterClient } finally { if (ownInflight && inFlightClusterClients.get(clusterId) === ownInflight) { @@ -408,6 +415,7 @@ async function connectToValkeyLocked( credentials, useTLS, verifyTlsCertificate, + caCertPath, databaseId: db, }) clients.set(connectionId, { client: standaloneClient }) @@ -430,6 +438,7 @@ async function connectToValkeyLocked( }, }) + if (authType === "gcp-iam") registerGcpTokenRefresh(standaloneClient, connectionId) return standaloneClient } catch (err) { @@ -479,7 +488,7 @@ export async function discoverTopology( const { discoveryId, connectionDetails } = payload const { host, port, username, password, tls: useTLS, - verifyTlsCertificate, authType, awsRegion, awsReplicationGroupId, + verifyTlsCertificate, caCertPath, authType, awsRegion, awsReplicationGroupId, } = connectionDetails const addresses = [{ host, port: Number(port) }] @@ -493,7 +502,9 @@ export async function discoverTopology( region: awsRegion!, }, } - : password ? { username, password } : undefined + : authType === "gcp-iam" + ? { username: "default", password: await mintGcpAccessToken() } + : password ? { username, password } : undefined let client: GlideClient | undefined try { @@ -501,7 +512,7 @@ export async function discoverTopology( // database selection is irrelevant for cluster commands. Skip // `databaseId` entirely so Glide does not issue `SELECT` (cluster nodes // reject `SELECT` even for db 0). - client = await createStandaloneValkeyClient({ addresses, credentials, useTLS, verifyTlsCertificate }) + client = await createStandaloneValkeyClient({ addresses, credentials, useTLS, verifyTlsCertificate, caCertPath }) const { discoveredClusterNodes } = await discoverCluster(client, { connectionDetails }) if (Object.keys(discoveredClusterNodes).length < 1) { throw new Error("Unable to discover cluster") @@ -617,8 +628,12 @@ export async function discoverCluster( awsRegion: payload.connectionDetails.awsRegion, awsReplicationGroupId: payload.connectionDetails.awsReplicationGroupId, }), + ...(payload.connectionDetails.authType === "gcp-iam" && { + authType: "gcp-iam" as const, + }), tls: payload.connectionDetails.tls, verifyTlsCertificate: payload.connectionDetails.verifyTlsCertificate, + caCertPath: payload.connectionDetails.caCertPath, replicas: [], } } @@ -642,6 +657,7 @@ export async function discoverCluster( username?: string, tls: boolean, verifyTlsCertificate: boolean, + caCertPath?: string, replicas: { id: string; host: string; port: number }[]; }>) @@ -753,6 +769,7 @@ export function teardownConnection( clients.delete(connectionId) if (connection && ![...clients.values()].some((c) => c.client === connection.client)) { + unregisterGcpTokenRefresh(connection.client) try { connection.client.close() } catch (error) { diff --git a/apps/server/src/gcp-iam-provider.ts b/apps/server/src/gcp-iam-provider.ts new file mode 100644 index 00000000..fac70635 --- /dev/null +++ b/apps/server/src/gcp-iam-provider.ts @@ -0,0 +1,16 @@ +import { GoogleAuth } from "google-auth-library" + +// GCP Memorystore for Valkey IAM auth uses a short-lived OAuth2 access token as the AUTH password. +// We mint the token from Application Default Credentials (Workload Identity in GKE, +// the metadata server on Google Compute Engine, or GOOGLE_APPLICATION_CREDENTIALS locally). +const auth = new GoogleAuth({ + scopes: "https://www.googleapis.com/auth/cloud-platform", +}) + +export async function mintGcpAccessToken(): Promise { + const token = await auth.getAccessToken() + if (!token) { + throw new Error("Unable to mint a GCP access token from Application Default Credentials") + } + return token +} diff --git a/apps/server/src/iam-token-refresh.ts b/apps/server/src/iam-token-refresh.ts new file mode 100644 index 00000000..61302d68 --- /dev/null +++ b/apps/server/src/iam-token-refresh.ts @@ -0,0 +1,44 @@ +import { GlideClient, GlideClusterClient, ClosingError } from "@valkey/valkey-glide" +import { mintGcpAccessToken } from "./gcp-iam-provider" + +// GCP OAuth2 access tokens live ~1 hour. Glide keeps existing connections +// authenticated after expiry, but new/reconnecting connections need a fresh +// token, so we re-mint and push it to every node connection well before expiry. +const REFRESH_INTERVAL_MS = 45 * 60 * 1000 + +type RefreshableClient = GlideClient | GlideClusterClient + +const refreshTimers = new Map() + +// Rotate the connection password for a gcp-iam client on a timer. +// Keyed by the client instance so shared cluster clients are only scheduled once; the timer +// self-clears once the client is closed (updateConnectionPassword throws ClosingError), +// so callers do not have to unregister at every close site. +export function registerGcpTokenRefresh(client: RefreshableClient, label: string): void { + if (refreshTimers.has(client)) return + + const timer = setInterval(async () => { + try { + const token = await mintGcpAccessToken() + await client.updateConnectionPassword(token, true) + } catch (error) { + if (error instanceof ClosingError) { + unregisterGcpTokenRefresh(client) + return + } + console.error(`Error refreshing GCP IAM token for ${label}:`, error) + } + }, REFRESH_INTERVAL_MS) + + // Do not keep the process alive solely for the refresh timer. + timer.unref?.() + refreshTimers.set(client, timer) +} + +export function unregisterGcpTokenRefresh(client: RefreshableClient): void { + const timer = refreshTimers.get(client) + if (timer) { + clearInterval(timer) + refreshTimers.delete(client) + } +} diff --git a/apps/server/src/metrics-orchestrator.ts b/apps/server/src/metrics-orchestrator.ts index 7b2cef0e..78da3034 100644 --- a/apps/server/src/metrics-orchestrator.ts +++ b/apps/server/src/metrics-orchestrator.ts @@ -21,6 +21,8 @@ import { DEPLOYMENT_TYPE, sanitizeUrl, toNodeId } from "valkey-common" import { discoverCluster, belongsToCluster } from "./connection" import { ConnectionDetails } from "./actions/connection" import { createOrchestratorValkeyClient } from "./valkey-client" +import { mintGcpAccessToken } from "./gcp-iam-provider" +import { registerGcpTokenRefresh } from "./iam-token-refresh" // Assumes nodeId is unique among all clusters export type MetricsServerMap = Map { // Surface any insecure TLS connection: disabling certificate validation exposes the @@ -30,6 +33,17 @@ const buildSharedOptions = ({ ) } + // Glide's TLS runs in its Rust core, so a private CA (the connection's + // `caCertPath`) must be passed via `rootCertificates`; Node's trust store + // and NODE_EXTRA_CA_CERTS do not apply. + const tlsAdvancedConfiguration = !useTLS + ? undefined + : verifyTlsCertificate === false + ? { insecure: true } + : caCertPath + ? { rootCertificates: readFileSync(caCertPath) } + : undefined + return { addresses, credentials, @@ -41,11 +55,7 @@ const buildSharedOptions = ({ // mandatory for cluster. ...(typeof databaseId === "number" && databaseId > 0 && { databaseId }), advancedConfiguration: { - ...(useTLS && verifyTlsCertificate === false && { - tlsAdvancedConfiguration: { - insecure: true, - }, - }), + ...(tlsAdvancedConfiguration && { tlsAdvancedConfiguration }), connectionTimeout: 30000, }, requestTimeout: 5000, diff --git a/docker/description.md b/docker/description.md index 6d80f234..1a665237 100644 --- a/docker/description.md +++ b/docker/description.md @@ -67,6 +67,21 @@ $ docker run -d --name valkey-admin -p 8080:8080 \ valkey/valkey-admin ``` +### GCP Memorystore for Valkey with IAM authentication + +Runs with Application Default Credentials (Workload Identity in GKE, the metadata +server on GCE, or a mounted `GOOGLE_APPLICATION_CREDENTIALS` key file locally). +No password is supplied — the token is minted and rotated automatically. + +```console +$ docker run -d --name valkey-admin -p 8080:8080 \ + -e VALKEY_HOST=your-instance-endpoint \ + -e VALKEY_PORT=6379 \ + -e VALKEY_AUTH_TYPE=gcp-iam \ + -e VALKEY_TLS=true \ + valkey/valkey-admin +``` + ### Environment variables Here are all the relevant environment variables for configuration @@ -80,7 +95,7 @@ Here are all the relevant environment variables for configuration | `VALKEY_USERNAME` | Valkey username | — | | `VALKEY_PASSWORD` | Valkey password | — | | `VALKEY_TLS` | Enable TLS | `false` | -| `VALKEY_AUTH_TYPE` | Authentication type (`password`, `iam`) | `password` | +| `VALKEY_AUTH_TYPE` | Authentication type (`password`, `iam`, `gcp-iam`) | `password` | | `HOT_KEYS_COUNT` | Maximum hot keys returned per query | `50` | | `COMMAND_LOGS_COUNT` | Maximum command log entries returned per query | `100` | | `KEY_VALUE_SIZE_LIMIT_BYTES` | Max value size (bytes) shown in the Key Browser before a size warning | `2048` | diff --git a/docs-site/src/content/docs/configuration/metrics.md b/docs-site/src/content/docs/configuration/metrics.md index 27c9a6d0..6354490b 100644 --- a/docs-site/src/content/docs/configuration/metrics.md +++ b/docs-site/src/content/docs/configuration/metrics.md @@ -99,11 +99,16 @@ Enable TLS. Compared as the literal string `"true"`. Verify the TLS server certificate. When TLS is enabled and this is `"false"`, certificate verification is skipped — useful for development against self-signed certs, but not for production. +### `VALKEY_CA_CERT_PATH` + +Filesystem path to a PEM CA certificate used to verify the Valkey server's TLS certificate — for both the Glide client and the iovalkey MONITOR stream. Only consulted when `VALKEY_TLS=true` and verification is enabled. Glide performs TLS in its Rust core, so a private CA must be supplied this way rather than via Node's trust store or `NODE_EXTRA_CA_CERTS`. + ### `VALKEY_AUTH_TYPE` Selects the credentials provider. - **`"iam"`** — use AWS ElastiCache IAM authentication via `ElastiCacheIAMProvider`. Requires `VALKEY_USERNAME`, `VALKEY_AWS_REGION`, and `VALKEY_REPLICATION_GROUP_ID`. +- **`"gcp-iam"`** — use GCP Memorystore for Valkey IAM authentication via `GcpIAMProvider`. Mints a short-lived OAuth2 access token from Application Default Credentials and rotates it before expiry. Authenticates as the `default` user — the only username Memorystore supports — so `VALKEY_USERNAME` is ignored. - **anything else** — password authentication using `VALKEY_USERNAME` / `VALKEY_PASSWORD`. ### `VALKEY_AWS_REGION` diff --git a/docs-site/src/content/docs/configuration/server.md b/docs-site/src/content/docs/configuration/server.md index 0baf99c1..97134935 100644 --- a/docs-site/src/content/docs/configuration/server.md +++ b/docs-site/src/content/docs/configuration/server.md @@ -115,6 +115,10 @@ Verify the TLS server certificate. Compared as the literal string `"true"`. Leav - **Default:** `false` +### `VALKEY_CA_CERT_PATH` + +Filesystem path to a PEM CA certificate used to verify the Valkey server's TLS certificate. Only consulted when `VALKEY_TLS=true` and verification is enabled (`VALKEY_VERIFY_CERT` is not `false`). Glide performs TLS in its Rust core, so a private CA must be supplied this way — Node's trust store and `NODE_EXTRA_CA_CERTS` do not apply. Typical use: mount your server CA (for example a Memorystore for Valkey CA) as a secret and point this at the mounted file. + ### `VALKEY_ENDPOINT_TYPE` Tells the orchestrator how to interpret `VALKEY_HOST` / `VALKEY_PORT` when discovering cluster topology. @@ -127,6 +131,7 @@ Tells the orchestrator how to interpret `VALKEY_HOST` / `VALKEY_PORT` when disco Selects the credentials provider for the initial connection. - **`"iam"`** — use AWS ElastiCache IAM authentication. Requires `VALKEY_USERNAME`, `VALKEY_AWS_REGION`, and `VALKEY_REPLICATION_GROUP_ID`. +- **`"gcp-iam"`** — use GCP Memorystore for Valkey IAM authentication. Mints a short-lived OAuth2 access token from Application Default Credentials (Workload Identity in GKE, the metadata server on GCE, or `GOOGLE_APPLICATION_CREDENTIALS` locally) and rotates it before expiry. Authenticates as the `default` user — the only username Memorystore supports — so `VALKEY_USERNAME` is ignored. - **anything else** — fall back to password authentication using `VALKEY_USERNAME` / `VALKEY_PASSWORD`. - **Default:** `"password"` diff --git a/docs-site/src/content/docs/features/connections.md b/docs-site/src/content/docs/features/connections.md index 35215262..e8e2b351 100644 --- a/docs-site/src/content/docs/features/connections.md +++ b/docs-site/src/content/docs/features/connections.md @@ -68,6 +68,10 @@ For ElastiCache clusters with IAM authentication enabled, provide: Valkey Admin generates short-lived IAM auth tokens automatically. +### GCP IAM Authentication + +For Memorystore for Valkey instances with IAM authentication enabled, select **GCP IAM**. No username or password is required — Valkey Admin mints a short-lived OAuth2 access token from the ambient Application Default Credentials (Workload Identity in GKE, the metadata server on GCE, or `GOOGLE_APPLICATION_CREDENTIALS` locally) and rotates it before expiry, including across every cluster node. Connections authenticate as the `default` user, which is the only username Memorystore for Valkey supports. Enable TLS as well, since Memorystore recommends in-transit encryption whenever IAM authentication is used. + ## TLS Enable TLS for encrypted connections. Optionally verify the server certificate — disable verification only for self-signed certificates in development environments. diff --git a/docs-site/src/content/docs/reference/limitations.md b/docs-site/src/content/docs/reference/limitations.md index 5498754a..0015255b 100644 --- a/docs-site/src/content/docs/reference/limitations.md +++ b/docs-site/src/content/docs/reference/limitations.md @@ -14,7 +14,7 @@ For runtime issues and fixes, see the [Troubleshooting guide](/reference/trouble ## TLS -- **mTLS is not currently supported.** Standard TLS with password authentication or AWS ElastiCache IAM authentication is available. +- **mTLS is not currently supported.** Standard TLS with password authentication, AWS ElastiCache IAM authentication, or GCP Memorystore for Valkey IAM authentication is available. ## Managed services diff --git a/package-lock.json b/package-lock.json index 972687c5..c60bd84c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -87,6 +87,7 @@ "@smithy/signature-v4": "^5.3.13", "@valkey/valkey-glide": "^2.4.0", "express": "^4.21.2", + "google-auth-library": "^11.0.2", "heap-js": "^2.7.1", "iovalkey": "^0.3.3", "ramda": "^0.31.3", @@ -104,6 +105,7 @@ "@valkey/valkey-glide": "^2.4.0", "express": "^4.21.2", "express-rate-limit": "^8.3.1", + "google-auth-library": "^11.0.2", "helmet": "^8.2.0", "p-limit": "^6.1.0", "ramda": "^0.31.3", @@ -5506,7 +5508,6 @@ "version": "7.1.4", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", - "dev": true, "license": "MIT", "engines": { "node": ">= 14" @@ -6078,7 +6079,6 @@ "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", - "dev": true, "funding": [ { "type": "github", @@ -6108,6 +6108,14 @@ "node": ">=6.0.0" } }, + "node_modules/bignumber.js": { + "version": "9.3.1", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.3.1.tgz", + "integrity": "sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==", + "engines": { + "node": "*" + } + }, "node_modules/body-parser": { "version": "1.20.6", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.6.tgz", @@ -6257,6 +6265,11 @@ "node": "*" } }, + "node_modules/buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==" + }, "node_modules/buffer-from": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", @@ -7013,6 +7026,14 @@ "node": ">=12" } }, + "node_modules/data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", + "engines": { + "node": ">= 12" + } + }, "node_modules/data-urls": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-5.0.0.tgz", @@ -7541,6 +7562,14 @@ "node": ">= 0.4" } }, + "node_modules/ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "dependencies": { + "safe-buffer": "^5.0.1" + } + }, "node_modules/ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", @@ -8659,6 +8688,11 @@ "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", "license": "MIT" }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" + }, "node_modules/extract-zip": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz", @@ -8764,6 +8798,28 @@ } } }, + "node_modules/fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "dependencies": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, "node_modules/fflate": { "version": "0.8.2", "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.8.2.tgz", @@ -8985,6 +9041,17 @@ "node": ">= 6" } }, + "node_modules/formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "dependencies": { + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": ">=12.20.0" + } + }, "node_modules/forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -9083,6 +9150,32 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/gaxios": { + "version": "7.3.1", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-7.3.1.tgz", + "integrity": "sha512-kB3rzJV7d9juLZh8/56QTXCwQfxyhdOMdyYk1HdQKFtF8TJTDTZQJtixWIwXdE9Jji91mC41DUNpjleo4L4eAQ==", + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^7.0.1", + "node-fetch": "^3.3.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/gcp-metadata": { + "version": "9.0.3", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-9.0.3.tgz", + "integrity": "sha512-2YYnIlHaKBGT2IPg3G2M57hia9Galz15zsEOvw9T3oRf0lSn6KN6VcHQLqby7x8ksYKnjXvp3rp2KJyLCN6zfQ==", + "dependencies": { + "gaxios": "^7.1.3", + "google-logging-utils": "^2.0.0", + "json-bigint": "^1.0.0" + }, + "engines": { + "node": ">=22" + } + }, "node_modules/generator-function": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/generator-function/-/generator-function-2.0.1.tgz", @@ -9293,6 +9386,30 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/google-auth-library": { + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-11.0.2.tgz", + "integrity": "sha512-vzpgPutxrghPsnjrjpzLX2bdv8IOL719Rh0oEjGnQu8YCIbnbMuTTQ5zU9LcKvLdOPgCxBwppbvnhgW90Qna5Q==", + "dependencies": { + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "gaxios": "^7.1.4", + "gcp-metadata": "^9.0.0", + "google-logging-utils": "^2.0.0", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=22" + } + }, + "node_modules/google-logging-utils": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-2.0.1.tgz", + "integrity": "sha512-HMhaQghlOTvbcb3c4T5jmmOMtG3JUF1iOQMezaJXL86CDS+Tm2vHd0IeLFRAx3+ewd+bo9E1HFHoy17X5aJa9A==", + "engines": { + "node": ">=22" + } + }, "node_modules/gopd": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", @@ -9542,7 +9659,6 @@ "version": "7.0.6", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", - "dev": true, "license": "MIT", "dependencies": { "agent-base": "^7.1.2", @@ -10409,6 +10525,14 @@ "node": ">=6" } }, + "node_modules/json-bigint": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", + "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", + "dependencies": { + "bignumber.js": "^9.0.0" + } + }, "node_modules/json-buffer": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", @@ -10477,6 +10601,25 @@ "node": ">=4.0" } }, + "node_modules/jwa": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", + "integrity": "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==", + "dependencies": { + "buffer-equal-constant-time": "^1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/jws": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.1.tgz", + "integrity": "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==", + "dependencies": { + "jwa": "^2.0.1", + "safe-buffer": "^5.0.1" + } + }, "node_modules/keyv": { "version": "4.5.4", "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", @@ -11268,6 +11411,25 @@ "node": ">=10" } }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "deprecated": "Use your platform's native DOMException instead", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "engines": { + "node": ">=10.5.0" + } + }, "node_modules/node-exports-info": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/node-exports-info/-/node-exports-info-1.6.0.tgz", @@ -11287,6 +11449,23 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, "node_modules/node-gyp": { "version": "12.3.0", "resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-12.3.0.tgz", @@ -15496,6 +15675,14 @@ "node": ">=18" } }, + "node_modules/web-streams-polyfill": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", + "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", + "engines": { + "node": ">= 8" + } + }, "node_modules/whatwg-encoding": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-3.1.1.tgz", From 30904b862e4bf8fd19e578a2ca541aad8fea6079 Mon Sep 17 00:00:00 2001 From: Reza Karamad Date: Sun, 6 Sep 2026 19:15:50 +0200 Subject: [PATCH 2/9] fix: require verified TLS for GCP IAM, preserve node caCertPath, align docs Signed-off-by: Reza Karamad --- .../cluster-topology/cluster-node-row.tsx | 1 + apps/metrics/src/utils/gcp-iam-provider.js | 11 ++++ apps/server/src/__tests__/connection.test.ts | 49 +++++++++++++++++ apps/server/src/connection.ts | 55 ++++++++++--------- apps/server/src/gcp-iam-provider.ts | 13 ++++- apps/server/src/metrics-orchestrator.ts | 2 +- docker/description.md | 8 ++- .../src/content/docs/configuration/server.md | 4 +- .../src/content/docs/features/connections.md | 4 +- 9 files changed, 113 insertions(+), 34 deletions(-) diff --git a/apps/frontend/src/components/cluster-topology/cluster-node-row.tsx b/apps/frontend/src/components/cluster-topology/cluster-node-row.tsx index cf9f2821..ffbd6372 100644 --- a/apps/frontend/src/components/cluster-topology/cluster-node-row.tsx +++ b/apps/frontend/src/components/cluster-topology/cluster-node-row.tsx @@ -90,6 +90,7 @@ export function ClusterNodeRow({ port: port.toString(), tls: primaryConfig.tls, verifyTlsCertificate: primaryConfig.verifyTlsCertificate, + caCertPath: primaryConfig.caCertPath, endpointType: "node", db: clusterDb, } diff --git a/apps/metrics/src/utils/gcp-iam-provider.js b/apps/metrics/src/utils/gcp-iam-provider.js index 6eef4636..4b7b836d 100644 --- a/apps/metrics/src/utils/gcp-iam-provider.js +++ b/apps/metrics/src/utils/gcp-iam-provider.js @@ -13,6 +13,17 @@ class GcpIAMProvider { } async getCredentials() { + // The token is an OAuth2 bearer credential; refuse to mint it for a transport + // that could leak it (plaintext, or TLS without certificate verification). + if (process.env.VALKEY_TLS !== "true") { + throw new Error("GCP IAM authentication requires TLS. Set VALKEY_TLS=true.") + } + if (process.env.VALKEY_VERIFY_CERT === "false") { + throw new Error( + "GCP IAM authentication requires TLS certificate verification. " + + "Do not disable VALKEY_VERIFY_CERT; provide the server CA via VALKEY_CA_CERT_PATH instead.", + ) + } const token = await this.#auth.getAccessToken() if (!token) { throw new Error("Unable to mint a GCP access token from Application Default Credentials") diff --git a/apps/server/src/__tests__/connection.test.ts b/apps/server/src/__tests__/connection.test.ts index dfbb2702..110215a6 100644 --- a/apps/server/src/__tests__/connection.test.ts +++ b/apps/server/src/__tests__/connection.test.ts @@ -362,6 +362,8 @@ describe("connectToValkey", () => { ...DEFAULT_PAYLOAD.connectionDetails, authType: "gcp-iam" as const, username: "ignored@project.iam.gserviceaccount.com", + tls: true, + verifyTlsCertificate: true, password: undefined, }, } @@ -378,6 +380,53 @@ describe("connectToValkey", () => { }) }) + it("rejects gcp-iam without TLS instead of sending the token in cleartext", async () => { + const standalone = buildStandaloneMock({ clusterEnabled: "1" }) + const cluster = buildClusterMock() + mock.method(GoogleAuth.prototype, "getAccessToken", async () => "fake-gcp-token") + + const insecurePayload = { + ...DEFAULT_PAYLOAD, + connectionDetails: { + ...DEFAULT_PAYLOAD.connectionDetails, + authType: "gcp-iam" as const, + tls: false, + password: undefined, + }, + } + + await withMockedClients(standalone, cluster, async () => { + const result = await connectToValkey(ctx(), mockWs, insecurePayload) + assert.strictEqual(result, undefined) + assert.strictEqual((GlideClusterClient.createClient as any).mock.calls.length, 0) + assert.strictEqual((GlideClient.createClient as any).mock.calls.length, 0) + }) + }) + + it("rejects gcp-iam when certificate verification is disabled", async () => { + const standalone = buildStandaloneMock({ clusterEnabled: "1" }) + const cluster = buildClusterMock() + mock.method(GoogleAuth.prototype, "getAccessToken", async () => "fake-gcp-token") + + const insecurePayload = { + ...DEFAULT_PAYLOAD, + connectionDetails: { + ...DEFAULT_PAYLOAD.connectionDetails, + authType: "gcp-iam" as const, + tls: true, + verifyTlsCertificate: false, + password: undefined, + }, + } + + await withMockedClients(standalone, cluster, async () => { + const result = await connectToValkey(ctx(), mockWs, insecurePayload) + assert.strictEqual(result, undefined) + assert.strictEqual((GlideClusterClient.createClient as any).mock.calls.length, 0) + assert.strictEqual((GlideClient.createClient as any).mock.calls.length, 0) + }) + }) + it("passes the connection's caCertPath to Glide as rootCertificates", async () => { const standalone = buildStandaloneMock({ clusterEnabled: "1" }) const cluster = buildClusterMock() diff --git a/apps/server/src/connection.ts b/apps/server/src/connection.ts index 84d8a32a..e582f9cf 100644 --- a/apps/server/src/connection.ts +++ b/apps/server/src/connection.ts @@ -185,23 +185,23 @@ async function connectToValkeyLocked( port: Number(port), }, ] - const credentials: ServerCredentials | undefined = - authType === "iam" - ? { - username: username!, - iamConfig: { - clusterName: awsReplicationGroupId!, - service: ServiceType.Elasticache, - region: awsRegion!, - }, - } - : authType === "gcp-iam" - ? { username: "default", password: await mintGcpAccessToken() } - : password ? { username, password } : undefined - let standaloneClient: GlideClient | undefined try { + const credentials: ServerCredentials | undefined = + authType === "iam" + ? { + username: username!, + iamConfig: { + clusterName: awsReplicationGroupId!, + service: ServiceType.Elasticache, + region: awsRegion!, + }, + } + : authType === "gcp-iam" + ? { username: "default", password: await mintGcpAccessToken(useTLS, verifyTlsCertificate) } + : password ? { username, password } : undefined + if (!isValidDatabaseIndex(db)) { throw new ConnectionRejectedError( "Invalid Database_Index: must be a non-negative integer", @@ -492,22 +492,23 @@ export async function discoverTopology( } = connectionDetails const addresses = [{ host, port: Number(port) }] - const credentials: ServerCredentials | undefined = - authType === "iam" - ? { - username: username!, - iamConfig: { - clusterName: awsReplicationGroupId!, - service: ServiceType.Elasticache, - region: awsRegion!, - }, - } - : authType === "gcp-iam" - ? { username: "default", password: await mintGcpAccessToken() } - : password ? { username, password } : undefined let client: GlideClient | undefined try { + const credentials: ServerCredentials | undefined = + authType === "iam" + ? { + username: username!, + iamConfig: { + clusterName: awsReplicationGroupId!, + service: ServiceType.Elasticache, + region: awsRegion!, + }, + } + : authType === "gcp-iam" + ? { username: "default", password: await mintGcpAccessToken(useTLS, verifyTlsCertificate) } + : password ? { username, password } : undefined + // Cluster discovery is read-only `CLUSTER SLOTS` against the seed node; // database selection is irrelevant for cluster commands. Skip // `databaseId` entirely so Glide does not issue `SELECT` (cluster nodes diff --git a/apps/server/src/gcp-iam-provider.ts b/apps/server/src/gcp-iam-provider.ts index fac70635..a9f638ab 100644 --- a/apps/server/src/gcp-iam-provider.ts +++ b/apps/server/src/gcp-iam-provider.ts @@ -7,7 +7,18 @@ const auth = new GoogleAuth({ scopes: "https://www.googleapis.com/auth/cloud-platform", }) -export async function mintGcpAccessToken(): Promise { +// The token is an OAuth2 bearer credential; refuse to mint it for a transport +// that could leak it (plaintext, or TLS without certificate verification). +export async function mintGcpAccessToken(useTLS: boolean, verifyTlsCertificate: boolean): Promise { + if (!useTLS) { + throw new Error("GCP IAM authentication requires TLS. Set VALKEY_TLS=true.") + } + if (verifyTlsCertificate === false) { + throw new Error( + "GCP IAM authentication requires TLS certificate verification. " + + "Do not disable VALKEY_VERIFY_CERT; provide the server CA via VALKEY_CA_CERT_PATH instead.", + ) + } const token = await auth.getAccessToken() if (!token) { throw new Error("Unable to mint a GCP access token from Application Default Credentials") diff --git a/apps/server/src/metrics-orchestrator.ts b/apps/server/src/metrics-orchestrator.ts index 78da3034..5e4be8c5 100644 --- a/apps/server/src/metrics-orchestrator.ts +++ b/apps/server/src/metrics-orchestrator.ts @@ -182,7 +182,7 @@ async function createClient(connectionDetails: ConnectionDetails) { }, } : authType === "gcp-iam" - ? { username: "default", password: await mintGcpAccessToken() } + ? { username: "default", password: await mintGcpAccessToken(tls, verifyTlsCertificate) } : password ? { username, password } : undefined return await createOrchestratorValkeyClient({ addresses, credentials, useTLS: tls, verifyTlsCertificate, caCertPath, databaseId: db }) diff --git a/docker/description.md b/docker/description.md index 1a665237..2b167f3f 100644 --- a/docker/description.md +++ b/docker/description.md @@ -71,7 +71,9 @@ $ docker run -d --name valkey-admin -p 8080:8080 \ Runs with Application Default Credentials (Workload Identity in GKE, the metadata server on GCE, or a mounted `GOOGLE_APPLICATION_CREDENTIALS` key file locally). -No password is supplied — the token is minted and rotated automatically. +No password is supplied — the token is minted and rotated automatically. TLS with +certificate verification is required for `gcp-iam`, so provide the instance's +server CA via `VALKEY_CA_CERT_PATH`. ```console $ docker run -d --name valkey-admin -p 8080:8080 \ @@ -79,6 +81,8 @@ $ docker run -d --name valkey-admin -p 8080:8080 \ -e VALKEY_PORT=6379 \ -e VALKEY_AUTH_TYPE=gcp-iam \ -e VALKEY_TLS=true \ + -e VALKEY_CA_CERT_PATH=/etc/valkey/tls/valkey-ca.pem \ + -v /path/to/valkey-ca.pem:/etc/valkey/tls/valkey-ca.pem:ro \ valkey/valkey-admin ``` @@ -95,6 +99,8 @@ Here are all the relevant environment variables for configuration | `VALKEY_USERNAME` | Valkey username | — | | `VALKEY_PASSWORD` | Valkey password | — | | `VALKEY_TLS` | Enable TLS | `false` | +| `VALKEY_VERIFY_CERT` | Verify the server's TLS certificate; set `false` to disable (not allowed for `gcp-iam`) | `true` | +| `VALKEY_CA_CERT_PATH` | Path to a PEM CA cert used to verify the server's TLS certificate | — | | `VALKEY_AUTH_TYPE` | Authentication type (`password`, `iam`, `gcp-iam`) | `password` | | `HOT_KEYS_COUNT` | Maximum hot keys returned per query | `50` | | `COMMAND_LOGS_COUNT` | Maximum command log entries returned per query | `100` | diff --git a/docs-site/src/content/docs/configuration/server.md b/docs-site/src/content/docs/configuration/server.md index 97134935..b08b9c3a 100644 --- a/docs-site/src/content/docs/configuration/server.md +++ b/docs-site/src/content/docs/configuration/server.md @@ -111,9 +111,9 @@ Enable TLS for the Valkey connection. Compared as the literal string `"true"`. ### `VALKEY_VERIFY_CERT` -Verify the TLS server certificate. Compared as the literal string `"true"`. Leave this off only when you are knowingly talking to a node with a self-signed cert. +Verify the TLS server certificate. Verification is **on** unless this is set to the literal string `"false"`. Disable it only when you knowingly talk to a node with a self-signed cert and cannot supply its CA via `VALKEY_CA_CERT_PATH`. Verification cannot be disabled for `gcp-iam` — the IAM token is a bearer credential and requires a verified TLS channel. -- **Default:** `false` +- **Default:** `true` ### `VALKEY_CA_CERT_PATH` diff --git a/docs-site/src/content/docs/features/connections.md b/docs-site/src/content/docs/features/connections.md index e8e2b351..29648385 100644 --- a/docs-site/src/content/docs/features/connections.md +++ b/docs-site/src/content/docs/features/connections.md @@ -53,7 +53,7 @@ All Valkey versions support numbered databases in standalone mode, up to the ser ## Authentication -Valkey Admin supports two authentication methods: +Valkey Admin supports three authentication methods: ### Password Authentication @@ -70,7 +70,7 @@ Valkey Admin generates short-lived IAM auth tokens automatically. ### GCP IAM Authentication -For Memorystore for Valkey instances with IAM authentication enabled, select **GCP IAM**. No username or password is required — Valkey Admin mints a short-lived OAuth2 access token from the ambient Application Default Credentials (Workload Identity in GKE, the metadata server on GCE, or `GOOGLE_APPLICATION_CREDENTIALS` locally) and rotates it before expiry, including across every cluster node. Connections authenticate as the `default` user, which is the only username Memorystore for Valkey supports. Enable TLS as well, since Memorystore recommends in-transit encryption whenever IAM authentication is used. +For Memorystore for Valkey instances with IAM authentication enabled, select **GCP IAM**. No username or password is required — Valkey Admin mints a short-lived OAuth2 access token from the ambient Application Default Credentials (Workload Identity in GKE, the metadata server on GCE, or `GOOGLE_APPLICATION_CREDENTIALS` locally) and rotates it before expiry, including across every cluster node. Connections authenticate as the `default` user, which is the only username Memorystore for Valkey supports. TLS **with certificate verification** is required (the token is a bearer credential); supply the instance's server CA via `VALKEY_CA_CERT_PATH`. ## TLS From 795dfac2b0f3e21be71f9281cd697c214f6c4620 Mon Sep 17 00:00:00 2001 From: Reza Karamad Date: Wed, 16 Sep 2026 19:23:23 +0200 Subject: [PATCH 3/9] fix: thread TLS settings into GCP IAM token refresh, guard metrics client Signed-off-by: Reza Karamad --- apps/metrics/src/valkey-client.js | 10 ++++++++++ apps/server/src/connection.ts | 4 ++-- apps/server/src/iam-token-refresh.ts | 9 +++++++-- apps/server/src/metrics-orchestrator.ts | 7 ++++++- 4 files changed, 25 insertions(+), 5 deletions(-) diff --git a/apps/metrics/src/valkey-client.js b/apps/metrics/src/valkey-client.js index f21dcdff..56353191 100644 --- a/apps/metrics/src/valkey-client.js +++ b/apps/metrics/src/valkey-client.js @@ -45,6 +45,16 @@ export const createValkeyClient = async (cfg = {}) => { } : undefined const useTLS = process.env.VALKEY_TLS === "true" + // The GCP IAM token is a bearer credential, so it must never travel over an + // unverified TLS channel. Refuse insecure TLS for gcp-iam regardless of + // VALKEY_VERIFY_CERT (mirrors the guard in GcpIAMProvider.getCredentials()). + const isGcpIam = process.env.VALKEY_AUTH_TYPE === "gcp-iam" + if (isGcpIam && (!useTLS || process.env.VALKEY_VERIFY_CERT === "false")) { + throw new Error( + "GCP IAM authentication requires TLS with certificate verification. " + + "Set VALKEY_TLS=true, do not disable VALKEY_VERIFY_CERT, and provide the server CA via VALKEY_CA_CERT_PATH.", + ) + } // Glide's TLS runs in its Rust core, so a custom CA must be passed explicitly // via `rootCertificates` (Node's trust store / NODE_EXTRA_CA_CERTS do not apply). const caCertPath = process.env.VALKEY_CA_CERT_PATH diff --git a/apps/server/src/connection.ts b/apps/server/src/connection.ts index e582f9cf..14467f2d 100644 --- a/apps/server/src/connection.ts +++ b/apps/server/src/connection.ts @@ -372,7 +372,7 @@ async function connectToValkeyLocked( } shouldCloseClusterClientOnError = false - if (authType === "gcp-iam") registerGcpTokenRefresh(clusterClient, `cluster ${clusterId}`) + if (authType === "gcp-iam") registerGcpTokenRefresh(clusterClient, `cluster ${clusterId}`, useTLS, verifyTlsCertificate) return clusterClient } finally { if (ownInflight && inFlightClusterClients.get(clusterId) === ownInflight) { @@ -438,7 +438,7 @@ async function connectToValkeyLocked( }, }) - if (authType === "gcp-iam") registerGcpTokenRefresh(standaloneClient, connectionId) + if (authType === "gcp-iam") registerGcpTokenRefresh(standaloneClient, connectionId, useTLS, verifyTlsCertificate) return standaloneClient } catch (err) { diff --git a/apps/server/src/iam-token-refresh.ts b/apps/server/src/iam-token-refresh.ts index 61302d68..ab1132cd 100644 --- a/apps/server/src/iam-token-refresh.ts +++ b/apps/server/src/iam-token-refresh.ts @@ -14,12 +14,17 @@ const refreshTimers = new Map() // Keyed by the client instance so shared cluster clients are only scheduled once; the timer // self-clears once the client is closed (updateConnectionPassword throws ClosingError), // so callers do not have to unregister at every close site. -export function registerGcpTokenRefresh(client: RefreshableClient, label: string): void { +export function registerGcpTokenRefresh( + client: RefreshableClient, + label: string, + useTLS: boolean, + verifyTlsCertificate: boolean, +): void { if (refreshTimers.has(client)) return const timer = setInterval(async () => { try { - const token = await mintGcpAccessToken() + const token = await mintGcpAccessToken(useTLS, verifyTlsCertificate) await client.updateConnectionPassword(token, true) } catch (error) { if (error instanceof ClosingError) { diff --git a/apps/server/src/metrics-orchestrator.ts b/apps/server/src/metrics-orchestrator.ts index 5e4be8c5..13bde995 100644 --- a/apps/server/src/metrics-orchestrator.ts +++ b/apps/server/src/metrics-orchestrator.ts @@ -159,7 +159,12 @@ export async function getInitialClient() { if (!initialClient) { initialClient = await createClient(initialConnectionDetails) if (initialConnectionDetails.authType === "gcp-iam") { - registerGcpTokenRefresh(initialClient, "orchestrator") + registerGcpTokenRefresh( + initialClient, + "orchestrator", + initialConnectionDetails.tls, + initialConnectionDetails.verifyTlsCertificate, + ) } } return initialClient From 1f2f957027da7d1289bce3b5dbf50c65b666aa86 Mon Sep 17 00:00:00 2001 From: Reza Karamad Date: Wed, 16 Sep 2026 19:54:04 +0200 Subject: [PATCH 4/9] fix: unregister GCP token refresh before closing replaced cluster client Signed-off-by: Reza Karamad --- apps/server/src/connection.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apps/server/src/connection.ts b/apps/server/src/connection.ts index be537d23..d97adde7 100644 --- a/apps/server/src/connection.ts +++ b/apps/server/src/connection.ts @@ -554,6 +554,9 @@ function updateClusterNodesClient( .filter(([, entry]) => entry.client === existingClusterConnection.client) .map(([id]) => id) + // Stop the GCP IAM token-refresh timer before closing so the stale client is + // released immediately rather than lingering until the next refresh interval. + unregisterGcpTokenRefresh(existingClusterConnection.client) try { existingClusterConnection.client.close() } catch (error) { console.error(`Error closing stale client for ${existingClusterConnection.clusterId}:`, error) } From 04b277ad82f529f1bbe0a3454d8f1156d82a8a5f Mon Sep 17 00:00:00 2001 From: Reza Karamad Date: Wed, 16 Sep 2026 20:42:59 +0200 Subject: [PATCH 5/9] fix: gcp-iam default monitor user, CA read guard, bounded token-refresh retries Signed-off-by: Reza Karamad --- apps/metrics/src/effects/monitor-stream.js | 5 +- apps/metrics/src/index.js | 31 +++++++-- apps/server/src/iam-token-refresh.ts | 66 +++++++++++++++---- apps/server/src/valkey-client.ts | 22 ++++++- .../src/content/docs/configuration/metrics.md | 2 +- 5 files changed, 103 insertions(+), 23 deletions(-) diff --git a/apps/metrics/src/effects/monitor-stream.js b/apps/metrics/src/effects/monitor-stream.js index 08d1748b..8dff5ba8 100644 --- a/apps/metrics/src/effects/monitor-stream.js +++ b/apps/metrics/src/effects/monitor-stream.js @@ -8,7 +8,10 @@ import { GcpIAMProvider } from "../utils/gcp-iam-provider.js" function getConnectionOptions() { const host = process.env.VALKEY_HOST const port = Number(process.env.VALKEY_PORT) - const username = process.env.VALKEY_USERNAME + // GCP IAM authenticates as the fixed "default" user; any other username is rejected. + const username = process.env.VALKEY_AUTH_TYPE === "gcp-iam" + ? "default" + : process.env.VALKEY_USERNAME const verifyTlsCertificate = process.env.VALKEY_VERIFY_CERT let tls = undefined if (process.env.VALKEY_TLS === "true") { diff --git a/apps/metrics/src/index.js b/apps/metrics/src/index.js index 48b94b3b..9aeac19f 100644 --- a/apps/metrics/src/index.js +++ b/apps/metrics/src/index.js @@ -36,14 +36,30 @@ async function main() { // GCP OAuth2 tokens expire ~1h; rotate the connection password before then so // reconnects keep authenticating. AWS IAM refreshes natively inside Glide. - const gcpTokenRefresh = process.env.VALKEY_AUTH_TYPE === "gcp-iam" - ? setInterval(async () => { - try { - await client.updateConnectionPassword(await new GcpIAMProvider().getCredentials(), true) - } catch (err) { - console.error("[gcp-iam] token refresh error:", err.message) + let gcpRetryTimer + const refreshGcpToken = async () => { + try { + await client.updateConnectionPassword(await new GcpIAMProvider().getCredentials(), true) + // Success: drop any retry queued by an earlier failure. + if (gcpRetryTimer) { + clearTimeout(gcpRetryTimer) + gcpRetryTimer = undefined } - }, 45 * 60 * 1000) + } catch (err) { + console.error("[gcp-iam] token refresh error:", err.message) + // Retry sooner than the next interval so a fresh token lands before the + // ~1h token expires and reconnects start failing. + if (!gcpRetryTimer) { + gcpRetryTimer = setTimeout(() => { + gcpRetryTimer = undefined + refreshGcpToken() + }, 5 * 60 * 1000) + gcpRetryTimer.unref?.() + } + } + } + const gcpTokenRefresh = process.env.VALKEY_AUTH_TYPE === "gcp-iam" + ? setInterval(refreshGcpToken, 45 * 60 * 1000) : undefined gcpTokenRefresh?.unref?.() @@ -262,6 +278,7 @@ async function main() { await stopNdjsonCleaner() await stopCollectors() if (gcpTokenRefresh) clearInterval(gcpTokenRefresh) + if (gcpRetryTimer) clearTimeout(gcpRetryTimer) if (client) { client.close() } diff --git a/apps/server/src/iam-token-refresh.ts b/apps/server/src/iam-token-refresh.ts index ab1132cd..ed8cc01d 100644 --- a/apps/server/src/iam-token-refresh.ts +++ b/apps/server/src/iam-token-refresh.ts @@ -5,12 +5,62 @@ import { mintGcpAccessToken } from "./gcp-iam-provider" // authenticated after expiry, but new/reconnecting connections need a fresh // token, so we re-mint and push it to every node connection well before expiry. const REFRESH_INTERVAL_MS = 45 * 60 * 1000 +// A refresh failure only logs and leaves the previous password in place. Since +// the token expires ~1h in, waiting a full interval could leave reconnects using +// an expired token; retry sooner so a fresh token lands well before expiry. +const RETRY_DELAY_MS = 5 * 60 * 1000 type RefreshableClient = GlideClient | GlideClusterClient const refreshTimers = new Map() +const retryTimers = new Map() -// Rotate the connection password for a gcp-iam client on a timer. +function clearRetry(client: RefreshableClient): void { + const retry = retryTimers.get(client) + if (retry) { + clearTimeout(retry) + retryTimers.delete(client) + } +} + +function scheduleRetry( + client: RefreshableClient, + label: string, + useTLS: boolean, + verifyTlsCertificate: boolean, +): void { + // Skip if the client was unregistered or a retry is already pending. + if (!refreshTimers.has(client) || retryTimers.has(client)) return + const retry = setTimeout(() => { + retryTimers.delete(client) + void refreshToken(client, label, useTLS, verifyTlsCertificate) + }, RETRY_DELAY_MS) + retry.unref?.() + retryTimers.set(client, retry) +} + +async function refreshToken( + client: RefreshableClient, + label: string, + useTLS: boolean, + verifyTlsCertificate: boolean, +): Promise { + try { + const token = await mintGcpAccessToken(useTLS, verifyTlsCertificate) + await client.updateConnectionPassword(token, true) + // Success: drop any retry queued by an earlier failure. + clearRetry(client) + } catch (error) { + if (error instanceof ClosingError) { + unregisterGcpTokenRefresh(client) + return + } + console.error(`Error refreshing GCP IAM token for ${label}:`, error) + scheduleRetry(client, label, useTLS, verifyTlsCertificate) + } +} + +// Rotate the connection password for a gcp-iam client on a timer. // Keyed by the client instance so shared cluster clients are only scheduled once; the timer // self-clears once the client is closed (updateConnectionPassword throws ClosingError), // so callers do not have to unregister at every close site. @@ -22,17 +72,8 @@ export function registerGcpTokenRefresh( ): void { if (refreshTimers.has(client)) return - const timer = setInterval(async () => { - try { - const token = await mintGcpAccessToken(useTLS, verifyTlsCertificate) - await client.updateConnectionPassword(token, true) - } catch (error) { - if (error instanceof ClosingError) { - unregisterGcpTokenRefresh(client) - return - } - console.error(`Error refreshing GCP IAM token for ${label}:`, error) - } + const timer = setInterval(() => { + void refreshToken(client, label, useTLS, verifyTlsCertificate) }, REFRESH_INTERVAL_MS) // Do not keep the process alive solely for the refresh timer. @@ -46,4 +87,5 @@ export function unregisterGcpTokenRefresh(client: RefreshableClient): void { clearInterval(timer) refreshTimers.delete(client) } + clearRetry(client) } diff --git a/apps/server/src/valkey-client.ts b/apps/server/src/valkey-client.ts index 37166f01..2fa693d1 100644 --- a/apps/server/src/valkey-client.ts +++ b/apps/server/src/valkey-client.ts @@ -1,5 +1,5 @@ import { GlideClient, GlideClusterClient, NodeDiscoveryMode, type ServerCredentials } from "@valkey/valkey-glide" -import { readFileSync } from "node:fs" +import { readFileSync, statSync } from "node:fs" import { APP_VERSION, deploymentSuffix } from "valkey-common" type Address = { @@ -18,6 +18,24 @@ type ClientOptions = { const clientInfoTag = `valkey-admin-${deploymentSuffix()}:${APP_VERSION}` +// A connection's `caCertPath` can originate from a client-supplied connection +// request, so guard the synchronous read: reject non-regular files (FIFOs or +// devices that would block the event loop) and oversized files before reading. +const MAX_CA_CERT_BYTES = 1024 * 1024 + +const readCaCertificate = (caCertPath: string): Buffer => { + const stats = statSync(caCertPath) + if (!stats.isFile()) { + throw new Error(`CA certificate path is not a regular file: ${caCertPath}`) + } + if (stats.size > MAX_CA_CERT_BYTES) { + throw new Error( + `CA certificate file exceeds ${MAX_CA_CERT_BYTES} bytes (${stats.size}): ${caCertPath}`, + ) + } + return readFileSync(caCertPath) +} + const buildSharedOptions = ({ addresses, credentials, @@ -44,7 +62,7 @@ const buildSharedOptions = ({ : verifyTlsCertificate === false ? { insecure: true } : caCertPath - ? { rootCertificates: readFileSync(caCertPath) } + ? { rootCertificates: readCaCertificate(caCertPath) } : undefined return { diff --git a/docs-site/src/content/docs/configuration/metrics.md b/docs-site/src/content/docs/configuration/metrics.md index b1dcce43..ba7797da 100644 --- a/docs-site/src/content/docs/configuration/metrics.md +++ b/docs-site/src/content/docs/configuration/metrics.md @@ -108,7 +108,7 @@ Filesystem path to a PEM CA certificate used to verify the Valkey server's TLS c Selects the credentials provider. - **`"iam"`** — use AWS ElastiCache IAM authentication via `ElastiCacheIAMProvider`. Requires `VALKEY_USERNAME`, `VALKEY_AWS_REGION`, and `VALKEY_REPLICATION_GROUP_ID`. -- **`"gcp-iam"`** — use GCP Memorystore for Valkey IAM authentication via `GcpIAMProvider`. Mints a short-lived OAuth2 access token from Application Default Credentials and rotates it before expiry. Authenticates as the `default` user — the only username Memorystore supports — so `VALKEY_USERNAME` is ignored. +- **`"gcp-iam"`** — use GCP Memorystore for Valkey IAM authentication via `GcpIAMProvider`. Mints a short-lived OAuth2 access token from Application Default Credentials and rotates it before expiry. Authenticates as the `default` user — the only username Memorystore supports — so `VALKEY_USERNAME` is ignored. Requires TLS with certificate verification: `VALKEY_TLS=true` and `VALKEY_VERIFY_CERT` must not be `"false"` (the general verification opt-out does not apply to `gcp-iam`). Startup fails otherwise, since the IAM token is a bearer credential that must not travel over an unverified channel. - **anything else** — password authentication using `VALKEY_USERNAME` / `VALKEY_PASSWORD`. ### `VALKEY_AWS_REGION` From 123e07ec0c1058b8d0f22ba8a600529cfed90e24 Mon Sep 17 00:00:00 2001 From: Reza Karamad Date: Wed, 16 Sep 2026 21:12:54 +0200 Subject: [PATCH 6/9] fix: cap GCP IAM token-refresh to one retry per interval Signed-off-by: Reza Karamad --- apps/metrics/src/index.js | 13 +++++++++---- apps/server/src/iam-token-refresh.ts | 11 +++++++++-- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/apps/metrics/src/index.js b/apps/metrics/src/index.js index 9aeac19f..ddc24889 100644 --- a/apps/metrics/src/index.js +++ b/apps/metrics/src/index.js @@ -37,6 +37,7 @@ async function main() { // GCP OAuth2 tokens expire ~1h; rotate the connection password before then so // reconnects keep authenticating. AWS IAM refreshes natively inside Glide. let gcpRetryTimer + let gcpRetryUsedThisInterval = false const refreshGcpToken = async () => { try { await client.updateConnectionPassword(await new GcpIAMProvider().getCredentials(), true) @@ -47,9 +48,10 @@ async function main() { } } catch (err) { console.error("[gcp-iam] token refresh error:", err.message) - // Retry sooner than the next interval so a fresh token lands before the - // ~1h token expires and reconnects start failing. - if (!gcpRetryTimer) { + // At most one retry per interval so persistent ADC failures don't spin; + // it lands a fresh token before the ~1h token expires and reconnects fail. + if (!gcpRetryUsedThisInterval && !gcpRetryTimer) { + gcpRetryUsedThisInterval = true gcpRetryTimer = setTimeout(() => { gcpRetryTimer = undefined refreshGcpToken() @@ -59,7 +61,10 @@ async function main() { } } const gcpTokenRefresh = process.env.VALKEY_AUTH_TYPE === "gcp-iam" - ? setInterval(refreshGcpToken, 45 * 60 * 1000) + ? setInterval(() => { + gcpRetryUsedThisInterval = false + refreshGcpToken() + }, 45 * 60 * 1000) : undefined gcpTokenRefresh?.unref?.() diff --git a/apps/server/src/iam-token-refresh.ts b/apps/server/src/iam-token-refresh.ts index ed8cc01d..499ba719 100644 --- a/apps/server/src/iam-token-refresh.ts +++ b/apps/server/src/iam-token-refresh.ts @@ -14,6 +14,8 @@ type RefreshableClient = GlideClient | GlideClusterClient const refreshTimers = new Map() const retryTimers = new Map() +// Clients whose current interval has already consumed its single retry. +const retryUsed = new Set() function clearRetry(client: RefreshableClient): void { const retry = retryTimers.get(client) @@ -29,8 +31,10 @@ function scheduleRetry( useTLS: boolean, verifyTlsCertificate: boolean, ): void { - // Skip if the client was unregistered or a retry is already pending. - if (!refreshTimers.has(client) || retryTimers.has(client)) return + // At most one retry per interval: skip if unregistered, already retried this + // interval, or a retry is already pending. + if (!refreshTimers.has(client) || retryUsed.has(client) || retryTimers.has(client)) return + retryUsed.add(client) const retry = setTimeout(() => { retryTimers.delete(client) void refreshToken(client, label, useTLS, verifyTlsCertificate) @@ -73,6 +77,8 @@ export function registerGcpTokenRefresh( if (refreshTimers.has(client)) return const timer = setInterval(() => { + // Each interval gets a fresh retry budget. + retryUsed.delete(client) void refreshToken(client, label, useTLS, verifyTlsCertificate) }, REFRESH_INTERVAL_MS) @@ -88,4 +94,5 @@ export function unregisterGcpTokenRefresh(client: RefreshableClient): void { refreshTimers.delete(client) } clearRetry(client) + retryUsed.delete(client) } From 4f59feee0d4ed1e63271f7b8198a3f5c0cb1fb68 Mon Sep 17 00:00:00 2001 From: Reza Karamad Date: Thu, 17 Sep 2026 07:56:51 +0200 Subject: [PATCH 7/9] refactor: consolidate GCP IAM provider and token refresh into common Signed-off-by: Reza Karamad --- apps/metrics/src/effects/monitor-stream.js | 7 +- apps/metrics/src/index.js | 42 ++------ apps/metrics/src/utils/gcp-iam-provider.js | 35 ------- apps/metrics/src/valkey-client.js | 19 +--- apps/server/src/connection.ts | 3 +- apps/server/src/iam-token-refresh.ts | 98 ------------------- apps/server/src/metrics-orchestrator.ts | 6 +- common/package.json | 3 + .../server => common}/src/gcp-iam-provider.ts | 0 common/src/iam-token-refresh.ts | 85 ++++++++++++++++ common/src/index.ts | 2 + .../src/content/docs/configuration/metrics.md | 2 +- package-lock.json | 15 +-- 13 files changed, 117 insertions(+), 200 deletions(-) delete mode 100644 apps/metrics/src/utils/gcp-iam-provider.js delete mode 100644 apps/server/src/iam-token-refresh.ts rename {apps/server => common}/src/gcp-iam-provider.ts (100%) create mode 100644 common/src/iam-token-refresh.ts diff --git a/apps/metrics/src/effects/monitor-stream.js b/apps/metrics/src/effects/monitor-stream.js index 8dff5ba8..e317398c 100644 --- a/apps/metrics/src/effects/monitor-stream.js +++ b/apps/metrics/src/effects/monitor-stream.js @@ -3,7 +3,7 @@ import { exhaustMap, catchError, map } from "rxjs" import Valkey from "iovalkey" import { readFileSync } from "node:fs" import { ElastiCacheIAMProvider } from "../utils/elasticache-iam-provider.js" -import { GcpIAMProvider } from "../utils/gcp-iam-provider.js" +import { mintGcpAccessToken } from "valkey-common" function getConnectionOptions() { const host = process.env.VALKEY_HOST @@ -32,7 +32,10 @@ async function getPassword() { return await new ElastiCacheIAMProvider(username, process.env.VALKEY_REPLICATION_GROUP_ID, process.env.VALKEY_AWS_REGION).getCredentials() } if (process.env.VALKEY_AUTH_TYPE === "gcp-iam") { - return await new GcpIAMProvider().getCredentials() + return await mintGcpAccessToken( + process.env.VALKEY_TLS === "true", + process.env.VALKEY_VERIFY_CERT !== "false", + ) } return process.env.VALKEY_PASSWORD } diff --git a/apps/metrics/src/index.js b/apps/metrics/src/index.js index ddc24889..26a49e14 100644 --- a/apps/metrics/src/index.js +++ b/apps/metrics/src/index.js @@ -17,7 +17,7 @@ import { sanitizeUrl } from "./utils/helpers.js" import { buildPingRequest, buildRegisterRequest, readOrchestratorKey } from "./utils/orchestrator-auth.js" import { setupNdjsonCleaner, stopNdjsonCleaner } from "./effects/ndjson-cleaner.js" import { createValkeyClient } from "./valkey-client.js" -import { GcpIAMProvider } from "./utils/gcp-iam-provider.js" +import { registerGcpTokenRefresh, unregisterGcpTokenRefresh } from "valkey-common" import { scanBigKeys } from "./analyzers/scan-big-keys.js" async function main() { @@ -36,37 +36,14 @@ async function main() { // GCP OAuth2 tokens expire ~1h; rotate the connection password before then so // reconnects keep authenticating. AWS IAM refreshes natively inside Glide. - let gcpRetryTimer - let gcpRetryUsedThisInterval = false - const refreshGcpToken = async () => { - try { - await client.updateConnectionPassword(await new GcpIAMProvider().getCredentials(), true) - // Success: drop any retry queued by an earlier failure. - if (gcpRetryTimer) { - clearTimeout(gcpRetryTimer) - gcpRetryTimer = undefined - } - } catch (err) { - console.error("[gcp-iam] token refresh error:", err.message) - // At most one retry per interval so persistent ADC failures don't spin; - // it lands a fresh token before the ~1h token expires and reconnects fail. - if (!gcpRetryUsedThisInterval && !gcpRetryTimer) { - gcpRetryUsedThisInterval = true - gcpRetryTimer = setTimeout(() => { - gcpRetryTimer = undefined - refreshGcpToken() - }, 5 * 60 * 1000) - gcpRetryTimer.unref?.() - } - } + if (process.env.VALKEY_AUTH_TYPE === "gcp-iam") { + registerGcpTokenRefresh( + client, + "metrics", + process.env.VALKEY_TLS === "true", + process.env.VALKEY_VERIFY_CERT !== "false", + ) } - const gcpTokenRefresh = process.env.VALKEY_AUTH_TYPE === "gcp-iam" - ? setInterval(() => { - gcpRetryUsedThisInterval = false - refreshGcpToken() - }, 45 * 60 * 1000) - : undefined - gcpTokenRefresh?.unref?.() await setupNdjsonCleaner(cfg) await setupCollectors(client, cfg) @@ -282,8 +259,7 @@ async function main() { try { await stopNdjsonCleaner() await stopCollectors() - if (gcpTokenRefresh) clearInterval(gcpTokenRefresh) - if (gcpRetryTimer) clearTimeout(gcpRetryTimer) + unregisterGcpTokenRefresh(client) if (client) { client.close() } diff --git a/apps/metrics/src/utils/gcp-iam-provider.js b/apps/metrics/src/utils/gcp-iam-provider.js deleted file mode 100644 index 4b7b836d..00000000 --- a/apps/metrics/src/utils/gcp-iam-provider.js +++ /dev/null @@ -1,35 +0,0 @@ -import { GoogleAuth } from "google-auth-library" - -// GCP Memorystore for Valkey IAM auth uses a short-lived OAuth2 access token as the AUTH password. -// We mint the token from Application Default Credentials (Workload Identity in GKE, -// the metadata server on Google Compute Engine, or GOOGLE_APPLICATION_CREDENTIALS locally). -class GcpIAMProvider { - #auth - - constructor() { - this.#auth = new GoogleAuth({ - scopes: "https://www.googleapis.com/auth/cloud-platform", - }) - } - - async getCredentials() { - // The token is an OAuth2 bearer credential; refuse to mint it for a transport - // that could leak it (plaintext, or TLS without certificate verification). - if (process.env.VALKEY_TLS !== "true") { - throw new Error("GCP IAM authentication requires TLS. Set VALKEY_TLS=true.") - } - if (process.env.VALKEY_VERIFY_CERT === "false") { - throw new Error( - "GCP IAM authentication requires TLS certificate verification. " - + "Do not disable VALKEY_VERIFY_CERT; provide the server CA via VALKEY_CA_CERT_PATH instead.", - ) - } - const token = await this.#auth.getAccessToken() - if (!token) { - throw new Error("Unable to mint a GCP access token from Application Default Credentials") - } - return token - } -} - -export { GcpIAMProvider } diff --git a/apps/metrics/src/valkey-client.js b/apps/metrics/src/valkey-client.js index 9fd56598..2a7a8697 100644 --- a/apps/metrics/src/valkey-client.js +++ b/apps/metrics/src/valkey-client.js @@ -1,7 +1,6 @@ import { GlideClient, GlideClusterClient, ServiceType, NodeDiscoveryMode } from "@valkey/valkey-glide" import { readFileSync } from "node:fs" -import { GcpIAMProvider } from "./utils/gcp-iam-provider.js" -import { APP_VERSION ,deploymentSuffix } from "valkey-common" +import { APP_VERSION ,deploymentSuffix, mintGcpAccessToken } from "valkey-common" const clientInfoTag = `valkey-admin-metrics-${deploymentSuffix()}:${APP_VERSION}` @@ -25,6 +24,8 @@ export const createValkeyClient = async (cfg = {}) => { port: Number(process.env.VALKEY_PORT), }, ] + const useTLS = process.env.VALKEY_TLS === "true" + const verifyTlsCertificate = process.env.VALKEY_VERIFY_CERT !== "false" const credentials = process.env.VALKEY_AUTH_TYPE === "iam" ? { @@ -39,25 +40,15 @@ export const createValkeyClient = async (cfg = {}) => { ? { // "default" is the only supported username for GCP IAM authentication // https://docs.cloud.google.com/memorystore/docs/valkey/manage-iam-auth#error-messages + // mintGcpAccessToken rejects non-TLS / unverified transports for this bearer token. username: "default", - password: await new GcpIAMProvider().getCredentials(), + password: await mintGcpAccessToken(useTLS, verifyTlsCertificate), } : process.env.VALKEY_PASSWORD ? { username: process.env.VALKEY_USERNAME, password: process.env.VALKEY_PASSWORD, } : undefined - const useTLS = process.env.VALKEY_TLS === "true" - // The GCP IAM token is a bearer credential, so it must never travel over an - // unverified TLS channel. Refuse insecure TLS for gcp-iam regardless of - // VALKEY_VERIFY_CERT (mirrors the guard in GcpIAMProvider.getCredentials()). - const isGcpIam = process.env.VALKEY_AUTH_TYPE === "gcp-iam" - if (isGcpIam && (!useTLS || process.env.VALKEY_VERIFY_CERT === "false")) { - throw new Error( - "GCP IAM authentication requires TLS with certificate verification. " - + "Set VALKEY_TLS=true, do not disable VALKEY_VERIFY_CERT, and provide the server CA via VALKEY_CA_CERT_PATH.", - ) - } // Glide's TLS runs in its Rust core, so a custom CA must be passed explicitly // via `rootCertificates` (Node's trust store / NODE_EXTRA_CA_CERTS do not apply). const caCertPath = process.env.VALKEY_CA_CERT_PATH diff --git a/apps/server/src/connection.ts b/apps/server/src/connection.ts index d97adde7..68cb3fcf 100644 --- a/apps/server/src/connection.ts +++ b/apps/server/src/connection.ts @@ -3,6 +3,7 @@ import * as R from "ramda" import WebSocket from "ws" import { VALKEY } from "valkey-common" import { buildConnectionId, isValidDatabaseIndex, sanitizeUrl, toNodeId, buildUrl } from "valkey-common" +import { mintGcpAccessToken, registerGcpTokenRefresh, unregisterGcpTokenRefresh } from "valkey-common" import { KeyEvictionPolicy } from "common/dist" import { getExistingClusterClient, @@ -27,8 +28,6 @@ import { subscribe } from "./node-watchers" import { clearCpuSamples } from "./node-utilization" import { createClusterValkeyClient, createStandaloneValkeyClient } from "./valkey-client" import { isConnectionAuthorized } from "./session" -import { mintGcpAccessToken } from "./gcp-iam-provider" -import { registerGcpTokenRefresh, unregisterGcpTokenRefresh } from "./iam-token-refresh" export type ConnectionContext = { clients: Map diff --git a/apps/server/src/iam-token-refresh.ts b/apps/server/src/iam-token-refresh.ts deleted file mode 100644 index 499ba719..00000000 --- a/apps/server/src/iam-token-refresh.ts +++ /dev/null @@ -1,98 +0,0 @@ -import { GlideClient, GlideClusterClient, ClosingError } from "@valkey/valkey-glide" -import { mintGcpAccessToken } from "./gcp-iam-provider" - -// GCP OAuth2 access tokens live ~1 hour. Glide keeps existing connections -// authenticated after expiry, but new/reconnecting connections need a fresh -// token, so we re-mint and push it to every node connection well before expiry. -const REFRESH_INTERVAL_MS = 45 * 60 * 1000 -// A refresh failure only logs and leaves the previous password in place. Since -// the token expires ~1h in, waiting a full interval could leave reconnects using -// an expired token; retry sooner so a fresh token lands well before expiry. -const RETRY_DELAY_MS = 5 * 60 * 1000 - -type RefreshableClient = GlideClient | GlideClusterClient - -const refreshTimers = new Map() -const retryTimers = new Map() -// Clients whose current interval has already consumed its single retry. -const retryUsed = new Set() - -function clearRetry(client: RefreshableClient): void { - const retry = retryTimers.get(client) - if (retry) { - clearTimeout(retry) - retryTimers.delete(client) - } -} - -function scheduleRetry( - client: RefreshableClient, - label: string, - useTLS: boolean, - verifyTlsCertificate: boolean, -): void { - // At most one retry per interval: skip if unregistered, already retried this - // interval, or a retry is already pending. - if (!refreshTimers.has(client) || retryUsed.has(client) || retryTimers.has(client)) return - retryUsed.add(client) - const retry = setTimeout(() => { - retryTimers.delete(client) - void refreshToken(client, label, useTLS, verifyTlsCertificate) - }, RETRY_DELAY_MS) - retry.unref?.() - retryTimers.set(client, retry) -} - -async function refreshToken( - client: RefreshableClient, - label: string, - useTLS: boolean, - verifyTlsCertificate: boolean, -): Promise { - try { - const token = await mintGcpAccessToken(useTLS, verifyTlsCertificate) - await client.updateConnectionPassword(token, true) - // Success: drop any retry queued by an earlier failure. - clearRetry(client) - } catch (error) { - if (error instanceof ClosingError) { - unregisterGcpTokenRefresh(client) - return - } - console.error(`Error refreshing GCP IAM token for ${label}:`, error) - scheduleRetry(client, label, useTLS, verifyTlsCertificate) - } -} - -// Rotate the connection password for a gcp-iam client on a timer. -// Keyed by the client instance so shared cluster clients are only scheduled once; the timer -// self-clears once the client is closed (updateConnectionPassword throws ClosingError), -// so callers do not have to unregister at every close site. -export function registerGcpTokenRefresh( - client: RefreshableClient, - label: string, - useTLS: boolean, - verifyTlsCertificate: boolean, -): void { - if (refreshTimers.has(client)) return - - const timer = setInterval(() => { - // Each interval gets a fresh retry budget. - retryUsed.delete(client) - void refreshToken(client, label, useTLS, verifyTlsCertificate) - }, REFRESH_INTERVAL_MS) - - // Do not keep the process alive solely for the refresh timer. - timer.unref?.() - refreshTimers.set(client, timer) -} - -export function unregisterGcpTokenRefresh(client: RefreshableClient): void { - const timer = refreshTimers.get(client) - if (timer) { - clearInterval(timer) - refreshTimers.delete(client) - } - clearRetry(client) - retryUsed.delete(client) -} diff --git a/apps/server/src/metrics-orchestrator.ts b/apps/server/src/metrics-orchestrator.ts index fc89c47f..f43ae54c 100644 --- a/apps/server/src/metrics-orchestrator.ts +++ b/apps/server/src/metrics-orchestrator.ts @@ -28,13 +28,13 @@ import { resolveOrchestratorAuthWindowMs, sanitizeUrl, toNodeId, - verifyOrchestratorAuthCredential + verifyOrchestratorAuthCredential, + mintGcpAccessToken, + registerGcpTokenRefresh } from "valkey-common" import { discoverCluster, belongsToCluster } from "./connection" import { ConnectionDetails } from "./actions/connection" import { createOrchestratorValkeyClient } from "./valkey-client" -import { mintGcpAccessToken } from "./gcp-iam-provider" -import { registerGcpTokenRefresh } from "./iam-token-refresh" // Assumes nodeId is unique among all clusters export type MetricsServerMap = Map +} + +// GCP OAuth2 access tokens live ~1 hour. Glide keeps existing connections +// authenticated after expiry, but new/reconnecting connections need a fresh +// token, so we re-mint and push it to every node connection well before expiry. +const REFRESH_INTERVAL_MS = 45 * 60 * 1000 + +interface RefreshState { + label: string + useTLS: boolean + verifyTlsCertificate: boolean + timer?: NodeJS.Timeout // the next scheduled refresh (regular cadence or backoff retry) + failures: number // 0 while healthy; drives exponential backoff while failing +} + +const refreshStates = new Map() + +// Glide throws ClosingError once the client is closed; its `name` getter returns +// the constructor name, so we detect it without importing @valkey/valkey-glide. +function isClosingError(error: unknown): boolean { + return error instanceof Error && error.name === "ClosingError" +} + +function scheduleNext(client: RefreshableClient, delayMs: number): void { + const state = refreshStates.get(client) + if (!state) return + if (state.timer) clearTimeout(state.timer) + const timer = setTimeout(() => { + void refresh(client) + }, delayMs) + // Do not keep the process alive solely for the refresh timer. + timer.unref?.() + state.timer = timer +} + +async function refresh(client: RefreshableClient): Promise { + const state = refreshStates.get(client) + if (!state) return + try { + const token = await mintGcpAccessToken(state.useTLS, state.verifyTlsCertificate) + await client.updateConnectionPassword(token, true) + state.failures = 0 + scheduleNext(client, REFRESH_INTERVAL_MS) + } catch (error) { + if (isClosingError(error)) { + unregisterGcpTokenRefresh(client) + return + } + state.failures += 1 + console.error(`Error refreshing GCP IAM token for ${state.label}:`, error) + // Back off (fibonacci) but never wait longer than the regular interval, so a + // fresh token still lands before the ~1h token expires. A later success + // resets `failures` and restores the regular cadence. + scheduleNext(client, Math.min(retryDelay(state.failures), REFRESH_INTERVAL_MS)) + } +} + +// Rotate the connection password for a gcp-iam client on a timer. +// Keyed by the client instance so shared cluster clients are only scheduled once; the timer +// self-clears once the client is closed (updateConnectionPassword throws ClosingError), +// so callers do not have to unregister at every close site. +export function registerGcpTokenRefresh( + client: RefreshableClient, + label: string, + useTLS: boolean, + verifyTlsCertificate: boolean, +): void { + if (refreshStates.has(client)) return + refreshStates.set(client, { label, useTLS, verifyTlsCertificate, failures: 0 }) + scheduleNext(client, REFRESH_INTERVAL_MS) +} + +export function unregisterGcpTokenRefresh(client: RefreshableClient): void { + const state = refreshStates.get(client) + if (state?.timer) clearTimeout(state.timer) + refreshStates.delete(client) +} diff --git a/common/src/index.ts b/common/src/index.ts index 35888da3..3da0ec0d 100644 --- a/common/src/index.ts +++ b/common/src/index.ts @@ -7,6 +7,8 @@ export * from "./orchestrator-auth" export * from "./constants" export * from "./dashboard-metrics" export * from "./format-metric-value" +export * from "./gcp-iam-provider" +export * from "./iam-token-refresh" export * from "./json-utils" export * from "./key-tree-builder" export * from "./key-validators" diff --git a/docs-site/src/content/docs/configuration/metrics.md b/docs-site/src/content/docs/configuration/metrics.md index ba7797da..4b7292cf 100644 --- a/docs-site/src/content/docs/configuration/metrics.md +++ b/docs-site/src/content/docs/configuration/metrics.md @@ -108,7 +108,7 @@ Filesystem path to a PEM CA certificate used to verify the Valkey server's TLS c Selects the credentials provider. - **`"iam"`** — use AWS ElastiCache IAM authentication via `ElastiCacheIAMProvider`. Requires `VALKEY_USERNAME`, `VALKEY_AWS_REGION`, and `VALKEY_REPLICATION_GROUP_ID`. -- **`"gcp-iam"`** — use GCP Memorystore for Valkey IAM authentication via `GcpIAMProvider`. Mints a short-lived OAuth2 access token from Application Default Credentials and rotates it before expiry. Authenticates as the `default` user — the only username Memorystore supports — so `VALKEY_USERNAME` is ignored. Requires TLS with certificate verification: `VALKEY_TLS=true` and `VALKEY_VERIFY_CERT` must not be `"false"` (the general verification opt-out does not apply to `gcp-iam`). Startup fails otherwise, since the IAM token is a bearer credential that must not travel over an unverified channel. +- **`"gcp-iam"`** — use GCP Memorystore for Valkey IAM authentication. Mints a short-lived OAuth2 access token from Application Default Credentials and rotates it before expiry. Authenticates as the `default` user — the only username Memorystore supports — so `VALKEY_USERNAME` is ignored. Requires TLS with certificate verification: `VALKEY_TLS=true` and `VALKEY_VERIFY_CERT` must not be `"false"` (the general verification opt-out does not apply to `gcp-iam`). Startup fails otherwise, since the IAM token is a bearer credential that must not travel over an unverified channel. - **anything else** — password authentication using `VALKEY_USERNAME` / `VALKEY_PASSWORD`. ### `VALKEY_AWS_REGION` diff --git a/package-lock.json b/package-lock.json index dac2d56a..103ec9f1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -126,6 +126,9 @@ "common": { "name": "valkey-common", "version": "1.0.0", + "dependencies": { + "google-auth-library": "^11.0.2" + }, "devDependencies": { "@types/node": "^24.2.1", "tsup": "^8.5.1", @@ -5196,9 +5199,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -5212,9 +5212,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -5228,9 +5225,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -5244,9 +5238,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "Apache-2.0", "optional": true, "os": [ From e5a0c8489ded7ef7ff913ebfc2f1a8844a9b9019 Mon Sep 17 00:00:00 2001 From: Reza Karamad Date: Fri, 18 Sep 2026 15:44:39 +0200 Subject: [PATCH 8/9] chore: drop unused google-auth-library dep from metrics Signed-off-by: Reza Karamad --- apps/metrics/package.json | 1 - package-lock.json | 1 - 2 files changed, 2 deletions(-) diff --git a/apps/metrics/package.json b/apps/metrics/package.json index a7035570..8ed20598 100644 --- a/apps/metrics/package.json +++ b/apps/metrics/package.json @@ -17,7 +17,6 @@ "@smithy/signature-v4": "^5.3.13", "@valkey/valkey-glide": "^2.5.2", "express": "^4.21.2", - "google-auth-library": "^11.0.2", "heap-js": "^2.7.1", "iovalkey": "^0.3.3", "ramda": "^0.31.3", diff --git a/package-lock.json b/package-lock.json index 103ec9f1..93dc3b67 100644 --- a/package-lock.json +++ b/package-lock.json @@ -87,7 +87,6 @@ "@smithy/signature-v4": "^5.3.13", "@valkey/valkey-glide": "^2.5.2", "express": "^4.21.2", - "google-auth-library": "^11.0.2", "heap-js": "^2.7.1", "iovalkey": "^0.3.3", "ramda": "^0.31.3", From bd27b7907cc4e5df65b4f2145db658ddb529f865 Mon Sep 17 00:00:00 2001 From: Reza Karamad Date: Fri, 18 Sep 2026 16:04:00 +0200 Subject: [PATCH 9/9] style: fix import ordering in metrics (eslint) Signed-off-by: Reza Karamad --- apps/metrics/src/effects/monitor-stream.js | 2 +- apps/metrics/src/index.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/metrics/src/effects/monitor-stream.js b/apps/metrics/src/effects/monitor-stream.js index e317398c..8cc1f931 100644 --- a/apps/metrics/src/effects/monitor-stream.js +++ b/apps/metrics/src/effects/monitor-stream.js @@ -2,8 +2,8 @@ import { Subject, timer, race, firstValueFrom, defer, of } from "rxjs" import { exhaustMap, catchError, map } from "rxjs" import Valkey from "iovalkey" import { readFileSync } from "node:fs" -import { ElastiCacheIAMProvider } from "../utils/elasticache-iam-provider.js" import { mintGcpAccessToken } from "valkey-common" +import { ElastiCacheIAMProvider } from "../utils/elasticache-iam-provider.js" function getConnectionOptions() { const host = process.env.VALKEY_HOST diff --git a/apps/metrics/src/index.js b/apps/metrics/src/index.js index 26a49e14..2ba02aac 100644 --- a/apps/metrics/src/index.js +++ b/apps/metrics/src/index.js @@ -1,6 +1,7 @@ import fs from "node:fs" import express from "express" import { ORCHESTRATOR_AUTH_KEY_ENV, buildUrl } from "valkey-common" +import { registerGcpTokenRefresh, unregisterGcpTokenRefresh } from "valkey-common" import { getConfig } from "./config.js" import * as Streamer from "./effects/ndjson-streamer.js" import { setupCollectors, stopCollectors } from "./init-collectors.js" @@ -17,7 +18,6 @@ import { sanitizeUrl } from "./utils/helpers.js" import { buildPingRequest, buildRegisterRequest, readOrchestratorKey } from "./utils/orchestrator-auth.js" import { setupNdjsonCleaner, stopNdjsonCleaner } from "./effects/ndjson-cleaner.js" import { createValkeyClient } from "./valkey-client.js" -import { registerGcpTokenRefresh, unregisterGcpTokenRefresh } from "valkey-common" import { scanBigKeys } from "./analyzers/scan-big-keys.js" async function main() {