Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 44 additions & 0 deletions .github/workflows/bench.yml
Original file line number Diff line number Diff line change
Expand Up @@ -62,3 +62,47 @@ jobs:
name: tui-performance
path: ${{ runner.temp }}/tui-performance.json
if-no-files-found: warn

libfx-ttft:
name: libfx TTFT (report only)
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4

- name: Setup Zig
uses: mlugg/setup-zig@v2
with:
version: "0.16.0"

- name: Setup Node
uses: actions/setup-node@v4
with:
node-version: "24"

- name: Setup Bun
uses: oven-sh/setup-bun@v2
with:
bun-version: "1.3.14"

- name: Build libfx artifacts
run: |
zig build -Dnapi-surface=core -Doptimize=ReleaseSafe
zig build -Dwasm-surface=core -Doptimize=ReleaseSmall

- name: Install pinned Pi benchmark target
run: npm install --prefix /tmp/libfx-pi @earendil-works/pi-coding-agent@0.84.4

- name: Measure libfx and Pi
run: |
node benchmarks/libfx/bench-suite.mjs --samples 20 --pi-root /tmp/libfx-pi --out benchmarks/results/libfx
node benchmarks/libfx/bench-bridge.mjs --backend native --samples 20 > benchmarks/results/libfx/bridge-node-native.json
node --experimental-wasm-jspi benchmarks/libfx/bench-bridge.mjs --backend wasm --samples 20 > benchmarks/results/libfx/bridge-node-wasm.json
bun benchmarks/libfx/bench-bridge.mjs --backend native --samples 20 > benchmarks/results/libfx/bridge-bun-native.json
bun benchmarks/libfx/bench-bridge.mjs --backend wasm --samples 20 > benchmarks/results/libfx/bridge-bun-wasm.json

- name: Upload raw libfx TTFT samples
uses: actions/upload-artifact@v4
with:
name: libfx-ttft-${{ github.sha }}
path: benchmarks/results/libfx/*.json
if-no-files-found: error
68 changes: 68 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,13 @@ jobs:
- name: Test Node + WASM
run: npm run --prefix sdk test:node-wasm

- name: Test libfx Wasm integrations
run: |
node --experimental-wasm-jspi sdk/tests/test-host-tools.mjs wasm
node --experimental-wasm-jspi sdk/tests/test-host-tool-cancel.mjs wasm
node --experimental-wasm-jspi sdk/tests/test-mcp-adapter.mjs http wasm
node --experimental-wasm-jspi sdk/tests/test-skills-adapter.mjs host wasm

sdk-node-napi:
name: SDK (Node + N-API)
runs-on: ubuntu-latest
Expand All @@ -109,6 +116,67 @@ jobs:
- name: Test Node + N-API
run: npm run --prefix sdk test:node-napi

- name: Test minimal native integrations
run: |
node sdk/tests/test-minimal-api.mjs
node sdk/tests/test-host-tools.mjs native
node sdk/tests/test-host-tool-cancel.mjs native
node sdk/tests/test-mcp-adapter.mjs stdio native
node sdk/tests/test-skills-adapter.mjs disk native

sdk-bun:
name: SDK (Bun + N-API/WASM)
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4

- name: Setup Zig
uses: mlugg/setup-zig@v2
with:
version: "0.16.0"

- name: Setup Node
uses: actions/setup-node@v4
with:
node-version: "24"

- name: Setup Bun
uses: oven-sh/setup-bun@v2
with:
bun-version: "1.3.14"

- name: Build SDK artifacts
run: |
zig build -Dnapi-surface=core -Doptimize=ReleaseSafe
zig build -Dwasm-surface=core -Doptimize=ReleaseSmall
zig build -Dwasm-surface=term -Doptimize=ReleaseSmall

- name: Install terminal test dependencies
run: npm ci --prefix sdk/node

- name: Test Bun + N-API
run: bun sdk/tests/test-node-napi.mjs

- name: Test Bun + WASM
run: bun sdk/tests/test-node-wasm.mjs

- name: Test Bun benchmark path
run: bun sdk/tests/test-libfx-benchmark.mjs

- name: Test Bun cross-backend integrations
run: |
bun sdk/tests/test-agent-bootstrap.mjs auto
bun sdk/tests/test-checkpoint.mjs native wasm
bun sdk/tests/test-checkpoint.mjs wasm native
bun sdk/tests/test-host-tools.mjs native
bun sdk/tests/test-host-tools.mjs wasm
bun sdk/tests/test-host-tool-cancel.mjs native
bun sdk/tests/test-host-tool-cancel.mjs wasm
bun sdk/tests/test-mcp-adapter.mjs stdio native
bun sdk/tests/test-mcp-adapter.mjs http wasm
bun sdk/tests/test-skills-adapter.mjs disk native
bun sdk/tests/test-skills-adapter.mjs host wasm

sdk-browser-wasm:
name: SDK (Browser + WASM)
runs-on: ubuntu-latest
Expand Down
8 changes: 7 additions & 1 deletion .github/workflows/publish-libfx.yml
Original file line number Diff line number Diff line change
Expand Up @@ -261,13 +261,18 @@ jobs:

rm -rf sdk/dist/libfx
mkdir -p sdk/dist/libfx
cp sdk/package.json sdk/README.md sdk/browser.js sdk/node.js sdk/fx-sdk.js LICENSE sdk/dist/libfx/
cp sdk/package.json sdk/README.md sdk/browser.js sdk/node.js sdk/fx-sdk.js sdk/mcp.js sdk/skills.js sdk/skills-node.js LICENSE sdk/dist/libfx/
cp wasm-artifacts/fx-core.wasm wasm-artifacts/fx-term.wasm sdk/dist/libfx/
cp native-addons/*.node sdk/dist/libfx/
jq --arg version "$LIBFX_VERSION" '.version = $version | del(.files)' \
sdk/dist/libfx/package.json > sdk/dist/libfx/package.json.tmp
mv sdk/dist/libfx/package.json.tmp sdk/dist/libfx/package.json

- name: Run packed public API example
run: |
node sdk/tests/test-packed-example.mjs sdk/dist/libfx native
node --experimental-wasm-jspi sdk/tests/test-packed-example.mjs sdk/dist/libfx wasm

- name: Validate package archive
run: |
set -euo pipefail
Expand All @@ -277,6 +282,7 @@ jobs:
const names = new Set(report.files.map(({ path }) => path));
const required = [
"package.json", "README.md", "LICENSE", "browser.js", "node.js", "fx-sdk.js",
"mcp.js", "skills.js", "skills-node.js",
"fx-core.wasm", "fx-term.wasm",
"libfx.linux-x64.node", "libfx.linux-arm64.node",
"libfx.darwin-x64.node", "libfx.darwin-arm64.node",
Expand Down
92 changes: 92 additions & 0 deletions benchmarks/libfx/bench-bridge.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
#!/usr/bin/env node
import { readFile } from "node:fs/promises";
import { createServer } from "node:http";
import { resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { createFxAgent } from "../../sdk/node.js";

const args = process.argv.slice(2);
const value = (name, fallback) => {
const index = args.indexOf(name);
return index < 0 ? fallback : args[index + 1];
};
const backend = value("--backend", "native");
const samples = Number(value("--samples", "20"));
if (!new Set(["native", "wasm"]).has(backend) || !Number.isInteger(samples) || samples < 1 || samples > 1000) {
throw new Error("usage: bench-bridge.mjs --backend native|wasm --samples 1..1000");
}

const root = resolve(fileURLToPath(new URL("../..", import.meta.url)));
const stages = [];
let requestIndex = 0;
const server = createServer((request, response) => {
let body = "";
request.setEncoding("utf8");
request.on("data", (chunk) => { body += chunk; });
request.on("end", () => {
if (request.method === "GET") {
response.writeHead(200, { "content-type": "application/json" });
response.end('{"object":"list","data":[]}');
return;
}
const sample = Math.floor(requestIndex / 2);
const followup = requestIndex % 2 === 1;
requestIndex += 1;
response.writeHead(200, { "content-type": "text/event-stream" });
if (!followup) {
stages[sample] = { tool_event_at: performance.now() };
response.end(`data: {"type":"tool-call","toolCallId":"bridge_${sample}","toolName":"bridge_echo","input":{"value":"${sample}"}}\n\ndata: {"type":"finish","finishReason":{"unified":"tool-calls","raw":"tool-calls"}}\n\ndata: [DONE]\n\n`);
return;
}
stages[sample].followup_fetch_at = performance.now();
if (!body.includes(`bridge:${sample}`)) throw new Error("bridge result missing from follow-up request");
response.end('data: {"type":"text-delta","delta":"ok"}\n\ndata: {"type":"finish","finishReason":{"unified":"stop","raw":"stop"}}\n\ndata: [DONE]\n\n');
});
});
await new Promise((resolveListen) => server.listen(0, "127.0.0.1", resolveListen));

let activeSample = 0;
const agent = await createFxAgent({
backend,
nativeAddon: resolve(root, "zig-out/lib/libfx.node"),
...(backend === "wasm" ? { wasm: await readFile(resolve(root, "zig-out/bin/fx-core.wasm")) } : {}),
fetch,
tools: [{
name: "bridge_echo",
description: "Measure the host tool bridge",
inputSchema: { type: "object", properties: { value: { type: "string" } }, required: ["value"] },
execute(input) {
stages[activeSample].callback_at = performance.now();
return `bridge:${input.value}`;
},
}],
env: {
AI_GATEWAY_API_KEY: "bridge-key",
FX_GATEWAY_CHAT_URL: `http://127.0.0.1:${server.address().port}/chat`,
FX_MODEL: "bridge/model",
},
});

try {
for (activeSample = 0; activeSample < samples; activeSample++) {
const turn = agent.prompt(`bridge sample ${activeSample}`);
for await (const _ of turn) {}
if ((await turn.result).stopReason !== "end_turn") throw new Error("bridge sample did not finish");
}
const report = stages.map((stage) => ({
tool_event_to_callback_ms: stage.callback_at - stage.tool_event_at,
callback_to_followup_fetch_ms: stage.followup_fetch_at - stage.callback_at,
tool_round_trip_ms: stage.followup_fetch_at - stage.tool_event_at,
}));
process.stdout.write(`${JSON.stringify({
format_version: 1,
runtime: typeof Bun === "undefined" ? "node" : "bun",
runtime_version: typeof Bun === "undefined" ? process.version : Bun.version,
backend,
samples: report,
}, null, 2)}\n`);
} finally {
await agent.close();
server.closeAllConnections();
await new Promise((resolveClose) => server.close(resolveClose));
}
Loading
Loading