Skip to content

feat: add custom vfs support #3

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
pi0 wants to merge 1 commit into
base: main
Choose a base branch
from
+190 −17
Draft

feat: add custom vfs support #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 6 additions & 4 deletions lib/_rg.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,11 +37,13 @@ export async function createWasiRuntime({
nodeWasi,
preopens,
returnOnExit,
vfs,
}) {
const config = { args, stdout, stderr, env, preopens, returnOnExit };
const config = { args, stdout, stderr, env, preopens, returnOnExit, vfs };
// node:wasi only accepts numeric fd — fall back to the custom shim for stream-based overrides.
// Custom vfs also requires the custom shim since node:wasi doesn't support it.
const hasStreamStdio = (stdout && stdout.fd == null) || (stderr && stderr.fd == null);
if (!nodeWasi || hasStreamStdio) return createWasiShim(config);
if (!nodeWasi || hasStreamStdio || vfs) return createWasiShim(config);

try {
return await createNodeWasi(config);
Expand All @@ -52,9 +54,9 @@ export async function createWasiRuntime({

let _wasiShim;

async function createWasiShim({ args, stdout, stderr, env, preopens, returnOnExit }) {
async function createWasiShim({ args, stdout, stderr, env, preopens, returnOnExit, vfs }) {
const { createWasi } = await (_wasiShim ??= import("./_wasi.mjs"));
return createWasi({ args: ["rg", ...args], stdout, stderr, env, preopens, returnOnExit });
return createWasi({ args: ["rg", ...args], stdout, stderr, env, preopens, returnOnExit, vfs });
}

let _nodeWasi;
Expand Down
27 changes: 14 additions & 13 deletions lib/_wasi.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,8 @@ class WASIExit extends Error {
}
}

export function createWasi({ args, stdout, stderr, env, preopens, returnOnExit = true } = {}) {
export function createWasi({ args, stdout, stderr, env, preopens, returnOnExit = true, vfs } = {}) {
const _fs = vfs ? { ...fs, ...vfs } : fs;
const enc = new TextEncoder();
const dec = new TextDecoder();

Expand Down Expand Up @@ -162,7 +163,7 @@ export function createWasi({ args, stdout, stderr, env, preopens, returnOnExit =
let n = 0;
if (e.type === "file") {
const target = Buffer.from(mem.buffer, mem.byteOffset + bufPtr, bufLen);
n = fs.readSync(e.hostFd, target, 0, bufLen, null);
n = _fs.readSync(e.hostFd, target, 0, bufLen, null);
e.pos += BigInt(n);
} else if (e.type === "stdio" && e.which === 0) {
// No stdin support (would need blocking read). Signal EOF.
Expand Down Expand Up @@ -196,7 +197,7 @@ export function createWasi({ args, stdout, stderr, env, preopens, returnOnExit =
else return E.BADF;
total += bufLen;
} else if (e.type === "file") {
const n = fs.writeSync(e.hostFd, chunk, 0, bufLen, null);
const n = _fs.writeSync(e.hostFd, chunk, 0, bufLen, null);
e.pos += BigInt(n);
total += n;
if (n < bufLen) break;
Expand All @@ -214,7 +215,7 @@ export function createWasi({ args, stdout, stderr, env, preopens, returnOnExit =
const e = fds[fd];
if (!e) return E.BADF;
try {
if (e.type === "file") fs.closeSync(e.hostFd);
if (e.type === "file") _fs.closeSync(e.hostFd);
fds[fd] = undefined;
return E.SUCCESS;
} catch (err) {
Expand All @@ -231,7 +232,7 @@ export function createWasi({ args, stdout, stderr, env, preopens, returnOnExit =
const e = fds[fd];
if (!e || e.type !== "file") return E.BADF;
try {
const size = BigInt(fs.fstatSync(e.hostFd).size);
const size = BigInt(_fs.fstatSync(e.hostFd).size);
let base;
if (whence === 0)
base = 0n; // SET
Expand All @@ -256,7 +257,7 @@ export function createWasi({ args, stdout, stderr, env, preopens, returnOnExit =
const mem = u8();
try {
if (!e.dirents) {
const list = fs.readdirSync(e.hostPath, { withFileTypes: true });
const list = _fs.readdirSync(e.hostPath, { withFileTypes: true });
e.dirents = list.map((d) => ({
name: d.name,
nameBytes: enc.encode(d.name),
Expand Down Expand Up @@ -309,8 +310,8 @@ export function createWasi({ args, stdout, stderr, env, preopens, returnOnExit =
}
const st =
e.type === "file"
? fs.fstatSync(e.hostFd, { bigint: true })
: fs.statSync(e.hostPath, { bigint: true });
? _fs.fstatSync(e.hostFd, { bigint: true })
: _fs.statSync(e.hostPath, { bigint: true });
writeFilestat(dv(), filestatPtr, filestatFromNode(st));
return E.SUCCESS;
} catch (err) {
Expand Down Expand Up @@ -373,7 +374,7 @@ export function createWasi({ args, stdout, stderr, env, preopens, returnOnExit =
try {
let st;
try {
st = fs.statSync(fullPath);
st = _fs.statSync(fullPath);
} catch (err) {
// Only swallow ENOENT when O_CREAT is set; propagate everything else.
if (!(oflags & OFLAGS_CREAT) || err?.code !== "ENOENT") return errno(err);
Expand All @@ -396,7 +397,7 @@ export function createWasi({ args, stdout, stderr, env, preopens, returnOnExit =
if (oflags & OFLAGS_TRUNC) flags |= fs.constants.O_TRUNC;
if (fdflags & FDFLAGS_APPEND) flags |= fs.constants.O_APPEND;

const hostFd = fs.openSync(fullPath, flags);
const hostFd = _fs.openSync(fullPath, flags);
fds.push({ type: "file", hostFd, pos: 0n });
v.setUint32(openedFdPtr, fds.length - 1, true);
return E.SUCCESS;
Expand All @@ -413,8 +414,8 @@ export function createWasi({ args, stdout, stderr, env, preopens, returnOnExit =
// bit 0 of lookupflags = symlink_follow
const follow = (flags & 1) !== 0;
const st = follow
? fs.statSync(fullPath, { bigint: true })
: fs.lstatSync(fullPath, { bigint: true });
? _fs.statSync(fullPath, { bigint: true })
: _fs.lstatSync(fullPath, { bigint: true });
writeFilestat(dv(), filestatPtr, filestatFromNode(st));
return E.SUCCESS;
} catch (err) {
Expand All @@ -427,7 +428,7 @@ export function createWasi({ args, stdout, stderr, env, preopens, returnOnExit =
const relPath = dec.decode(u8().subarray(pathPtr, pathPtr + pathLen));
const fullPath = path.resolve(e.hostPath, relPath);
try {
const target = enc.encode(fs.readlinkSync(fullPath));
const target = enc.encode(_fs.readlinkSync(fullPath));
const n = Math.min(target.length, bufLen);
u8().set(target.subarray(0, n), bufPtr);
dv().setUint32(bufUsedPtr, n, true);
Expand Down
38 changes: 38 additions & 0 deletions lib/index.d.mts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,36 @@ export type RgFlag =
*/
export type RgArg = RgFlag | (string & {});

/**
* Virtual filesystem interface for the custom WASI shim.
* All methods mirror their `node:fs` equivalents. Only the methods
* ripgrep actually calls need to be implemented — missing methods
* fall back to the real `node:fs`.
*/
export interface RipgrepVfs {
openSync?(path: string, flags: number): number;
closeSync?(fd: number): void;
readSync?(
fd: number,
buffer: Buffer,
offset: number,
length: number,
position: number | null,
): number;
writeSync?(
fd: number,
buffer: Uint8Array,
offset: number,
length: number,
position: number | null,
): number;
fstatSync?(fd: number, options?: { bigint?: boolean }): any;
statSync?(path: string, options?: { bigint?: boolean }): any;
lstatSync?(path: string, options?: { bigint?: boolean }): any;
readdirSync?(path: string, options?: { withFileTypes?: boolean }): any[];
readlinkSync?(path: string): string;
}

/**
* Options for {@link ripgrep}.
*/
Expand Down Expand Up @@ -101,6 +131,14 @@ export interface ripgrepOptions {
* @default false
*/
nodeWasi?: boolean;

/**
* Custom virtual filesystem for the WASI shim. When provided, the
* custom WASI shim is always used (regardless of `nodeWasi`).
* Only the methods ripgrep calls need to be implemented — missing
* methods fall back to the real `node:fs`.
*/
vfs?: RipgrepVfs;
}

/**
Expand Down
2 changes: 2 additions & 0 deletions lib/index.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ export async function ripgrep(args = [], options = {}) {
preopens = { ".": process.cwd() },
returnOnExit = true,
nodeWasi = getDefaultNodeWasi(),
vfs,
} = options;

let stdoutChunks, stderrChunks;
Expand Down Expand Up @@ -55,6 +56,7 @@ export async function ripgrep(args = [], options = {}) {
nodeWasi,
preopens: resolvedPreopens,
returnOnExit,
vfs,
});

const wasm = await getRgWasmModule();
Expand Down
130 changes: 130 additions & 0 deletions test/ripgrep.test.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { describe, it, expect } from "vitest";
import { ripgrep, rgPath } from "../lib/index.mjs";
import * as fs from "node:fs";
import { existsSync } from "node:fs";
import { spawn } from "node:child_process";

Expand Down Expand Up @@ -137,6 +138,135 @@
});
});

describe("ripgrep (vfs)", () => {
it("searches using a custom vfs", async () => {
const virtualFiles = {
"/vfs/hello.txt": "hello from virtual filesystem\n",
};
const vfs = {
readdirSync(dirPath, opts) {
if (dirPath in virtualDirs()) {
const entries = virtualDirs()[dirPath];
if (opts?.withFileTypes) {
return entries.map((name) => ({
name,
isFile: () => !name.endsWith("/"),
isDirectory: () => name.endsWith("/"),
isSymbolicLink: () => false,
isBlockDevice: () => false,
isCharacterDevice: () => false,
}));
}
return entries;
}
return fs.readdirSync(dirPath, opts);
},
statSync(filePath, opts) {
if (filePath in virtualFiles) {
const size = Buffer.byteLength(virtualFiles[filePath]);
const s = {
isFile: () => true,
isDirectory: () => false,
isSymbolicLink: () => false,
isBlockDevice: () => false,
isCharacterDevice: () => false,
dev: opts?.bigint ? 0n : 0,
ino: opts?.bigint ? 0n : 0,
nlink: opts?.bigint ? 1n : 1,
size: opts?.bigint ? BigInt(size) : size,
atimeNs: opts?.bigint ? 0n : undefined,
mtimeNs: opts?.bigint ? 0n : undefined,
ctimeNs: opts?.bigint ? 0n : undefined,
};
return s;
}
if (filePath === "/vfs") {
return {
isFile: () => false,
isDirectory: () => true,
isSymbolicLink: () => false,
isBlockDevice: () => false,
isCharacterDevice: () => false,
dev: opts?.bigint ? 0n : 0,
ino: opts?.bigint ? 0n : 0,
nlink: opts?.bigint ? 1n : 1,
size: opts?.bigint ? 0n : 0,
atimeNs: opts?.bigint ? 0n : undefined,
mtimeNs: opts?.bigint ? 0n : undefined,
ctimeNs: opts?.bigint ? 0n : undefined,
};
}
return fs.statSync(filePath, opts);
},
lstatSync(filePath, opts) {
return this.statSync(filePath, opts);
},
openSync(filePath, flags) {
if (filePath in virtualFiles) {
// Use a high fd number to avoid collisions
const fd = 1000 + Object.keys(virtualFiles).indexOf(filePath);
vfs._openFiles = vfs._openFiles || {};
vfs._openFiles[fd] = { path: filePath, pos: 0 };
return fd;
}
return fs.openSync(filePath, flags);
},
closeSync(fd) {
if (vfs._openFiles?.[fd]) {
delete vfs._openFiles[fd];
return;
}
return fs.closeSync(fd);
},
readSync(fd, buffer, offset, length, position) {
if (vfs._openFiles?.[fd]) {
const f = vfs._openFiles[fd];
const content = Buffer.from(virtualFiles[f.path]);
const pos = position != null ? position : f.pos;
const n = Math.min(length, content.length - pos);
if (n <= 0) return 0;
content.copy(buffer, offset, pos, pos + n);
f.pos = pos + n;
return n;
}
return fs.readSync(fd, buffer, offset, length, position);
},
fstatSync(fd, opts) {
if (vfs._openFiles?.[fd]) {
return this.statSync(vfs._openFiles[fd].path, opts);
}
return fs.fstatSync(fd, opts);
},
};

function virtualDirs() {
return { "/vfs": ["hello.txt"] };
}

const res = await ripgrep(["hello", "/vfs"], {
buffer: true,
preopens: { "/vfs": "/vfs" },
vfs,
});
expect(res.code).toBe(0);

Check failure on line 251 in test/ripgrep.test.mjs

View workflow job for this annotation

GitHub Actions / test (windows-latest)

test/ripgrep.test.mjs > ripgrep (vfs) > searches using a custom vfs 

AssertionError: expected 2 to be +0 // Object.is equality

- Expected
+ Received

- 0
+ 2

 ❯ test/ripgrep.test.mjs:251:22
expect(res.stdout).toContain("hello from virtual filesystem");
});

it("falls back to real fs for methods not on vfs", async () => {
const calls = [];
const vfs = {
statSync(filePath, opts) {
calls.push(filePath);
return fs.statSync(filePath, opts);
},
};
const res = await ripgrep(["hello", HELLO], { buffer: true, vfs });
expect(res.code).toBe(0);
expect(res.stdout).toContain("hello ripgrep world");
expect(calls.length).toBeGreaterThan(0);
});
});

describe("ripgrep (non-buffered)", () => {
it("returns code without stdout/stderr fields", async () => {
const res = await ripgrep(["hello", HELLO]);
Expand Down
Loading