feat: add timeouts for creating test runners (#390)

ojeytonwilliams · web-flow · commit dd989e932fd4 · 2025-07-08T17:27:37.000+02:00
* feat: add timeout for DOM evaluator

* feat: add timeout when creating worker test runners
diff --git a/packages/main/src/index.ts b/packages/main/src/index.ts
@@ -6,6 +6,27 @@ declare global {
   }
 }
 
+interface EvaluatorConfig {
+  // The compiled user code, evaluated before the tests.
+  source?: string;
+  type: "dom" | "javascript" | "python";
+  // TODO: move assetPath to RunnerConfig when making a major version bump.
+  assetPath?: string;
+  // The original user code, available for the tests to use.
+  code?: { contents?: string; editableContents?: string };
+  hooks?: {
+    beforeAll?: string;
+    beforeEach?: string;
+    afterEach?: string;
+    afterAll?: string;
+  };
+  loadEnzyme?: boolean;
+}
+
+interface RunnerConfig {
+  timeout?: number;
+}
+
 export class FCCTestRunner {
   #DOMRunner: DOMTestRunner | null;
   #javascriptRunner: WorkerTestRunner | null;
@@ -31,29 +52,10 @@ export class FCCTestRunner {
     }
   }
 
-  async createTestRunner({
-    source,
-    type,
-    code,
-    assetPath,
-    hooks,
-    loadEnzyme,
-  }: {
-    // The compiled user code, evaluated before the tests.
-    source?: string;
-    type: "dom" | "javascript" | "python";
-    // TODO: can we avoid using `assetPath` and use `import.meta.url` instead?
-    assetPath?: string;
-    // The original user code, available for the tests to use.
-    code?: { contents?: string; editableContents?: string };
-    hooks?: {
-      beforeAll?: string;
-      beforeEach?: string;
-      afterEach?: string;
-      afterAll?: string;
-    };
-    loadEnzyme?: boolean;
-  }) {
+  async createTestRunner(
+    { source, type, code, assetPath, hooks, loadEnzyme }: EvaluatorConfig,
+    { timeout }: RunnerConfig = { timeout: 20000 },
+  ) {
     let testRunner: DOMTestRunner | WorkerTestRunner | null = null;
     // eslint-disable-next-line default-case
     switch (type) {
@@ -80,7 +82,7 @@ export class FCCTestRunner {
         break;
     }
 
-    await testRunner.init({ code, source, loadEnzyme, hooks });
+    await testRunner.init({ code, source, loadEnzyme, hooks }, timeout);
 
     return testRunner;
   }
diff --git a/packages/main/src/test-runner.ts b/packages/main/src/test-runner.ts
@@ -15,7 +15,7 @@ import {
 import { post } from "./awaitable-post";
 
 interface Runner {
-  init(opts?: InitOptions): Promise<void>;
+  init(opts?: InitOptions, timeout?: number): Promise<void>;
   // Note: timeouts are currently ignored in the FrameRunner, since the purpose
   // is to stop evaluation if it is looping indefinitely, but any abort
   // mechanism (e.g. Promise.race or AbortController) would not get called in
@@ -92,7 +92,7 @@ export class DOMTestRunner implements Runner {
   }
 
   // Rather than trying to create an async constructor, we'll use an init method
-  async init(opts: InitTestFrameOptions) {
+  async init(opts: InitTestFrameOptions, timeout?: number) {
     const { hooks } = opts;
     const hooksScript = hooks?.beforeAll
       ? `<script id='${TEST_EVALUATOR_HOOKS_ID}'>
@@ -102,9 +102,14 @@ ${hooks.beforeAll}
 
     this.#afterAll = hooks?.afterAll;
 
-    const isReady = new Promise((resolve) => {
+    const isReady = new Promise((resolve, reject) => {
+      const timerId = setTimeout(
+        () => reject(Error("Timed out waiting for the test frame to load")),
+        timeout,
+      );
       const listener = () => {
         this.#testEvaluator.removeEventListener("load", listener);
+        clearTimeout(timerId);
         resolve(true);
       };
 
@@ -181,6 +186,7 @@ ${opts.source}`;
 export class WorkerTestRunner implements Runner {
   #testEvaluator: Worker;
   #opts: InitWorkerOptions | null = null;
+  #timeout?: number;
   #scriptUrl = "";
 
   #createTestEvaluator({ assetPath, script }: RunnerConfig) {
@@ -192,24 +198,41 @@ export class WorkerTestRunner implements Runner {
     this.#testEvaluator = this.#createTestEvaluator(config);
   }
 
-  async init(opts: InitWorkerOptions) {
+  async init(opts: InitWorkerOptions, timeout?: number) {
     this.#opts = opts;
+    this.#timeout = timeout;
     const msg: InitEvent<InitWorkerOptions>["data"] = {
       type: "init",
       value: opts,
     };
-    await post({
+
+    let timerId;
+    const isFrozen = new Promise<void>((_resolve, reject) => {
+      timerId = setTimeout(
+        () =>
+          reject(
+            new Error("Timed out waiting for the test worker to initialize"),
+          ),
+        timeout,
+      );
+    });
+
+    const response = post({
       messenger: this.#testEvaluator,
       message: msg,
     });
+
+    await Promise.race([response, isFrozen]);
+
+    clearTimeout(timerId);
   }
 
   async #recreateRunner() {
     if (!this.#opts || !this.#scriptUrl) {
       throw new Error("WorkerTestRunner not initialized");
     } else {
       this.#testEvaluator = new Worker(this.#scriptUrl);
-      await this.init(this.#opts);
+      await this.init(this.#opts, this.#timeout);
     }
   }
 
diff --git a/packages/tests/integration-tests/index.test.ts b/packages/tests/integration-tests/index.test.ts
@@ -24,6 +24,34 @@ describe("Test Runner", () => {
     });
   });
 
+  it("should throw if createTestRunner times out while creating a DOM runner", async () => {
+    await expect(() =>
+      page.evaluate(async () => {
+        await window.FCCTestRunner.createTestRunner(
+          { type: "dom" },
+          { timeout: 0 },
+        );
+      }),
+    ).rejects.toThrow("Timed out waiting for the test frame to load");
+  });
+
+  it("should throw if createTestRunner times out while creating a Worker runner", async () => {
+    await expect(() =>
+      page.evaluate(async () => {
+        await window.FCCTestRunner.createTestRunner(
+          { type: "javascript" },
+          { timeout: 0 },
+        );
+      }),
+    ).rejects.toThrow("Timed out waiting for the test worker to initialize");
+  });
+
+  // Worker runners are not timing out yet, but if they start to we'll need this
+  // test to help debug
+  it.todo(
+    "should throw if createTestRunner times out while creating a worker runner",
+  );
+
   it("should add a FCCTestRunner to the window object", async () => {
     const sandbox = await page.evaluate(() => window.FCCTestRunner);
 
@@ -1148,19 +1176,12 @@ assert(mocked.find('.greeting').length === 1);
   });
 
   describe("Python evaluator", () => {
-    beforeAll(async () => {
-      await page.evaluate(async () => {
-        await window.FCCTestRunner.createTestRunner({
-          type: "python",
-        });
-      });
-    });
     it("should run tests after evaluating the source supplied to the runner", async () => {
       const source = `def get_five():
   return 5`;
       const result = await page.evaluate(async (source) => {
-        const runner = window.FCCTestRunner.getRunner("python");
-        await runner?.init({
+        const runner = await window.FCCTestRunner.createTestRunner({
+          type: "python",
           source,
         });
         return runner?.runTest(
@@ -1177,15 +1198,15 @@ test: () => assert.equal(runPython('get_five()'), 5),
       const source = `def get_five():
   return 5`;
       await page.evaluate(async (source) => {
-        const runner = window.FCCTestRunner.getRunner("python");
-        await runner?.init({
+        await window.FCCTestRunner.createTestRunner({
+          type: "python",
           source,
         });
       }, source);
 
       const result = await page.evaluate(async () => {
         const runner = window.FCCTestRunner.getRunner("python");
-        await runner?.init({});
+        await runner?.init({}, 1000);
         return runner?.runTest(
           `({
 test: () => assert.equal(runPython('get_five()'), 5),
@@ -1205,8 +1226,9 @@ test: () => assert.equal(runPython('get_five()'), 5),
 
     it("should set __name__ to __main__ when running tests", async () => {
       const result = await page.evaluate(async () => {
-        const runner = window.FCCTestRunner.getRunner("python");
-        await runner?.init({});
+        const runner = await window.FCCTestRunner.createTestRunner({
+          type: "python",
+        });
         return runner?.runTest(
           `({
 test: () => assert.equal(runPython('__name__'), '__main__'),
@@ -1219,8 +1241,8 @@ test: () => assert.equal(runPython('__name__'), '__main__'),
 
     it("should handle js-only tests", async () => {
       const result = await page.evaluate(async () => {
-        const runner = window.FCCTestRunner.getRunner("python");
-        await runner?.init({
+        const runner = await window.FCCTestRunner.createTestRunner({
+          type: "python",
           code: {
             contents: "# wrong comment for test",
           },
@@ -1243,8 +1265,9 @@ test: () => assert.equal(runPython('__name__'), '__main__'),
 
     it("should reject testStrings that evaluate to an invalid object", async () => {
       const result = await page.evaluate(async () => {
-        const runner = window.FCCTestRunner.getRunner("python");
-        await runner?.init({});
+        const runner = await window.FCCTestRunner.createTestRunner({
+          type: "python",
+        });
         return runner?.runTest(`({ invalid: 'test' })`);
       });
 
@@ -1262,13 +1285,14 @@ test: () => assert.equal(runPython('__name__'), '__main__'),
 
     it("should be able to test with mock input", async () => {
       const result = await page.evaluate(async () => {
-        const runner = window.FCCTestRunner.getRunner("python");
-        await runner?.init({
+        const runner = await window.FCCTestRunner.createTestRunner({
+          type: "python",
           source: `
 first = input()
 second = input()
 `,
         });
+
         return runner?.runTest(`({
 	input: ["argle", "bargle"],
   test: () => assert.equal(runPython('first + second'), "arglebargle")
@@ -1280,12 +1304,13 @@ second = input()
 
     it("should make user code available to the python code as the _code variable", async () => {
       const result = await page.evaluate(async () => {
-        const runner = window.FCCTestRunner.getRunner("python");
-        await runner?.init({
+        const runner = await window.FCCTestRunner.createTestRunner({
+          type: "python",
           code: {
             contents: "test = 'value'",
           },
         });
+
         return runner?.runTest(`({
   test: () => assert.equal(runPython('_code'), "test = 'value'")
 })`);
@@ -1296,8 +1321,9 @@ second = input()
 
     it("should make the AST helper available to the python code as _Node", async () => {
       const result = await page.evaluate(async () => {
-        const runner = window.FCCTestRunner.getRunner("python");
-        await runner?.init({});
+        const runner = await window.FCCTestRunner.createTestRunner({
+          type: "python",
+        });
         return runner?.runTest(`({
   test: () => assert.equal(runPython('_Node("x = 1").get_variable("x")'), 1)
 })`);
@@ -1308,15 +1334,16 @@ second = input()
 
     it("should have access to _code in the beforeEach hook", async () => {
       const result = await page.evaluate(async () => {
-        const runner = window.FCCTestRunner.getRunner("python");
-        await runner?.init({
+        const runner = await window.FCCTestRunner.createTestRunner({
+          type: "python",
           code: {
             contents: "test = 'value'",
           },
           hooks: {
             beforeEach: "assert.equal(runPython('_code'), \"test = 'value'\")",
           },
         });
+
         return runner?.runTest(`({
   test: () => {}
 })`);
@@ -1327,12 +1354,13 @@ second = input()
 
     it("should have access to runPython in the beforeEach hook", async () => {
       const result = await page.evaluate(async () => {
-        const runner = window.FCCTestRunner.getRunner("python");
-        await runner?.init({
+        const runner = await window.FCCTestRunner.createTestRunner({
+          type: "python",
           hooks: {
             beforeEach: "assert.equal(runPython('1 + 1'), 2)",
           },
         });
+
         return runner?.runTest("");
       });
 
@@ -1341,8 +1369,9 @@ second = input()
 
     it("should return error types if the python code raises an exception", async () => {
       const result = await page.evaluate(async () => {
-        const runner = window.FCCTestRunner.getRunner("python");
-        await runner?.init({});
+        const runner = await window.FCCTestRunner.createTestRunner({
+          type: "python",
+        });
         return runner?.runTest(`({
 	test: () => assert.equal(runPython('1 + "1"'), 2)
 })`);
@@ -1367,8 +1396,8 @@ second = input()
 pattern = re.compile('l+')
 `;
       const result = await page.evaluate(async (source) => {
-        const runner = window.FCCTestRunner.getRunner("python");
-        await runner?.init({
+        const runner = await window.FCCTestRunner.createTestRunner({
+          type: "python",
           source,
         });
         return runner?.runTest(`({
@@ -1394,8 +1423,8 @@ import re
 
 pattern = re.compile('l+')`;
       const result = await page.evaluate(async (source) => {
-        const runner = window.FCCTestRunner.getRunner("python");
-        await runner?.init({
+        const runner = await window.FCCTestRunner.createTestRunner({
+          type: "python",
           source,
         });
         // Since the comparison includes a PyProxy object, that will be
diff --git a/packages/tests/integration-tests/timeout.test.ts b/packages/tests/integration-tests/timeout.test.ts
