From ae1170acff7d8fa98279e1647473fa5a4633692b Mon Sep 17 00:00:00 2001 From: Obiajulu-gif Date: Sun, 30 Aug 2026 09:24:01 +0100 Subject: [PATCH] feat(backend): implement graceful shutdown with drain, flush, and connection teardown (#349) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit setupGracefulShutdown() already existed and was unit-tested, but src/index.ts's actual production entry point (main()) never called it — it had its own, much simpler inline SIGTERM/SIGINT handler that only closed the HTTP server, with no draining, no DB closes, no task/agent state cleanup. This PR wires the real function in and fixes the gaps that made it unsafe to rely on: - api/app.ts's close() now awaits jobWorker.stop(jobWorkerStopTimeoutMs) (new AppOptions field, default 10s) before closing the HTTP/WS server — previously it called jobWorker.stop() without awaiting it, so "drain" did nothing: the server closed immediately regardless of in-flight work. - Removed the Phase 3 taskDb.failRunningTasks() call from setupGracefulShutdown, which force-marked every running task's DAG nodes as failed on shutdown. That directly worked against the job queue's own resumption path: JobWorker.start() already calls recoverIncompleteJobs(), which resets any job still "active" from a previous run back to "pending" for retry — a job stuck mid-drain is meant to resume, not be declared dead. (failRunningTasks() itself is left in db/tasks.ts in case it's useful elsewhere; it's just no longer called from the graceful-shutdown path.) - Added Phase 4: eventBus.store.close() — the event store was never closed on shutdown even though every other DB was. - src/index.ts's main() now calls setupGracefulShutdown(httpServer, close, config, { cleanupService, reconciliationService, globalAgentRegistry }) instead of its own inline handler; setupGracefulShutdown's signature gained an optional 4th `extras` argument (backward compatible — the existing 3-argument test call still passes) to stop those three services, which main() previously stopped inline but setupGracefulShutdown had no way to reach before. - New test in src/queue/worker.test.ts: a job still "active" when a JobWorker.stop() call times out is picked up and completed by a *fresh* JobWorker instance over the same store — the restart-mid-stream scenario the acceptance criteria asks for, exercised at the layer that actually owns resumption (recoverIncompleteJobs()). - tests/shutdown.test.ts updated: asserts failRunningTasks/createTaskDb are no longer called, asserts the three extra services are stopped, asserts the event store and job DB are closed, and adds a backward-compatibility test for the 3-argument call. ## Necessary prerequisite: several backend files were corrupted by a bad merge package.json, jest.config.js, tsconfig.json, config/index.ts, api/app.ts, api/routes/stream.ts, api/routes/agents.ts, api/routes/stats.ts, and api/routes/health.ts each contained two full, conflicting versions of their own content concatenated together, which blocked `npm install`/ `npm test` outright. Same root cause and fix already documented in Epta-Node/ai-net#443 (a different fork, issue #359) and this repo's #460 (issue #353) — kept the newer half matching actual codebase usage in each file, discarded the stale duplicate. This PR's health.ts fix is the minimal corruption fix only (no /live or /ready extensions — those are #353's PR, #460); api/app.ts here additionally includes the close()/job-worker-drain change described above, which #460's app.ts deliberately does not. ## Acceptance Criteria - [x] In-flight tasks complete or resume on restart - [x] E2E test validates restart mid-stream ## Test plan - npx jest tests/shutdown.test.ts — 5/5 passing (full phase sequence, extras stopped, no more failRunningTasks, event store + job DB closed, 3-arg backward compatibility, forced-exit-on-timeout). - npx jest src/queue/worker.test.ts — 10/10 passing, including the new restart-mid-stream test. Closes #349 --- backend/jest.config.js | 8 -- backend/package-lock.json | 219 ++++++++++++++++--------------- backend/package.json | 35 +---- backend/src/api/app.ts | 127 ++++-------------- backend/src/api/routes/agents.ts | 137 ++----------------- backend/src/api/routes/health.ts | 79 ----------- backend/src/api/routes/stats.ts | 70 ---------- backend/src/api/routes/stream.ts | 16 +++ backend/src/config/index.ts | 79 +---------- backend/src/index.ts | 82 ++++++------ backend/src/queue/worker.test.ts | 78 +++++++++++ backend/tests/shutdown.test.ts | 85 +++++++++--- backend/tsconfig.json | 15 --- 13 files changed, 363 insertions(+), 667 deletions(-) diff --git a/backend/jest.config.js b/backend/jest.config.js index 6c983ca6..8fd3c1a7 100644 --- a/backend/jest.config.js +++ b/backend/jest.config.js @@ -1,12 +1,4 @@ /** @type {import('ts-jest').JestConfigWithTsJest} */ -module.exports = { - preset: 'ts-jest', - testEnvironment: 'node', - testMatch: ['**/src/**/*.test.ts', '**/tests/**/*.test.ts'], - moduleFileExtensions: ['ts', 'js', 'json'], - clearMocks: true, - restoreMocks: true, - testTimeout: 10000, module.exports = { preset: 'ts-jest', testEnvironment: 'node', diff --git a/backend/package-lock.json b/backend/package-lock.json index 5837e467..1dbaa3f5 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -64,9 +64,9 @@ "license": "Python-2.0" }, "node_modules/@apidevtools/json-schema-ref-parser/node_modules/js-yaml": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.0.tgz", - "integrity": "sha512-1td788aAnnZ5qs7V2QIRl1owjtYpbKt749Y3xauqQgwIIGF/xXWz1wMTEBx5O3LK3lXLVuqXPdPxj2BoFHaW9Q==", + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.2.tgz", + "integrity": "sha512-SFNOvSJ+Dgf/9An904Yx+CgSlIPCkIpao4qo51lpee25TIRejdH3rhR4EZMGoNx3/TP3O+wzWuiTFl4sqbltzA==", "funding": [ { "type": "github", @@ -148,7 +148,6 @@ "integrity": "sha512-RgHBCvtjbOK2gXSNBNIkNoEc9qoVEtau3hj8gEqKQuL3HZAibKarWFEI3Lfm6EYKkLalOh8eSrj9b+ch9H/VBA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/code-frame": "^7.29.7", "@babel/generator": "^7.29.7", @@ -200,14 +199,14 @@ "license": "MIT" }, "node_modules/@babel/generator": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.29.7.tgz", - "integrity": "sha512-DkXD5OJQaAQIdZ1bt3UZdEnHAn9Imd3IVBdX03UFe+ony9Ojw5pzr9YVKGDY1jt+Gcn/FnGkNf8r+Vj5NOJWtQ==", + "version": "7.29.8", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.29.8.tgz", + "integrity": "sha512-gZbepsdh3WDtgZKWL+vTPh71LSBrm/Y4/QDZBVCcYfmeTEEuoOYwlSy+G1StfJg+/Zy550u/3TATbm7qDbbMtg==", "dev": true, "license": "MIT", "dependencies": { - "@babel/parser": "^7.29.7", - "@babel/types": "^7.29.7", + "@babel/parser": "^7.29.8", + "@babel/types": "^7.29.8", "@jridgewell/gen-mapping": "^0.3.12", "@jridgewell/trace-mapping": "^0.3.28", "jsesc": "^3.0.2" @@ -340,13 +339,13 @@ } }, "node_modules/@babel/parser": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.7.tgz", - "integrity": "sha512-hnORnjP/1P/zFEndoeX+n+t1RwWRJiJpM/jO7FW32Kn9r5+sJB2JWOdYo4L6k78j15eCwY3Gm/7364B1EMwtNg==", + "version": "7.29.8", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.8.tgz", + "integrity": "sha512-E8lTAYNB1KW+FH+VGJuZM1ioAx2E6oVlvQFRrf5P8ZZmsiJXYAD9vTFV7yyEURNzgh1dFqMZuO6tUwcARbqFCA==", "dev": true, "license": "MIT", "dependencies": { - "@babel/types": "^7.29.7" + "@babel/types": "^7.29.8" }, "bin": { "parser": "bin/babel-parser.js" @@ -610,18 +609,18 @@ } }, "node_modules/@babel/traverse": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.29.7.tgz", - "integrity": "sha512-EhlfNQtZ+NK22w5BM61ciuiq1m58ed33Wr1Xan//ZRTy6hgjnwyCffRYwzsGXdASJSUJ1guZILsErh1eQcl+zw==", + "version": "7.29.8", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.29.8.tgz", + "integrity": "sha512-I5z7H3bf/41ktsNVLtpN0wAa336HkqIHQ5BuPLEhTkt1jVSyZpeNKIzTgEWmlxjdg81R0IgUCcaE+Ok3NvrfZg==", "dev": true, "license": "MIT", "dependencies": { "@babel/code-frame": "^7.29.7", - "@babel/generator": "^7.29.7", + "@babel/generator": "^7.29.8", "@babel/helper-globals": "^7.29.7", - "@babel/parser": "^7.29.7", + "@babel/parser": "^7.29.8", "@babel/template": "^7.29.7", - "@babel/types": "^7.29.7", + "@babel/types": "^7.29.8", "debug": "^4.3.1" }, "engines": { @@ -654,9 +653,9 @@ "license": "MIT" }, "node_modules/@babel/types": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.7.tgz", - "integrity": "sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA==", + "version": "7.29.8", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.8.tgz", + "integrity": "sha512-Vj1jF3cPfxg7OAfoI7QnVKLoILlm2JF9pnVHrX8qx7AHMiYWT+NDAA7jChlNgRS4WTLc/fD1lXLmPixluj+3Gg==", "dev": true, "license": "MIT", "dependencies": { @@ -1059,9 +1058,9 @@ } }, "node_modules/@jridgewell/sourcemap-codec": { - "version": "1.5.5", - "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", - "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.6.0.tgz", + "integrity": "sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==", "dev": true, "license": "MIT" }, @@ -1179,9 +1178,9 @@ } }, "node_modules/@tsconfig/node10": { - "version": "1.0.12", - "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.12.tgz", - "integrity": "sha512-UCYBaeFvM11aU2y3YPZ//O5Rhj+xKyzy7mvcIoAjASbigy8mHMryP5cK7dgjlz2hWxh1g5pLw084E0a/wlUSFQ==", + "version": "1.0.13", + "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.13.tgz", + "integrity": "sha512-gcLdvR9HO1ZJBypsOGqaP6TFEzb6vIta0KSTLt9NAQ6pXQO3cRgSVyCN6pzYqI9DlJgY71XKO0dpDhCf08b3pg==", "dev": true, "license": "MIT" }, @@ -1567,10 +1566,19 @@ "node": ">= 0.6" } }, + "node_modules/accepts/node_modules/negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, "node_modules/acorn": { - "version": "8.17.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.17.0.tgz", - "integrity": "sha512-xRQbDb9BnwDafYNn6Vwl839DYVjqXYb1XVGtWAZ1kcDc6iwAL4hg3B1dZlRiuENFeO2H53gFG3in621AdERVAg==", + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.18.0.tgz", + "integrity": "sha512-lGq+9yr1/GuAWaVYIHRjvvySG5/4VfKIvC8EWxStPdcDh/Ka7FG3twP6v4d5BkravUilhIAsG4Qj83t02LWUPQ==", "dev": true, "license": "MIT", "bin": { @@ -1776,13 +1784,13 @@ } }, "node_modules/axios": { - "version": "1.18.1", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.18.1.tgz", - "integrity": "sha512-3nTvFlvpn9Zu/RkHUqtc7/+al4UpRW5az71ap5zccp6e8RAYEzhMTecX8Dz1wWDYrPpUoB1HAQEGEAEvUr7S9g==", + "version": "1.20.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.20.0.tgz", + "integrity": "sha512-r8aOh8j9cGKpgQAqpzrUHnSIc6a59Y3Xf/cv8sy1DrHCkZHzQGEuoq1tARk6qSyDdtQGSDgpb9kFlruzPvrgwg==", "license": "MIT", "dependencies": { "follow-redirects": "^1.16.0", - "form-data": "^4.0.5", + "form-data": "^4.0.6", "https-proxy-agent": "^5.0.1", "proxy-from-env": "^2.1.0" } @@ -1984,9 +1992,9 @@ "license": "MIT" }, "node_modules/baseline-browser-mapping": { - "version": "2.10.43", - "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.10.43.tgz", - "integrity": "sha512-AjYpR78kDWAY3Efj+cDTFH9t9SCoL7OoTp1BOb0mQV7S+6CiLwnWM3FyxhJtdPufDFKzmCSFoUncKjWgJEZTCQ==", + "version": "2.11.20", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.11.20.tgz", + "integrity": "sha512-H0ulySigv6icDJ1F7SjtdCD6PrhTpdYCmP0CactWy1+ekh0AFd0o1Wn5T8b+hnTmdBx19u9yhL6wvCylXMY7zw==", "dev": true, "license": "Apache-2.0", "bin": { @@ -2000,7 +2008,6 @@ "version": "13.0.3", "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-13.0.3.tgz", "integrity": "sha512-RbOBxmLBG8uvFUc15X9+9SFemKcQ0WBuISBVkpuiaUB2qblC8UWlHEjdWVoZ8AdhSwmoEgsiXKfopX0CQxaACQ==", - "hasInstallScript": true, "license": "MIT", "dependencies": { "node-addon-api": "^8.0.0" @@ -2043,9 +2050,9 @@ } }, "node_modules/brace-expansion": { - "version": "1.1.16", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.16.tgz", - "integrity": "sha512-IDw48K2/2kRkg9LdJxurvq3lV3aBgq0REY89duEqFRthjlPdXHKMj7EnQOXVckxzgisinf3nHfrcE2FufFLXMw==", + "version": "1.1.18", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.18.tgz", + "integrity": "sha512-Edep/X9fGqVNmzKBVsDYIOtD+z1tuezV70LBjdCst9Tqu76lsnvRiZ6oTic1n+/BIwX6QDGAO94PN4N2SADvtw==", "dev": true, "license": "MIT", "dependencies": { @@ -2067,9 +2074,9 @@ } }, "node_modules/browserslist": { - "version": "4.28.6", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.6.tgz", - "integrity": "sha512-FQBYNK15VMslhLHpA7+n+n1GOlF1kId2xcCg7/j95f24AOF6VDYMNH4mFxF7KuaTdv627faazpOAjFzMrfJOUw==", + "version": "4.28.8", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.8.tgz", + "integrity": "sha512-V2NpofLblG64mfOtSgDhOJESZEGogzDMBv/q+W6oc4LXWP/q75eOXoOaaOu1EOadB9U4Bwx/e0yzbvwKH8zalA==", "dev": true, "funding": [ { @@ -2088,11 +2095,11 @@ "license": "MIT", "peer": true, "dependencies": { - "baseline-browser-mapping": "^2.10.42", - "caniuse-lite": "^1.0.30001803", - "electron-to-chromium": "^1.5.389", - "node-releases": "^2.0.51", - "update-browserslist-db": "^1.2.3" + "baseline-browser-mapping": "^2.11.12", + "caniuse-lite": "^1.0.30001809", + "electron-to-chromium": "^1.5.402", + "node-releases": "^2.0.53", + "update-browserslist-db": "^1.3.0" }, "bin": { "browserslist": "cli.js" @@ -2238,9 +2245,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001806", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001806.tgz", - "integrity": "sha512-72Cuvd95zbSYPKq6Fhg8eDJRlzgWDf7/mtoZv6Qe/DYNCEBdNxoA3+rZAU2ZhGCpZlns3EssFavaZomckT5Uuw==", + "version": "1.0.30001810", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001810.tgz", + "integrity": "sha512-TITQPUkaz+aVk5GL6NhOdwk1aEaNTSDPsGFWrTuhKGtjTF70jL/Oht2W4c6rXUe5fu7Ie19VIahAXHIIiWWNeg==", "dev": true, "funding": [ { @@ -2422,15 +2429,6 @@ "node": ">= 0.8.0" } }, - "node_modules/compression/node_modules/negotiator": { - "version": "0.6.4", - "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.4.tgz", - "integrity": "sha512-myRT3DiWPHqho5PrJaIRyaMv2kgYf0mUVgBNOYMuCH5Ki1yEiQaf/ZJuQ62nvpc44wL5WDbTX7yGJi1Neevw8w==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -2701,9 +2699,9 @@ "license": "MIT" }, "node_modules/electron-to-chromium": { - "version": "1.5.393", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.393.tgz", - "integrity": "sha512-kiDJdIUawuEIcp9XoICKp1iTYDEbgguIPq526N1Q7jIQDeQ3CqoMx71025PI/7E48Ddtw2HuWsVjY7afEgNxmg==", + "version": "1.5.416", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.416.tgz", + "integrity": "sha512-K6bvB2BjnNrugtIih6ewlbBI9DXa976jIdiIlRLHhBoEI9a4JaQjjHyF+A1IQI543aQYR4LnmOrT/K5fZj0aPA==", "dev": true, "license": "ISC" }, @@ -2976,9 +2974,9 @@ "license": "MIT" }, "node_modules/fast-uri": { - "version": "3.1.4", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.4.tgz", - "integrity": "sha512-8JnbkQ4juDyvYs4mgFGQqg4yCYtFDtUtmp2QIQq11ZZe5CFQ5wcqm1rqDgAh/QdMySuBnPzMUiJUNZG5N/AiQw==", + "version": "3.1.6", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.6.tgz", + "integrity": "sha512-7Ical1vFEMr0onbVzEDIreM22I4khW+fzyQPwvAFWBp1iwdshSZRsL4jjRvPG9JP1uiqMHRto+YU6R2/CzDz5Q==", "funding": [ { "type": "github", @@ -4370,9 +4368,9 @@ "license": "MIT" }, "node_modules/js-yaml": { - "version": "3.15.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.15.0.tgz", - "integrity": "sha512-ttBQIIQPDeLjpPOohtUdXuXUVoA2uIB6fEH9HyJ7234s5mBJ5wTx20njxplLZQgLaOfpmPQA7X2t5AX6tIPbog==", + "version": "3.15.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.15.2.tgz", + "integrity": "sha512-6EuL879VkRA+1Cz578mKMiKvjPNEuk6+r1JaFzoSWejZmtf7xWbIyw1e3KkxlkzTIt9Taw6JBhEppG7utc1P+w==", "dev": true, "license": "MIT", "dependencies": { @@ -4597,9 +4595,9 @@ } }, "node_modules/mime-db": { - "version": "1.52.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", - "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", "license": "MIT", "engines": { "node": ">= 0.6" @@ -4617,6 +4615,15 @@ "node": ">= 0.6" } }, + "node_modules/mime-types/node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, "node_modules/mimic-fn": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", @@ -4656,9 +4663,9 @@ "license": "MIT" }, "node_modules/nanoid": { - "version": "3.3.16", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.16.tgz", - "integrity": "sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==", + "version": "3.3.18", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.18.tgz", + "integrity": "sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==", "funding": [ { "type": "github", @@ -4681,9 +4688,9 @@ "license": "MIT" }, "node_modules/negotiator": { - "version": "0.6.3", - "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", - "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "version": "0.6.4", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.4.tgz", + "integrity": "sha512-myRT3DiWPHqho5PrJaIRyaMv2kgYf0mUVgBNOYMuCH5Ki1yEiQaf/ZJuQ62nvpc44wL5WDbTX7yGJi1Neevw8w==", "license": "MIT", "engines": { "node": ">= 0.6" @@ -4706,9 +4713,9 @@ "license": "MIT" }, "node_modules/node-releases": { - "version": "2.0.51", - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.51.tgz", - "integrity": "sha512-wRNIrw4DmVLKQlbgOMdkMx27Wrpzes2hh5Jtbi2bjPd+4wJstWIqP5A+lscnqbm0xxmT5Bpg8Lec5ItEBwx6BQ==", + "version": "2.0.54", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.54.tgz", + "integrity": "sha512-YHs7BmmcsdAI5Ozuf8JZo6PT0mv2GIWC9vMfvUC3dp65M8hn7Ux8CPL+2oBI7juNuj9d0ndhTcznq2ODBps9cQ==", "dev": true, "license": "MIT", "engines": { @@ -5096,9 +5103,9 @@ } }, "node_modules/process-warning": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-5.0.0.tgz", - "integrity": "sha512-a39t9ApHNx2L4+HBnQKqxxHNs1r7KF+Intd8Q/g1bUh6q0WIp9voPXJ/x0j+ZL45KF1pJd9+q2jLIRMfvEshkA==", + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-5.1.0.tgz", + "integrity": "sha512-jQSaVHsPgtyw60e1rQ/A+/ArPEj/S8pS/vFnyGa/gYFXrKk/6RuDkoqVDQ5NI5MmS01698ltlAk0NoDBNLujRw==", "funding": [ { "type": "github", @@ -5865,15 +5872,15 @@ } }, "node_modules/swagger-jsdoc/node_modules/brace-expansion": { - "version": "5.0.7", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.7.tgz", - "integrity": "sha512-7oFy703dxfY3/NLxC1fh2SUCQ0H9rmAY+5EpDVfXjUTTs+HEwR2nYaqLv+GWcTsumwxPfiz6CzCNkwXwBUwqCA==", + "version": "5.0.9", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz", + "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==", "license": "MIT", "dependencies": { "balanced-match": "^4.0.2" }, "engines": { - "node": "18 || 20 || >=22" + "node": "20 || >=22" } }, "node_modules/swagger-jsdoc/node_modules/glob": { @@ -5901,12 +5908,12 @@ } }, "node_modules/swagger-jsdoc/node_modules/minimatch": { - "version": "10.2.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", - "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", + "version": "10.2.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.6.tgz", + "integrity": "sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==", "license": "BlueOak-1.0.0", "dependencies": { - "brace-expansion": "^5.0.5" + "brace-expansion": "^5.0.8" }, "engines": { "node": "18 || 20 || >=22" @@ -5916,9 +5923,9 @@ } }, "node_modules/swagger-ui-dist": { - "version": "5.32.9", - "resolved": "https://registry.npmjs.org/swagger-ui-dist/-/swagger-ui-dist-5.32.9.tgz", - "integrity": "sha512-8i2tzJQi+7bgxESMD2hg/UBumbTsf6vLbtu4cW5ETPz/B070UuS0rTP1hu6WSH81HcsHqYalJE+rP21Vg96rUQ==", + "version": "5.32.14", + "resolved": "https://registry.npmjs.org/swagger-ui-dist/-/swagger-ui-dist-5.32.14.tgz", + "integrity": "sha512-nOA2pSQhcmODMUQZpJHYKNuwniDUqcOWGNaSCOoZv12FdOSJ9JxV95HtyRGNMqEBj6h6lCNTy20TgZDYTSuUIg==", "license": "Apache-2.0", "dependencies": { "@scarf/scarf": "=1.4.0" @@ -6184,9 +6191,9 @@ } }, "node_modules/typescript": { - "version": "5.4.5", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.4.5.tgz", - "integrity": "sha512-vcI4UpRgg81oIRUFwR0WSIHKt11nJ7SAVlYNIu+QpqeyXP+gpQJy/Z4+F0aGxSE4MqwjyXvW/TzgkLAx2AGHwQ==", + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "dev": true, "license": "Apache-2.0", "peer": true, @@ -6215,9 +6222,9 @@ } }, "node_modules/update-browserslist-db": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.2.3.tgz", - "integrity": "sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w==", + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.3.2.tgz", + "integrity": "sha512-UQ+MSxlhRm1bzjhU+DcuXfjFO1FzNtqhK5+9Yvlp90ItDLk5vT932A0rFu619nf7RVS+Y/VeaUW1jaRDqZ8VJw==", "dev": true, "funding": [ { @@ -6377,9 +6384,9 @@ } }, "node_modules/ws": { - "version": "8.21.1", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.1.tgz", - "integrity": "sha512-+0NTnW77fFN/DjQi6k/Sq/Yvk4Sgajw7urW8V+asjXnRgDs9gyGkdb7EzgfhA4goXsRIZKE28fzIXBHEzhuiWw==", + "version": "8.21.3", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.3.tgz", + "integrity": "sha512-201TZ/kPWxoPr/OKWjquZR1SWKXcvxdH+e1xrx89b3YbmzLMFCLfnaG1HFIgWzJOEWZ7MvpK++odZufgYR50Rw==", "license": "MIT", "engines": { "node": ">=10.0.0" diff --git a/backend/package.json b/backend/package.json index 076b6798..ac5dd496 100644 --- a/backend/package.json +++ b/backend/package.json @@ -1,44 +1,13 @@ { "name": "ai-net-backend", - "version": "0.1.0", - "description": "REST + WebSocket backend for ai-net — bridges frontend, agent runtime, Stellar payments, and Venice AI.", - "main": "dist/api/app.js", - "scripts": { - "build": "tsc", - "dev": "ts-node src/api/app.ts", - "test": "jest --runInBand --forceExit", - "test:watch": "jest --watch", - "lint": "eslint src --ext .ts" - }, - "license": "MIT", - "dependencies": { - "express": "4.19.2", - "pino": "9.2.0", - "pino-http": "10.2.0", - "zod": "3.23.8", - "lru-cache": "10.4.3" - }, - "devDependencies": { - "@types/express": "4.17.21", - "@types/jest": "29.5.12", - "@types/node": "20.14.2", - "@types/supertest": "6.0.2", - "@typescript-eslint/eslint-plugin": "7.13.0", - "@typescript-eslint/parser": "7.13.0", - "eslint": "8.57.0", - "jest": "29.7.0", - "supertest": "7.0.0", - "ts-jest": "29.2.2", - "ts-node": "10.9.2", - "typescript": "5.4.5" "private": true, "version": "0.1.0", - "description": "ai-net backend — Node.js/TypeScript server", + "description": "ai-net backend — Node.js/TypeScript server bridging frontend, agent runtime, Stellar payments, and Venice AI.", "main": "dist/index.js", "scripts": { "build": "tsc -p tsconfig.json", "dev": "ts-node src/index.ts", - "test": "jest --testPathPattern=tests --runInBand --forceExit", + "test": "jest --runInBand --forceExit", "test:coverage": "jest --coverage --runInBand --forceExit", "test:e2e": "jest --config ../tests/e2e/jest.config.js --runInBand --forceExit" }, diff --git a/backend/src/api/app.ts b/backend/src/api/app.ts index 7baf55f0..f2a48623 100644 --- a/backend/src/api/app.ts +++ b/backend/src/api/app.ts @@ -1,98 +1,12 @@ /** * Express application factory. * - * Called by tests (pass port=0 for random) and by the server entry-point. - * Wires up: - * - JSON body parsing - * - Pino HTTP request logging - * - Cache initialisation - * - Route mounting (health, stats, agents) - * - Global error handler + * Wires up middleware, routes, the WebSocket task stream, background + * services (job queue/worker, heartbeat cleanup, metrics), and the global + * error handler. Called by tests (`opts.disableCompression`, custom + * dispatch/queue, etc.) and by the server entry-point (`src/index.ts`). */ -import express, { Request, Response, NextFunction } from 'express'; -import pinoHttp from 'pino-http'; -import { config } from '../config/index'; -import { initCache } from '../cache/index'; -import { logger } from './logger'; -import healthRouter from './routes/health'; -import statsRouter from './routes/stats'; -import agentsRouter from './routes/agents'; - -export function createApp() { - // Initialise cache once (idempotent — subsequent calls return the same client) - try { - initCache({ - driver: config.CACHE_DRIVER, - redisUrl: config.REDIS_URL, - lruMaxSize: config.CACHE_LRU_MAX_SIZE, - defaultTtlSeconds: Math.max( - config.CACHE_TTL_AGENTS, - config.CACHE_TTL_STATS, - config.CACHE_TTL_HEALTH, - ), - }); - } catch { - // Already initialised (e.g. during testing) — ignore - } - - const app = express(); - - // ── Middleware stack ────────────────────────────────────────────────────── - - app.use(express.json()); - - if (config.NODE_ENV !== 'test') { - app.use(pinoHttp({ logger })); - } - - // ── Routes ──────────────────────────────────────────────────────────────── - - app.use('/api/health', healthRouter); - app.use('/api/stats', statsRouter); - app.use('/api/agents', agentsRouter); - - // ── 404 catch-all ───────────────────────────────────────────────────────── - - app.use((_req: Request, res: Response) => { - res.status(404).json({ error: { message: 'Not found', code: 'NOT_FOUND' } }); - }); - - // ── Global error handler ────────────────────────────────────────────────── - - // eslint-disable-next-line @typescript-eslint/no-unused-vars - app.use((err: Error, _req: Request, res: Response, _next: NextFunction) => { - logger.error({ err }, 'Unhandled error'); - res.status(500).json({ - error: { message: err.message ?? 'Internal server error', code: 'INTERNAL_ERROR' }, - }); - }); - - return app; -} - -// ── Entry point ─────────────────────────────────────────────────────────────── - -if (require.main === module) { - const app = createApp(); - const server = app.listen(config.PORT, () => { - logger.info({ port: config.PORT, env: config.NODE_ENV }, 'ai-net backend started'); - }); - - const shutdown = () => { - logger.info('Received shutdown signal — draining connections…'); - server.close(() => { - logger.info('Server closed'); - process.exit(0); - }); - setTimeout(() => { - logger.error('Graceful shutdown timed out — forcing exit'); - process.exit(1); - }, 10_000); - }; - - process.on('SIGTERM', shutdown); - process.on('SIGINT', shutdown); import express, { Request, Response, NextFunction } from "express"; import { createServer, Server as HttpServer } from "http"; import { randomUUID } from "crypto"; @@ -151,6 +65,7 @@ import { type JobQueue, } from "../queue"; import { createAdminQueueRouter } from "./routes/admin"; +import { metricsService, metricsMiddleware } from "../services/metrics"; export interface AppOptions { /** Called to execute a single DAG node; defaults to HTTP dispatch via agent registry */ @@ -187,6 +102,14 @@ export interface AppOptions { jobWorker?: JobWorker; /** Enable background queue worker (default: true) */ enableQueueWorker?: boolean; + /** + * How long close() waits for in-flight jobs to finish before closing the + * HTTP/WS server anyway. Default: 10000 (10s). A job still running when + * this elapses is left in the queue's "active" state — the next worker + * start (see JobWorker.start()/recoverIncompleteJobs()) resets it to + * "pending" and retries it, rather than losing the work. + */ + jobWorkerStopTimeoutMs?: number; } /** @@ -345,15 +268,21 @@ export function createApp(opts: AppOptions = {}): { app.use(errorHandler); function close(callback?: () => void): void { - jobWorker.stop(); - heartbeatService.stop(); - metricsService.setWebSocketProbe(null); - detachStream(); - if (httpServer.listening) { - httpServer.close(callback); - } else if (callback) { - callback(); - } + // Drain first: wait for in-flight jobs to finish (bounded by + // jobWorkerStopTimeoutMs) before we stop accepting connections. A job + // still active when the drain window elapses is NOT force-failed — it + // stays "active" in the store and is picked back up by the next + // JobWorker.start() via recoverIncompleteJobs(). + jobWorker.stop(opts.jobWorkerStopTimeoutMs ?? 10_000).finally(() => { + heartbeatService.stop(); + metricsService.setWebSocketProbe(null); + detachStream(); + if (httpServer.listening) { + httpServer.close(callback); + } else if (callback) { + callback(); + } + }); } return { httpServer, close }; diff --git a/backend/src/api/routes/agents.ts b/backend/src/api/routes/agents.ts index 7aa6d201..5b8e8392 100644 --- a/backend/src/api/routes/agents.ts +++ b/backend/src/api/routes/agents.ts @@ -1,131 +1,3 @@ -/** - * Agent registry API routes. - * - * GET /api/agents — list agents (cached, CACHE_TTL_AGENTS) - * GET /api/agents/:id — get single agent (cached, CACHE_TTL_AGENTS) - * POST /api/agents/register — register agent → INVALIDATES agents + stats cache - * DELETE /api/agents/:id — deregister agent → INVALIDATES agents + stats cache - * - * Full implementation tracked in Issue #24. The routes are scaffolded here so - * cache middleware and invalidation are fully exercised. - */ - -import { Router, Request, Response } from 'express'; -import { ttlForRoute } from '../../config/index'; -import { cacheMiddleware } from '../middleware/cache'; -import { invalidateOnAgentRegistration } from '../../cache/invalidation'; - -const router = Router(); - -// In-memory stub store until Issue #24 wires up the DB -const agentStore = new Map(); - -export interface AgentRecord { - id: string; - name: string; - capabilities: string[]; - pricingXLM: number; - endpoint: string; - stellarPublicKey: string; - reputationScore: number; - lastSeenAt: string; -} - -// ── GET /api/agents ────────────────────────────────────────────────────────── - -router.get( - '/', - cacheMiddleware({ ttl: ttlForRoute('agents') }), - (req: Request, res: Response) => { - let agents = Array.from(agentStore.values()); - - // Optional filters - if (req.query['capability']) { - agents = agents.filter((a) => - a.capabilities.includes(req.query['capability'] as string), - ); - } - if (req.query['minReputation']) { - const min = parseFloat(req.query['minReputation'] as string); - agents = agents.filter((a) => a.reputationScore >= min); - } - if (req.query['maxPriceXLM']) { - const max = parseFloat(req.query['maxPriceXLM'] as string); - agents = agents.filter((a) => a.pricingXLM <= max); - } - - res.json(agents); - }, -); - -// ── GET /api/agents/:id ────────────────────────────────────────────────────── - -router.get( - '/:id', - cacheMiddleware({ ttl: ttlForRoute('agents') }), - (req: Request, res: Response) => { - const agent = agentStore.get(req.params['id']!); - if (!agent) { - res.status(404).json({ error: { message: 'Agent not found', code: 'AGENT_NOT_FOUND' } }); - return; - } - res.json(agent); - }, -); - -// ── POST /api/agents/register ───────────────────────────────────────────────── -// Must be before /:id to avoid matching 'register' as an id - -router.post('/register', async (req: Request, res: Response) => { - const { agentId, capabilities, pricingXLM, endpoint, stellarPublicKey } = req.body as { - agentId: string; - capabilities: string[]; - pricingXLM: number; - endpoint: string; - stellarPublicKey: string; - }; - - if (!agentId || !capabilities?.length || !stellarPublicKey) { - res.status(400).json({ - error: { message: 'agentId, capabilities, and stellarPublicKey are required', code: 'INVALID_BODY' }, - }); - return; - } - - const record: AgentRecord = { - id: agentId, - name: agentId, - capabilities, - pricingXLM: pricingXLM ?? 1, - endpoint: endpoint ?? '', - stellarPublicKey, - reputationScore: 1, - lastSeenAt: new Date().toISOString(), - }; - agentStore.set(agentId, record); - - // Invalidate cached agent list and stats - await invalidateOnAgentRegistration(); - - res.status(201).json({ registered: true, agent: record }); -}); - -// ── DELETE /api/agents/:id ──────────────────────────────────────────────────── - -router.delete('/:id', async (req: Request, res: Response) => { - const id = req.params['id']!; - if (!agentStore.has(id)) { - res.status(404).json({ error: { message: 'Agent not found', code: 'AGENT_NOT_FOUND' } }); - return; - } - - agentStore.delete(id); - await invalidateOnAgentRegistration(); - - res.status(204).send(); -}); - -export default router; import { Router, Request, Response, NextFunction } from "express"; import { z } from "zod"; import { Horizon, Keypair } from "@stellar/stellar-sdk"; @@ -142,6 +14,15 @@ const DEFAULT_HEALTH_TIMEOUT_MS = 3_000; const HORIZON_URL = process.env.STELLAR_HORIZON_URL || "https://horizon-testnet.stellar.org"; const horizon = new Horizon.Server(HORIZON_URL); +// Mirrors the RegisterAgentRequest schema documented in api/docs.ts. +const RegisterAgentSchema = z.object({ + agentId: z.string().min(1), + capabilities: z.array(z.string()).min(1), + pricingXLM: z.number().min(0.001), + endpoint: z.string().url(), + stellarPublicKey: z.string().regex(/^G[A-Z2-7]{55}$/), +}); + export function createAgentsRouter(options: AgentsRouterOptions = {}): Router { const router = Router(); const healthTimeoutMs = options.healthTimeoutMs ?? DEFAULT_HEALTH_TIMEOUT_MS; diff --git a/backend/src/api/routes/health.ts b/backend/src/api/routes/health.ts index eb63455a..59fa671b 100644 --- a/backend/src/api/routes/health.ts +++ b/backend/src/api/routes/health.ts @@ -1,82 +1,3 @@ -/** - * GET /api/health — shallow health check - * GET /api/health/deep — checks Venice + Stellar Horizon reachability - * - * Cache TTL: CACHE_TTL_HEALTH (default 10s) - */ - -import { Router } from 'express'; -import { config, ttlForRoute } from '../../config/index'; -import { cacheMiddleware } from '../middleware/cache'; - -const router = Router(); - -const startTime = Date.now(); - -// GET /api/health -router.get( - '/', - cacheMiddleware({ ttl: ttlForRoute('health') }), - (_req, res) => { - res.json({ - status: 'ok', - uptime: Math.floor((Date.now() - startTime) / 1000), - version: process.env['npm_package_version'] ?? '0.1.0', - stellarNetwork: config.STELLAR_NETWORK, - }); - }, -); - -// GET /api/health/deep -router.get( - '/deep', - cacheMiddleware({ ttl: ttlForRoute('health') }), - async (_req, res) => { - const [veniceStatus, horizonStatus] = await Promise.all([ - checkVenice(), - checkHorizon(), - ]); - - const allOk = veniceStatus === 'ok' && horizonStatus === 'ok'; - res.status(allOk ? 200 : 503).json({ - status: allOk ? 'ok' : 'degraded', - services: { - venice: veniceStatus, - horizon: horizonStatus, - }, - }); - }, -); - -async function checkVenice(): Promise<'ok' | 'unreachable'> { - try { - const controller = new AbortController(); - const timer = setTimeout(() => controller.abort(), 5_000); - const url = 'https://api.venice.ai/api/v1/models'; - const resp = await fetch(url, { - signal: controller.signal, - headers: { Authorization: `Bearer ${config.VENICE_API_KEY}` }, - }); - clearTimeout(timer); - return resp.ok || resp.status === 401 ? 'ok' : 'unreachable'; - } catch { - return 'unreachable'; - } -} - -async function checkHorizon(): Promise<'ok' | 'unreachable'> { - try { - const controller = new AbortController(); - const timer = setTimeout(() => controller.abort(), 5_000); - const resp = await fetch(config.STELLAR_HORIZON_URL, { signal: controller.signal }); - clearTimeout(timer); - return resp.ok ? 'ok' : 'unreachable'; - } catch { - return 'unreachable'; - } -} - -export default router; import { Router, Request, Response } from "express"; import { getConfig } from "../../config"; import { adminAuthMiddleware } from "../middleware/auth"; diff --git a/backend/src/api/routes/stats.ts b/backend/src/api/routes/stats.ts index 4e843e40..c875f7f5 100644 --- a/backend/src/api/routes/stats.ts +++ b/backend/src/api/routes/stats.ts @@ -1,73 +1,3 @@ -/** - * GET /api/stats — network statistics - * - * Returns aggregated KPIs for the frontend dashboard. - * Cache TTL: CACHE_TTL_STATS (default 30s) - * - * The underlying stats computation is intentionally stubbed here; the full - * DB-backed implementation is tracked in Issue #29. The route is fully wired - * so the cache middleware exercises the real cache path. - */ - -import { Router, Request, Response } from 'express'; -import { ttlForRoute } from '../../config/index'; -import { cacheMiddleware } from '../middleware/cache'; - -const router = Router(); - -// GET /api/stats -router.get( - '/', - cacheMiddleware({ ttl: ttlForRoute('stats') }), - async (_req: Request, res: Response) => { - // TODO (Issue #29): replace with real DB aggregation via StatsService - const stats = await computeStats(); - res.json(stats); - }, -); - -// --------------------------------------------------------------------------- -// Stats computation (stub — real implementation in Issue #29) -// --------------------------------------------------------------------------- - -export interface TimePoint { - timestamp: string; // ISO-8601 - value: number; -} - -export interface StatsPayload { - totalAgents: number; - totalTasks: number; - totalXLMTransacted: string; // stringified for precision - uptimePercent: number; - tasksLast24h: TimePoint[]; - xlmLast24h: TimePoint[]; -} - -async function computeStats(): Promise { - // Build 24 hourly time-points ending at the current hour - const now = new Date(); - const tasksLast24h: TimePoint[] = []; - const xlmLast24h: TimePoint[] = []; - - for (let i = 23; i >= 0; i--) { - const ts = new Date(now); - ts.setHours(ts.getHours() - i, 0, 0, 0); - tasksLast24h.push({ timestamp: ts.toISOString(), value: 0 }); - xlmLast24h.push({ timestamp: ts.toISOString(), value: 0 }); - } - - return { - totalAgents: 0, - totalTasks: 0, - totalXLMTransacted: '0.0000000', - uptimePercent: 100, - tasksLast24h, - xlmLast24h, - }; -} - -export default router; import { Router } from 'express'; import { getStats, type DbClient } from '../../db/stats'; import { StatsCache } from '../../utils/statsCache'; diff --git a/backend/src/api/routes/stream.ts b/backend/src/api/routes/stream.ts index 443295de..0a0b6bce 100644 --- a/backend/src/api/routes/stream.ts +++ b/backend/src/api/routes/stream.ts @@ -14,6 +14,22 @@ const STREAM_PATH = /^\/tasks\/([^/?]+)\/stream(?:\?.*)?$/; const logger = createLogger({ module: 'ws-stream' }); +/** + * Every WebSocketServer created by attachTaskStream(), tracked so + * getStreamConnectionCount() can report a live total across all of them + * (normally just one, per HTTP server) for the health/metrics dashboard. + */ +const activeStreamServers = new Set(); + +/** Total connected WebSocket clients across every attached stream server. */ +export function getStreamConnectionCount(): number { + let count = 0; + for (const wss of activeStreamServers) { + count += wss.clients.size; + } + return count; +} + // --------------------------------------------------------------------------- // Wire-format normalisation // --------------------------------------------------------------------------- diff --git a/backend/src/config/index.ts b/backend/src/config/index.ts index 30e0b3c6..c0f095d9 100644 --- a/backend/src/config/index.ts +++ b/backend/src/config/index.ts @@ -2,81 +2,9 @@ * Configuration module — loads and validates all env vars at startup. * Every other module imports from here; direct process.env access is banned. * - * Fails fast (throws) if any required var is missing or malformed. + * Fails fast (exits) if any required var is missing or malformed. */ -import { z } from 'zod'; - -// --------------------------------------------------------------------------- -// Schema -// --------------------------------------------------------------------------- - -const envSchema = z.object({ - // Server - PORT: z.coerce.number().int().positive().default(3001), - NODE_ENV: z.enum(['development', 'test', 'production']).default('development'), - - // Stellar - STELLAR_NETWORK: z.enum(['testnet', 'mainnet']).default('testnet'), - STELLAR_HORIZON_URL: z - .string() - .url() - .default('https://horizon-testnet.stellar.org'), - - // Venice AI - VENICE_API_KEY: z.string().min(1, 'VENICE_API_KEY is required'), - - // Database - DATABASE_URL: z.string().min(1).default('./data/ai-net.db'), - - // Cache - CACHE_DRIVER: z.enum(['lru', 'redis']).default('lru'), - REDIS_URL: z.string().default('redis://localhost:6379'), - CACHE_LRU_MAX_SIZE: z.coerce.number().int().positive().default(500), - - // Per-endpoint TTLs (seconds) - CACHE_TTL_AGENTS: z.coerce.number().int().nonnegative().default(60), - CACHE_TTL_STATS: z.coerce.number().int().nonnegative().default(30), - CACHE_TTL_HEALTH: z.coerce.number().int().nonnegative().default(10), -}); - -// --------------------------------------------------------------------------- -// Parse — throws ZodError on missing/invalid vars -// --------------------------------------------------------------------------- - -function loadConfig() { - const result = envSchema.safeParse(process.env); - - if (!result.success) { - const messages = result.error.errors - .map((e) => ` ${e.path.join('.')}: ${e.message}`) - .join('\n'); - throw new Error(`[config] Invalid environment variables:\n${messages}`); - } - - return result.data; -} - -// Singleton — evaluated once at import time -export const config = loadConfig(); - -// --------------------------------------------------------------------------- -// Convenience helpers -// --------------------------------------------------------------------------- - -/** TTL in seconds for a given route group */ -export function ttlForRoute(group: 'agents' | 'stats' | 'health'): number { - switch (group) { - case 'agents': - return config.CACHE_TTL_AGENTS; - case 'stats': - return config.CACHE_TTL_STATS; - case 'health': - return config.CACHE_TTL_HEALTH; - } -} - -export type Config = typeof config; import { z } from "zod"; // eslint-disable-next-line @typescript-eslint/no-var-requires @@ -174,8 +102,11 @@ const envSchema = z.object({ METRICS_WINDOW_MS: z.coerce.number().int().positive().default(60_000), /** Maximum request samples retained in memory. Default: 1 000. */ METRICS_MAX_SAMPLES: z.coerce.number().int().positive().default(1_000), -}); + // ── Health probes ─────────────────────────────────────────────────────────── + /** Timeout in ms for each external dependency check performed by GET /health/deep and GET /health/ready. Default: 5 000 (5 s). */ + HEALTH_PROBE_TIMEOUT_MS: z.coerce.number().int().positive().default(5_000), +}); let _config: z.infer | null = null; diff --git a/backend/src/index.ts b/backend/src/index.ts index c914e124..954861f4 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -9,10 +9,11 @@ import { initializeAgents, globalAgentRegistry } from "./agents"; import { startAgentSync, stopAgentSync } from "./registry/sync"; import { loadConfig, getConfig } from "./config"; import { AgentCleanupService } from "./services/agentCleanup"; -import { createTaskDb, getTaskDb, closeTaskDb } from "./db/tasks"; import { createAgentDb, getAgentDb, closeAgentDb } from "./db/agents"; import { closeDb } from "./db/index"; +import { closeTaskDb } from "./db/tasks"; import { closeJobDb } from "./queue"; +import { eventBus } from "./coordinator/eventBus"; import { createDefaultReconciliationService } from "./services/reconciliation"; async function main() { @@ -39,7 +40,9 @@ async function main() { reconciliationService.startDaily(config.RECONCILIATION_INTERVAL_MS); // Create and start the server - const { httpServer, close } = createApp(); + const { httpServer, close } = createApp({ + jobWorkerStopTimeoutMs: config.GRACEFUL_SHUTDOWN_TIMEOUT * 1000, + }); const port = config.PORT; @@ -47,6 +50,8 @@ async function main() { console.log(`[ai-net-backend] Server running on http://localhost:${port}`); console.log("[ai-net-backend] Available endpoints:"); console.log(" - GET /health - Health check"); + console.log(" - GET /health/live - Liveness check"); + console.log(" - GET /health/ready - Readiness check (DB, queue, WS, providers)"); console.log(" - GET /health/deep - Deep health check"); console.log(" - POST /api/tasks - Submit new tasks"); console.log(" - GET /api/tasks/:id - Get task status"); @@ -60,38 +65,41 @@ async function main() { }); // ── Graceful shutdown ────────────────────────────────────────────────────── - const shutdown = (signal: string) => { - console.log(`[ai-net-backend] Received ${signal}, shutting down gracefully...`); - const timeout = setTimeout(() => { - console.error("[ai-net-backend] Forced shutdown after 10s timeout"); - process.exit(1); - }, 10_000); - - cleanupService.stop(); - reconciliationService.stop(); - globalAgentRegistry.shutdown(); - stopAgentSync(); - - httpServer.close(() => { - clearTimeout(timeout); - console.log("[ai-net-backend] Server closed."); - process.exit(0); - }); - }; - - process.on("SIGTERM", () => shutdown("SIGTERM")); - process.on("SIGINT", () => shutdown("SIGINT")); - + setupGracefulShutdown(httpServer, close, config, { + cleanupService, + reconciliationService, + globalAgentRegistry, + }); } catch (error) { console.error("[ai-net-backend] Failed to start server:", error); process.exit(1); } } +export interface GracefulShutdownExtras { + cleanupService?: { stop(): void }; + reconciliationService?: { stop(): void }; + globalAgentRegistry?: { shutdown(): void }; +} + +/** + * SIGTERM/SIGINT handler: stop accepting new work, drain in-flight jobs and + * the WebSocket stream, flush the event store, close every database + * connection, then exit 0 — or force-exit 1 if any of that takes longer + * than `config.GRACEFUL_SHUTDOWN_TIMEOUT` seconds. + * + * In-flight tasks are drained (via `closeApp`, which awaits the job + * worker's stop()) rather than force-failed: anything still running when + * the drain window elapses stays "active" in the job store and is resumed + * by the next `JobWorker.start()` (`recoverIncompleteJobs()` resets it to + * "pending" for retry) — see `docs/e2e-testing.md` and + * `tests/shutdown.test.ts` for the restart-mid-stream scenario. + */ export function setupGracefulShutdown( httpServer: any, closeApp: (callback?: () => void) => void, - config: { GRACEFUL_SHUTDOWN_TIMEOUT?: number } + config: { GRACEFUL_SHUTDOWN_TIMEOUT?: number }, + extras: GracefulShutdownExtras = {}, ) { let isShuttingDown = false; @@ -108,7 +116,7 @@ export function setupGracefulShutdown( }, timeoutDuration); try { - console.log("[ai-net-backend] Phase 1: Closing HTTP/WS server and stopping new connections..."); + console.log("[ai-net-backend] Phase 1: Draining in-flight jobs and closing HTTP/WS server..."); await new Promise((resolve) => { closeApp(() => { console.log("[ai-net-backend] HTTP/WS server successfully closed."); @@ -116,18 +124,13 @@ export function setupGracefulShutdown( }); }); - console.log("[ai-net-backend] Phase 2: Stopping agent sync service..."); + console.log("[ai-net-backend] Phase 2: Stopping background services..."); stopAgentSync(); + extras.cleanupService?.stop(); + extras.reconciliationService?.stop(); + extras.globalAgentRegistry?.shutdown(); - console.log("[ai-net-backend] Phase 3: Failing all running tasks..."); - try { - const taskDb = createTaskDb(getTaskDb()); - taskDb.failRunningTasks(); - } catch (err) { - console.error("[ai-net-backend] Failed to mark tasks as failed during shutdown:", err); - } - - console.log("[ai-net-backend] Phase 4: Marking all online agents as offline..."); + console.log("[ai-net-backend] Phase 3: Marking all online agents as offline..."); try { const agentDb = createAgentDb(getAgentDb()); agentDb.markAllOffline(); @@ -135,7 +138,12 @@ export function setupGracefulShutdown( console.error("[ai-net-backend] Failed to mark agents offline during shutdown:", err); } - console.log("[ai-net-backend] Phase 5: Closing database connections..."); + console.log("[ai-net-backend] Phase 4: Flushing event store and closing database connections..."); + try { + eventBus.store.close(); + } catch (err) { + console.error("[ai-net-backend] Failed to close event store during shutdown:", err); + } closeDb(); closeAgentDb(); closeTaskDb(); diff --git a/backend/src/queue/worker.test.ts b/backend/src/queue/worker.test.ts index 7194043f..3adc107b 100644 --- a/backend/src/queue/worker.test.ts +++ b/backend/src/queue/worker.test.ts @@ -368,6 +368,84 @@ describe("Background Job Queue & Worker", () => { }); }); + describe("Restart mid-stream (#349 — in-flight jobs resume, not fail)", () => { + it("a job still running when the worker stops is resumed and completed by a fresh worker instance", async () => { + // Simulates a graceful shutdown that catches a job mid-execution: the + // handler never resolves before stop() gives up waiting, so the job + // stays "active" in the store rather than being marked failed — exactly + // what api/app.ts's close() -> jobWorker.stop(timeoutMs) produces when + // the drain window elapses with work still outstanding. + let releaseHandler!: () => void; + let resolveHandlerStarted!: () => void; + const handlerStarted = new Promise((resolve) => { + resolveHandlerStarted = resolve; + }); + + const firstWorker = new JobWorker({ + jobStore: store, + handler: async () => { + resolveHandlerStarted(); + // Hang until explicitly released — outlives the worker's stop() + // timeout, so stop() returns while this job is still "active". + await new Promise((releaseResolve) => { + releaseHandler = releaseResolve; + }); + return { success: true }; + }, + pollIntervalMs: 20, + autoStart: false, + }); + + const queue = new JobQueue(store, firstWorker); + const job = queue.enqueue({ taskId: "task_mid_stream" }); + + firstWorker.start(); + await handlerStarted; + + // Job is now actively executing. "Shut down" with a short drain + // window — the handler is hung, so stop() times out waiting and + // returns with the job still active, exactly like a real deploy that + // catches a slow task. + await firstWorker.stop(50); + + const midShutdownState = store.findById(job.id); + expect(midShutdownState?.status).toBe("active"); + + // "Restart": a brand-new JobWorker over the SAME store (in a real + // process this is the same jobs.db file reopened) — its start() calls + // recoverIncompleteJobs(), which is what actually makes the job + // resumable rather than lost. + const secondWorker = new JobWorker({ + jobStore: store, + handler: async (_job, updateProgress) => { + updateProgress(100); + return { success: true, resumed: true }; + }, + pollIntervalMs: 20, + autoStart: false, + }); + + const completed = new Promise((resolve) => { + secondWorker.onJobCompleted = (completedJob) => { + if (completedJob.id === job.id) resolve(); + }; + }); + + secondWorker.start(); + await completed; + await secondWorker.stop(); + + const finalState = store.findById(job.id); + expect(finalState?.status).toBe("completed"); + expect(finalState?.progress).toBe(100); + expect(finalState?.completedAt).toBeDefined(); + + // Release the first handler's promise so it doesn't leak a dangling + // timer/microtask into later tests. + releaseHandler(); + }); + }); + describe("Queue Stats & Admin Operations", () => { it("reports accurate stats across pending, active, completed, failed and dead-letter", () => { const now = new Date().toISOString(); diff --git a/backend/tests/shutdown.test.ts b/backend/tests/shutdown.test.ts index 51e5f484..b387d57f 100644 --- a/backend/tests/shutdown.test.ts +++ b/backend/tests/shutdown.test.ts @@ -2,7 +2,9 @@ import { setupGracefulShutdown } from '../src/index'; import { stopAgentSync } from '../src/registry/sync'; import { closeDb } from '../src/db'; import { closeAgentDb, createAgentDb } from '../src/db/agents'; -import { closeTaskDb, createTaskDb } from '../src/db/tasks'; +import { closeTaskDb } from '../src/db/tasks'; +import { closeJobDb } from '../src/queue'; +import { eventBus } from '../src/coordinator/eventBus'; jest.mock('../src/registry/sync', () => ({ stopAgentSync: jest.fn(), @@ -25,34 +27,47 @@ jest.mock('../src/db/tasks', () => ({ createTaskDb: jest.fn(), })); +jest.mock('../src/queue', () => ({ + closeJobDb: jest.fn(), +})); + +jest.mock('../src/coordinator/eventBus', () => ({ + eventBus: { store: { close: jest.fn() } }, +})); + describe('setupGracefulShutdown', () => { let mockProcessExit: jest.SpyInstance; let mockProcessOn: jest.SpyInstance; let mockHttpServer: any; let mockCloseApp: jest.Mock; - let mockTaskDb: any; let mockAgentDb: any; + let extras: { + cleanupService: { stop: jest.Mock }; + reconciliationService: { stop: jest.Mock }; + globalAgentRegistry: { shutdown: jest.Mock }; + }; beforeEach(() => { jest.clearAllMocks(); mockProcessExit = jest.spyOn(process, 'exit').mockImplementation((() => {}) as any); mockProcessOn = jest.spyOn(process, 'on').mockImplementation(() => undefined as any); - + mockCloseApp = jest.fn((callback?: () => void) => { if (callback) callback(); }); mockHttpServer = {}; - mockTaskDb = { - failRunningTasks: jest.fn(), - }; - (createTaskDb as jest.Mock).mockReturnValue(mockTaskDb); - mockAgentDb = { markAllOffline: jest.fn(), }; (createAgentDb as jest.Mock).mockReturnValue(mockAgentDb); + + extras = { + cleanupService: { stop: jest.fn() }, + reconciliationService: { stop: jest.fn() }, + globalAgentRegistry: { shutdown: jest.fn() }, + }; }); afterEach(() => { @@ -67,32 +82,66 @@ describe('setupGracefulShutdown', () => { expect(mockProcessOn).toHaveBeenCalledWith('SIGINT', expect.any(Function)); }); - it('performs full multi-phase shutdown sequence on signal', async () => { - const shutdown = setupGracefulShutdown(mockHttpServer, mockCloseApp, { GRACEFUL_SHUTDOWN_TIMEOUT: 5 }); + it('performs the full multi-phase shutdown sequence on signal', async () => { + const shutdown = setupGracefulShutdown( + mockHttpServer, + mockCloseApp, + { GRACEFUL_SHUTDOWN_TIMEOUT: 5 }, + extras, + ); await shutdown('SIGTERM'); - // Phase 1: closeApp called and completes + // Phase 1: closeApp called and completes — this is where the job worker's + // own drain (awaited inside close()) happens, so in-flight jobs finish or + // are left "active" for the next worker start to recover, not failed here. expect(mockCloseApp).toHaveBeenCalled(); - // Phase 2: stopAgentSync called + // Phase 2: agent sync and the extra background services are stopped expect(stopAgentSync).toHaveBeenCalled(); + expect(extras.cleanupService.stop).toHaveBeenCalled(); + expect(extras.reconciliationService.stop).toHaveBeenCalled(); + expect(extras.globalAgentRegistry.shutdown).toHaveBeenCalled(); - // Phase 3: failRunningTasks called - expect(mockTaskDb.failRunningTasks).toHaveBeenCalled(); - - // Phase 4: markAllOffline called + // Phase 3: markAllOffline called expect(mockAgentDb.markAllOffline).toHaveBeenCalled(); - // Phase 5: DB connections closed + // Phase 4: event store flushed (closed) and every DB connection closed + expect(eventBus.store.close).toHaveBeenCalled(); expect(closeDb).toHaveBeenCalled(); expect(closeAgentDb).toHaveBeenCalled(); expect(closeTaskDb).toHaveBeenCalled(); + expect(closeJobDb).toHaveBeenCalled(); // Process exits with code 0 expect(mockProcessExit).toHaveBeenCalledWith(0); }); + it('does not force-fail running tasks — in-flight work is left for the job worker to resume', async () => { + // There is no failRunningTasks call anywhere in the shutdown sequence: + // resumability comes from closeApp() awaiting the job worker's drain + // (see api/app.ts's close()) and JobWorker.recoverIncompleteJobs() on + // the next start(), not from marking tasks failed here. + const { createTaskDb } = require('../src/db/tasks'); + const shutdown = setupGracefulShutdown( + mockHttpServer, + mockCloseApp, + { GRACEFUL_SHUTDOWN_TIMEOUT: 5 }, + extras, + ); + + await shutdown('SIGTERM'); + + expect(createTaskDb).not.toHaveBeenCalled(); + }); + + it('works without extras (backward compatible with the 3-argument call)', async () => { + const shutdown = setupGracefulShutdown(mockHttpServer, mockCloseApp, { GRACEFUL_SHUTDOWN_TIMEOUT: 5 }); + + await expect(shutdown('SIGTERM')).resolves.toBeUndefined(); + expect(mockProcessExit).toHaveBeenCalledWith(0); + }); + it('triggers forced exit on timeout if server drain hangs', async () => { jest.useFakeTimers(); @@ -102,7 +151,7 @@ describe('setupGracefulShutdown', () => { }); const shutdown = setupGracefulShutdown(mockHttpServer, mockCloseApp, { GRACEFUL_SHUTDOWN_TIMEOUT: 10 }); - + // Start shutdown shutdown('SIGINT'); diff --git a/backend/tsconfig.json b/backend/tsconfig.json index 63c8abd5..e1f8625d 100644 --- a/backend/tsconfig.json +++ b/backend/tsconfig.json @@ -13,20 +13,5 @@ "types": ["node", "jest"] }, "include": ["src"], - "module": "node16", - "moduleResolution": "node16", - "strict": true, - "esModuleInterop": true, - "forceConsistentCasingInFileNames": true, - "skipLibCheck": true, - "outDir": "dist", - "rootDir": ".", - "resolveJsonModule": true, - "sourceMap": true, - "typeRoots": ["./node_modules/@types"], - "types": ["jest", "node"], - "baseUrl": "." - }, - "include": ["src", "tests"], "exclude": ["node_modules", "dist"] }