From 8fc82cdeea473bef3535fb83b63554da6fd8740c Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Wed, 4 Mar 2026 15:30:17 +0100 Subject: [PATCH 01/10] [feat] unify Agent setup and Publisher async integration --- README.md | 10 + apps/agent/README.md | 5 + apps/agent/env.d.ts | 9 + apps/agent/package.json | 1 + apps/agent/src/server/helpers.ts | 4 + apps/agent/src/server/index.ts | 217 +++++----- apps/agent/src/server/scripts/setup.ts | 385 +++++++++++++++--- apps/agent/src/server/setupPublisher.ts | 119 ++++++ .../agent/tests/integration/dkg-publisher.env | 6 +- .../plugins/epcis-plugin.md | 8 +- package-lock.json | 1 + .../src/plugins/dkg-tools.ts | 219 ++++++++-- .../tests/dkg-tools.spec.ts | 94 ++++- packages/plugin-dkg-publisher/README.md | 14 +- ...G Node - Publisher.postman_collection.json | 6 +- .../scripts/check-queue-status.js | 11 +- .../scripts/clear-failed-jobs.js | 11 +- packages/plugin-dkg-publisher/src/config.ts | 144 +++++++ .../src/database/bootstrap.ts | 4 +- packages/plugin-dkg-publisher/src/index.ts | 162 ++------ .../plugin-dkg-publisher/src/provision.ts | 93 +++++ .../src/services/AssetService.ts | 5 +- .../src/services/DkgService.ts | 14 +- .../src/services/PublishingService.ts | 9 +- .../src/services/StorageService.ts | 9 +- 25 files changed, 1177 insertions(+), 383 deletions(-) create mode 100644 apps/agent/src/server/setupPublisher.ts create mode 100644 packages/plugin-dkg-publisher/src/config.ts create mode 100644 packages/plugin-dkg-publisher/src/provision.ts diff --git a/README.md b/README.md index 25a345fe..b2f9149e 100644 --- a/README.md +++ b/README.md @@ -71,8 +71,18 @@ The setup script will: - Prompt for required environment variables - Create `.env` and `.env.development.local` files - Set up the SQLite database with migrations +- Optionally enable async publishing and provision the Publisher database using the Engine-derived MySQL password - Create an admin user (username: `admin`, password: `admin123`) +If you enable async publishing during setup, the Agent writes the consolidated Publisher settings into `apps/agent/.env`: + +- `ASYNC_PUBLISHING_ENABLED=true` +- `MYSQL_PASSWORD` +- `DKGP_DATABASE_URL` +- `REDIS_URL` + +The Agent server only loads `@dkg/plugin-dkg-publisher` when `ASYNC_PUBLISHING_ENABLED=true`. + ### 4. Start Development ```bash diff --git a/apps/agent/README.md b/apps/agent/README.md index 9792c5ed..5536ad13 100644 --- a/apps/agent/README.md +++ b/apps/agent/README.md @@ -23,6 +23,11 @@ npm run script:setup ``` + The setup flow can also enable async publishing. When enabled, it reuses the + Engine MySQL password from `dkg-engine/current/.env`, writes the consolidated + Publisher settings into `apps/agent/.env`, and the Agent loads the Publisher + plugin only when `ASYNC_PUBLISHING_ENABLED=true`. + Now you can run the app in development mode using: ```bash diff --git a/apps/agent/env.d.ts b/apps/agent/env.d.ts index 4a4a9c1d..377522e3 100644 --- a/apps/agent/env.d.ts +++ b/apps/agent/env.d.ts @@ -14,6 +14,15 @@ declare global { DKG_PUBLISH_WALLET: string; DKG_BLOCKCHAIN: string; DKG_OTNODE_URL: string; + ASYNC_PUBLISHING_ENABLED?: string; + MYSQL_PASSWORD?: string; + DKGP_DATABASE_URL?: string; + REDIS_URL?: string; + WORKER_COUNT?: string; + POLL_FREQUENCY?: string; + STORAGE_TYPE?: string; + STORAGE_PATH?: string; + STORAGE_BASE_URL?: string; SMTP_HOST: string; SMTP_PORT: string; SMTP_USER: string; diff --git a/apps/agent/package.json b/apps/agent/package.json index 9563affb..cd932d92 100644 --- a/apps/agent/package.json +++ b/apps/agent/package.json @@ -32,6 +32,7 @@ "dependencies": { "@dkg/expo-forcegraph": "^0.0.0", "@dkg/plugin-dkg-essentials": "^0.0.3", + "@dkg/plugin-dkg-publisher": "^1.0.0", "@dkg/plugin-example": "^0.0.3", "@dkg/plugin-oauth": "^0.0.2", "@dkg/plugin-swagger": "^0.0.2", diff --git a/apps/agent/src/server/helpers.ts b/apps/agent/src/server/helpers.ts index 321a385f..9cca9a3a 100644 --- a/apps/agent/src/server/helpers.ts +++ b/apps/agent/src/server/helpers.ts @@ -34,6 +34,10 @@ export async function createFileWithContent(filePath: string, content: string) { } } +export async function writeFileWithContent(filePath: string, content: string) { + await fs.writeFile(filePath, content, { encoding: "utf8" }); +} + export function configEnv() { dotenv.config(); if (process.argv.includes("--dev")) { diff --git a/apps/agent/src/server/index.ts b/apps/agent/src/server/index.ts index f78f2ffc..550a9bda 100644 --- a/apps/agent/src/server/index.ts +++ b/apps/agent/src/server/index.ts @@ -5,105 +5,85 @@ import dkgEssentialsPlugin from "@dkg/plugin-dkg-essentials"; import createFsBlobStorage from "@dkg/plugin-dkg-essentials/createFsBlobStorage"; import examplePlugin from "@dkg/plugin-example"; import swaggerPlugin from "@dkg/plugin-swagger"; -//@ts-expect-error No types for dkg.js ... +// @ts-expect-error No types for dkg.js ... import DKG from "dkg.js"; import { eq } from "drizzle-orm"; +import { getTestMessageUrl } from "nodemailer"; import { userCredentialsSchema } from "@/shared/auth"; import { processStreamingCompletion } from "@/shared/chat"; import { verify } from "@node-rs/argon2"; -import { configDatabase, configEnv } from "./helpers"; -import webInterfacePlugin from "./webInterfacePlugin"; import createAccountManagementPlugin from "./accountManagementPlugin"; import { - users, - SqliteOAuthStorageProvider, SqliteAccountManagementProvider, + SqliteOAuthStorageProvider, + users, } from "./database/sqlite"; +import { configDatabase, configEnv } from "./helpers"; import mailer from "./mailer"; -import { getTestMessageUrl } from "nodemailer"; - -configEnv(); -const db = configDatabase(); - -const version = "1.0.0"; - -const { oauthPlugin, openapiSecurityScheme } = createOAuthPlugin({ - storage: new SqliteOAuthStorageProvider(db), - issuerUrl: new URL(process.env.EXPO_PUBLIC_MCP_URL), - scopesSupported: [ - "mcp", - "llm", - "scope123", - "blob", - "epcis.read", - "epcis.write", - ], - loginPageUrl: new URL(process.env.EXPO_PUBLIC_APP_URL + "/login"), - schema: userCredentialsSchema, - async login(credentials) { - const user = await db - .select() - .from(users) - .where(eq(users.email, credentials.email)) - .then((r) => r.at(0)); - if (!user) throw new Error("Invalid credentials"); +import webInterfacePlugin from "./webInterfacePlugin"; - const isValid = await verify(user.password, credentials.password); - if (!isValid) throw new Error("Invalid credentials"); +async function main() { + configEnv(); + const db = configDatabase(); + const version = "1.0.0"; - return { scopes: user.scope.split(" "), extra: { userId: user.id } }; - }, -}); + const { oauthPlugin, openapiSecurityScheme } = createOAuthPlugin({ + storage: new SqliteOAuthStorageProvider(db), + issuerUrl: new URL(process.env.EXPO_PUBLIC_MCP_URL), + scopesSupported: [ + "mcp", + "llm", + "scope123", + "blob", + "epcis.read", + "epcis.write", + ], + loginPageUrl: new URL(process.env.EXPO_PUBLIC_APP_URL + "/login"), + schema: userCredentialsSchema, + async login(credentials) { + const user = await db + .select() + .from(users) + .where(eq(users.email, credentials.email)) + .then((results) => results.at(0)); + if (!user) throw new Error("Invalid credentials"); -const accountManagementPlugin = createAccountManagementPlugin({ - provider: new SqliteAccountManagementProvider(db), - async sendMail(toEmail, code) { - const m = await mailer(); - if (!m) throw new Error("No SMTP transport available"); + const isValid = await verify(user.password, credentials.password); + if (!isValid) throw new Error("Invalid credentials"); - await m - .sendMail({ - to: toEmail, - subject: "Password reset request | DKG Node", - text: - `Your password reset code is ${code}.` + - `Link: ${process.env.EXPO_PUBLIC_APP_URL}/password-reset?code=${code}`, - html: - `

Your password reset code is ${code}.

` + - `

Please click here to reset your password.

`, - }) - .then((info) => { - console.debug(info); - console.debug(getTestMessageUrl(info)); - }); - }, -}); + return { scopes: user.scope.split(" "), extra: { userId: user.id } }; + }, + }); -const blobStorage = createFsBlobStorage(path.join(__dirname, "../data")); + const accountManagementPlugin = createAccountManagementPlugin({ + provider: new SqliteAccountManagementProvider(db), + async sendMail(toEmail, code) { + const transport = await mailer(); + if (!transport) throw new Error("No SMTP transport available"); -const otnodeUrl = new URL(process.env.DKG_OTNODE_URL); + await transport + .sendMail({ + to: toEmail, + subject: "Password reset request | DKG Node", + text: + `Your password reset code is ${code}.` + + `Link: ${process.env.EXPO_PUBLIC_APP_URL}/password-reset?code=${code}`, + html: + `

Your password reset code is ${code}.

` + + `

Please click here to reset your password.

`, + }) + .then((info) => { + console.debug(info); + console.debug(getTestMessageUrl(info)); + }); + }, + }); -const app = createPluginServer({ - name: "DKG API", - version, - context: { - blob: blobStorage, - dkg: new DKG({ - endpoint: `${otnodeUrl.protocol}//${otnodeUrl.hostname}`, - port: otnodeUrl.port || "8900", - blockchain: { - name: process.env.DKG_BLOCKCHAIN, - privateKey: process.env.DKG_PUBLISH_WALLET, - }, - maxNumberOfRetries: 300, - frequency: 2, - contentType: "all", - nodeApiVersion: "/v1", - }), - }, - plugins: [ + const blobStorage = createFsBlobStorage(path.join(__dirname, "../data")); + const otnodeUrl = new URL(process.env.DKG_OTNODE_URL); + const plugins = [ defaultPlugin, oauthPlugin, (_, __, api) => { @@ -119,7 +99,6 @@ const app = createPluginServer({ api.use("/change-password", authorized([])); api.use("/profile", authorized([])); }, - // Streaming LLM middleware โ€” intercepts SSE requests before Expo Router (_, __, api) => { api.post("/llm", (req, res, next) => { if (!req.headers.accept?.includes("text/event-stream")) return next(); @@ -128,8 +107,18 @@ const app = createPluginServer({ }, accountManagementPlugin, dkgEssentialsPlugin, + ]; + + if (process.env.ASYNC_PUBLISHING_ENABLED === "true") { + const { default: dkgPublisherPlugin } = await import( + "@dkg/plugin-dkg-publisher" + ); + plugins.push(dkgPublisherPlugin); + } + + plugins.push( examplePlugin.withNamespace("protected", { - middlewares: [authorized(["scope123"])], // Allow only users with the "scope123" scope + middlewares: [authorized(["scope123"])], }), swaggerPlugin({ version, @@ -145,28 +134,54 @@ const app = createPluginServer({ ], }), webInterfacePlugin(path.join(__dirname, "./app")), - ], -}); - -const port = process.env.PORT || 9200; -const server = app.listen(port, (err) => { - if (err) { - console.error(err); - process.exit(1); - } - console.log(`Server running at http://localhost:${port}/`); + ); - process.on("SIGINT", () => { - server.close(); - process.exit(0); + const app = createPluginServer({ + name: "DKG API", + version, + context: { + blob: blobStorage, + dkg: new DKG({ + endpoint: `${otnodeUrl.protocol}//${otnodeUrl.hostname}`, + port: otnodeUrl.port || "8900", + blockchain: { + name: process.env.DKG_BLOCKCHAIN, + privateKey: process.env.DKG_PUBLISH_WALLET, + }, + maxNumberOfRetries: 300, + frequency: 2, + contentType: "all", + nodeApiVersion: "/v1", + }), + }, + plugins, }); - process.on("SIGTERM", () => { - server.close((err) => { - if (err) { - console.error(err); - process.exit(1); - } + + const port = process.env.PORT || 9200; + const server = app.listen(port, (error) => { + if (error) { + console.error(error); + process.exit(1); + } + console.log(`Server running at http://localhost:${port}/`); + + process.on("SIGINT", () => { + server.close(); process.exit(0); }); + process.on("SIGTERM", () => { + server.close((closeError) => { + if (closeError) { + console.error(closeError); + process.exit(1); + } + process.exit(0); + }); + }); }); +} + +main().catch((error) => { + console.error(error); + process.exit(1); }); diff --git a/apps/agent/src/server/scripts/setup.ts b/apps/agent/src/server/scripts/setup.ts index 9a4fc915..fcb938e8 100644 --- a/apps/agent/src/server/scripts/setup.ts +++ b/apps/agent/src/server/scripts/setup.ts @@ -1,19 +1,65 @@ import path from "path"; import prompts from "prompts"; + +import { getLLMProviderApiKeyEnvName, LLMProvider } from "@/shared/chat"; +import { DEFAULT_SYSTEM_PROMPT } from "@/shared/prompts/defaultSystemPrompt"; + import { configDatabase, configEnv, createFileWithContent, createUser, + writeFileWithContent, } from "../helpers"; import { - getLLMProviderApiKeyEnvName, - LLMProvider, -} from "@/shared/chat"; -import { DEFAULT_SYSTEM_PROMPT } from "@/shared/prompts/defaultSystemPrompt"; + buildPublisherDefaults, + isValidMysqlIdentifier, + resolveEngineMysqlPassword, + provisionAsyncPublishing, +} from "../setupPublisher"; + +function formatEnvValue(value: string) { + return `"${value.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`; +} + +function isValidPrivateKey(value: string) { + return /^0x[0-9a-fA-F]{64}$/.test(value); +} async function setup() { - const r = await prompts([ + const enginePassword = await resolveEngineMysqlPassword(); + + if (enginePassword.status === "found") { + console.log( + `Detected MySQL password from ${enginePassword.envPath}. Async publishing can reuse it.`, + ); + } else if (enginePassword.status === "missing-file") { + console.log( + `Engine config not found at ${enginePassword.envPath}. Async publishing defaults will require advanced setup.`, + ); + } else { + console.log( + `Engine config found at ${enginePassword.envPath}, but REPOSITORY_PASSWORD is missing.`, + ); + } + + const asyncPublishingChoices = + enginePassword.status === "found" + ? [ + { + title: + "Yes (Recommended) - more seamless publishing and tracking", + value: "recommended", + }, + { title: "No", value: "disabled" }, + { title: "Yes, with advanced configuration", value: "advanced" }, + ] + : [ + { title: "No (Recommended)", value: "disabled" }, + { title: "Yes, with advanced configuration", value: "advanced" }, + ]; + + const response = await prompts([ { type: "select", name: "llmProvider", @@ -31,13 +77,13 @@ async function setup() { { type: "text", name: "llmModel", - message: (prev) => "Model name", - validate: (val) => val.length || "Model name is required", + message: "Model name", + validate: (value) => value.length || "Model name is required", }, { type: "number", name: "llmTemperature", - message: (prev) => "Temperature", + message: "Temperature", initial: 1, min: 0, max: 1, @@ -46,28 +92,29 @@ async function setup() { { type: "text", name: "llmSystemPrompt", - message: (prev) => "System prompt", + message: "System prompt", initial: DEFAULT_SYSTEM_PROMPT, - format: (val) => (val === DEFAULT_SYSTEM_PROMPT ? "" : val.trim()), + format: (value) => (value === DEFAULT_SYSTEM_PROMPT ? "" : value.trim()), }, { type: "select", name: "docConversionProvider", message: "Document conversion provider", choices: [ - { title: "unpdf โ€” basic PDF only", value: "unpdf" }, - { title: "Mistral OCR โ€” complex PDF/DOCX/PPTX", value: "mistral" }, + { title: "unpdf - basic PDF only", value: "unpdf" }, + { title: "Mistral OCR - complex PDF/DOCX/PPTX", value: "mistral" }, ], initial: 0, }, { - type: (_, a) => - a.docConversionProvider === "mistral" && a.llmProvider !== "mistralai" + type: (_, answers) => + answers.docConversionProvider === "mistral" && + answers.llmProvider !== "mistralai" ? "text" : null, name: "mistralApiKey", message: "MISTRAL_API_KEY", - validate: (val) => val.length || "Required for Mistral OCR provider", + validate: (value) => value.length || "Required for Mistral OCR provider", }, { type: "select", @@ -80,12 +127,13 @@ async function setup() { ], }, { - type: (_, a) => (a.dkgEnv === "development" ? "text" : "select"), + type: (_, answers) => (answers.dkgEnv === "development" ? "text" : "select"), name: "dkgBlockchain", message: "DKG blockchain", - initial: (_, a) => (a.dkgEnv === "development" ? "hardhat1:31337" : ""), - choices: (prev) => - prev === "mainnet" + initial: (_, answers) => + answers.dkgEnv === "development" ? "hardhat1:31337" : "", + choices: (previous) => + previous === "mainnet" ? [ { title: "NeuroWeb", value: "otp:2043" }, { title: "Base", value: "base:8453" }, @@ -101,11 +149,104 @@ async function setup() { type: "text", name: "dkgPublishWallet", message: "Publish wallet private key", - initial: (_, a) => - a.dkgEnv === "development" + initial: (_, answers) => + answers.dkgEnv === "development" ? "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" : "", - validate: (val) => val.length || "Required", + validate: (value) => { + if (!value.length) return "Required"; + return ( + isValidPrivateKey(value) || + "Private key must be a 0x-prefixed 32-byte hex string" + ); + }, + }, + { + type: "select", + name: "asyncPublishingMode", + message: + "Enable async publishing for more seamless publishing and tracking of published knowledge assets?", + choices: asyncPublishingChoices, + initial: 0, + }, + { + type: (_, answers) => + answers.asyncPublishingMode === "advanced" && + enginePassword.status !== "found" + ? "password" + : null, + name: "mysqlPassword", + message: "MYSQL_PASSWORD", + validate: (value) => value.length || "Required for async publishing", + }, + { + type: (_, answers) => + answers.asyncPublishingMode === "advanced" ? "text" : null, + name: "mysqlHost", + message: "Publisher MySQL host", + initial: "localhost", + }, + { + type: (_, answers) => + answers.asyncPublishingMode === "advanced" ? "number" : null, + name: "mysqlPort", + message: "Publisher MySQL port", + initial: 3306, + min: 0, + }, + { + type: (_, answers) => + answers.asyncPublishingMode === "advanced" ? "text" : null, + name: "mysqlUser", + message: "Publisher MySQL username", + initial: "root", + }, + { + type: (_, answers) => + answers.asyncPublishingMode === "advanced" ? "text" : null, + name: "mysqlDatabase", + message: "Publisher MySQL database name", + initial: "dkg_publisher_db", + validate: (value) => + isValidMysqlIdentifier(value) || + "Use letters, numbers, and underscores only", + }, + { + type: (_, answers) => + answers.asyncPublishingMode === "advanced" ? "text" : null, + name: "redisUrl", + message: "Publisher Redis URL", + initial: "redis://localhost:6379", + }, + { + type: (_, answers) => + answers.asyncPublishingMode === "advanced" ? "number" : null, + name: "workerCount", + message: "Publisher worker count", + initial: 1, + min: 1, + }, + { + type: (_, answers) => + answers.asyncPublishingMode === "advanced" ? "number" : null, + name: "pollFrequency", + message: "Publisher poll frequency (ms)", + initial: 2000, + min: 100, + }, + { + type: (_, answers) => + answers.asyncPublishingMode === "advanced" ? "text" : null, + name: "storagePath", + message: "Publisher storage path", + initial: "./data/publisher", + }, + { + type: (_, answers) => + answers.asyncPublishingMode === "advanced" ? "text" : null, + name: "storageBaseUrl", + message: "Publisher storage base URL", + initial: "http://localhost:9200/storage", }, { type: "confirm", @@ -114,36 +255,36 @@ async function setup() { initial: true, }, { - type: (_, a) => (a.smtpEnabled ? "text" : null), + type: (_, answers) => (answers.smtpEnabled ? "text" : null), name: "smtpHost", message: "SMTP Host", - validate: (val) => val.length || "Required", + validate: (value) => value.length || "Required", }, { - type: (_, a) => (a.smtpEnabled ? "number" : null), + type: (_, answers) => (answers.smtpEnabled ? "number" : null), name: "smtpPort", message: "SMTP Port", initial: 587, min: 0, }, { - type: (_, a) => (a.smtpEnabled ? "text" : null), + type: (_, answers) => (answers.smtpEnabled ? "text" : null), name: "smtpUsername", message: "SMTP Username", }, { - type: (_, a) => (a.smtpEnabled ? "password" : null), + type: (_, answers) => (answers.smtpEnabled ? "password" : null), name: "smtpPassword", message: "SMTP Password", }, { - type: (_, a) => (a.smtpEnabled ? "confirm" : null), + type: (_, answers) => (answers.smtpEnabled ? "confirm" : null), name: "smtpSecure", message: "SMTP Secure", initial: true, }, { - type: (_, a) => (a.smtpEnabled ? "text" : null), + type: (_, answers) => (answers.smtpEnabled ? "text" : null), name: "smtpFrom", message: "SMTP Sender email", initial: "noreply@example.com", @@ -152,67 +293,179 @@ async function setup() { type: "text", name: "dbFilename", message: "Database filename (e.g. example.db)", - validate: (val) => val.length || "Required", - format: (val) => (val.endsWith(".db") ? val : `${val}.db`), + validate: (value) => value.length || "Required", + format: (value) => (value.endsWith(".db") ? value : `${value}.db`), + }, + ], { + onCancel: () => { + throw new Error("Setup cancelled by user"); }, - ]); + }); - console.log("\nCreating .env file..."); - await createFileWithContent( + const requestedAsyncPublishing = response.asyncPublishingMode !== "disabled"; + const appUrl = "http://localhost:9200"; + const mysqlPassword = + enginePassword.mysqlPassword || response.mysqlPassword || ""; + const publisherDefaults = + requestedAsyncPublishing && mysqlPassword + ? buildPublisherDefaults(appUrl, mysqlPassword, { + mysqlHost: response.mysqlHost, + mysqlPort: response.mysqlPort, + mysqlUser: response.mysqlUser, + mysqlDatabase: response.mysqlDatabase, + redisUrl: response.redisUrl, + workerCount: response.workerCount, + pollFrequency: response.pollFrequency, + storagePath: response.storagePath, + storageBaseUrl: response.storageBaseUrl, + }) + : null; + + if (requestedAsyncPublishing && !publisherDefaults) { + throw new Error("Async publishing requires a MySQL password"); + } + + let publisherProvisionResult: + | { databaseCreated: boolean; walletInserted: boolean } + | null = null; + let publisherProvisionError: string | null = null; + + if (publisherDefaults) { + try { + publisherProvisionResult = await provisionAsyncPublishing( + publisherDefaults.databaseUrl, + { + privateKey: response.dkgPublishWallet, + blockchain: response.dkgBlockchain, + }, + ); + } catch (error: any) { + publisherProvisionError = error.message; + console.warn( + `Publisher provisioning warning: ${publisherProvisionError}`, + ); + } + } + + const asyncPublishingEnabled = + requestedAsyncPublishing && publisherProvisionError === null; + + const envLines = [ + "PORT=9200", + `EXPO_PUBLIC_MCP_URL=${formatEnvValue(appUrl)}`, + `EXPO_PUBLIC_APP_URL=${formatEnvValue(appUrl)}`, + `DATABASE_URL=${formatEnvValue(response.dbFilename)}`, + `LLM_PROVIDER=${formatEnvValue(response.llmProvider)}`, + `LLM_MODEL=${formatEnvValue(response.llmModel)}`, + `LLM_TEMPERATURE=${formatEnvValue(String(response.llmTemperature))}`, + `LLM_SYSTEM_PROMPT=${formatEnvValue(response.llmSystemPrompt)}`, + `${getLLMProviderApiKeyEnvName(response.llmProvider)}=${formatEnvValue(response.llmApiKey)}`, + `DKG_PUBLISH_WALLET=${formatEnvValue(response.dkgPublishWallet)}`, + `DKG_BLOCKCHAIN=${formatEnvValue(response.dkgBlockchain)}`, + 'DKG_OTNODE_URL="http://localhost:8900"', + `ASYNC_PUBLISHING_ENABLED=${asyncPublishingEnabled ? "true" : "false"}`, + `SMTP_HOST=${formatEnvValue(response.smtpHost || "")}`, + `SMTP_PORT=${formatEnvValue(String(response.smtpPort || ""))}`, + `SMTP_USER=${formatEnvValue(response.smtpUsername || "")}`, + `SMTP_PASS=${formatEnvValue(response.smtpPassword || "")}`, + `SMTP_SECURE=${response.smtpSecure === undefined ? "true" : response.smtpSecure}`, + `SMTP_FROM=${formatEnvValue(response.smtpFrom || "")}`, + `DOCUMENT_CONVERSION_PROVIDER=${formatEnvValue(response.docConversionProvider)}`, + ]; + + if ( + response.docConversionProvider === "mistral" && + response.llmProvider !== "mistralai" + ) { + envLines.push(`MISTRAL_API_KEY=${formatEnvValue(response.mistralApiKey)}`); + } + + if (publisherDefaults) { + envLines.push(`MYSQL_PASSWORD=${formatEnvValue(publisherDefaults.mysqlPassword)}`); + envLines.push(`DKGP_DATABASE_URL=${formatEnvValue(publisherDefaults.databaseUrl)}`); + envLines.push(`REDIS_URL=${formatEnvValue(publisherDefaults.redisUrl)}`); + envLines.push(`WORKER_COUNT=${formatEnvValue(String(publisherDefaults.workerCount))}`); + + if (response.asyncPublishingMode === "advanced") { + envLines.push( + `POLL_FREQUENCY=${formatEnvValue(String(publisherDefaults.pollFrequency))}`, + ); + envLines.push(`STORAGE_TYPE="filesystem"`); + envLines.push(`STORAGE_PATH=${formatEnvValue(publisherDefaults.storagePath)}`); + envLines.push( + `STORAGE_BASE_URL=${formatEnvValue(publisherDefaults.storageBaseUrl)}`, + ); + } + } + + console.log("\nWriting .env file..."); + await writeFileWithContent( path.resolve(process.cwd(), ".env"), - `PORT=9200 -EXPO_PUBLIC_MCP_URL="http://localhost:9200" -EXPO_PUBLIC_APP_URL="http://localhost:9200" -DATABASE_URL="${r.dbFilename}" -LLM_PROVIDER="${r.llmProvider}" -LLM_MODEL="${r.llmModel}" -LLM_TEMPERATURE="${r.llmTemperature}" -LLM_SYSTEM_PROMPT="${r.llmSystemPrompt}" -${getLLMProviderApiKeyEnvName(r.llmProvider)}="${r.llmApiKey}" -DKG_PUBLISH_WALLET="${r.dkgPublishWallet}" -DKG_BLOCKCHAIN="${r.dkgBlockchain}" -DKG_OTNODE_URL="http://localhost:8900" -SMTP_HOST="${r.smtpHost || ""}" -SMTP_PORT="${r.smtpPort || ""}" -SMTP_USER="${r.smtpUsername || ""}" -SMTP_PASS="${r.smtpPassword || ""}" -SMTP_SECURE=${r.smtpSecure === undefined ? "true" : r.smtpSecure} -SMTP_FROM="${r.smtpFrom || ""}" -DOCUMENT_CONVERSION_PROVIDER="${r.docConversionProvider}" -${r.docConversionProvider === "mistral" && r.llmProvider !== "mistralai" ? `MISTRAL_API_KEY="${r.mistralApiKey}"\n` : ""}`, + `${envLines.join("\n")}\n`, ); - console.log("Creating .env.development.local file..."); + console.log("Ensuring .env.development.local exists..."); await createFileWithContent( path.resolve(process.cwd(), ".env.development.local"), - `# These values will override the .env file during the development + `# These values will override the .env file during development EXPO_PUBLIC_APP_URL="http://localhost:8081" `, ); configEnv(); + process.env.DATABASE_URL = response.dbFilename; + process.env.EXPO_PUBLIC_MCP_URL = appUrl; + process.env.EXPO_PUBLIC_APP_URL = appUrl; console.log("Configuring database..."); console.log("Running migrations..."); const db = configDatabase(); console.log("Creating admin user..."); - const userId = await createUser( - db, - { - email: "admin@example.com", - password: "admin123", - }, - ["mcp", "llm", "blob", "scope123"], - ); - console.log(`Created admin user: - ID: ${userId} + try { + const user = await createUser( + db, + { + email: "admin@example.com", + password: "admin123", + }, + ["mcp", "llm", "blob", "scope123"], + ); + + console.log(`Created admin user: + ID: ${user.id} Email: admin@example.com Password: admin123 Scope: mcp, llm, blob, scope123 To create new users, run 'npm run script:createUser' inside of the agent directory. `); + } catch (error: any) { + if (error.message?.includes("already exists")) { + console.log("Admin user already exists. Skipping creation."); + } else { + throw error; + } + } + + console.log(`Async publishing: ${asyncPublishingEnabled ? "enabled" : "disabled"}`); + if (publisherDefaults) { + const passwordSource = + enginePassword.status === "found" + ? `Detected and copied MYSQL_PASSWORD from ${enginePassword.envPath}` + : "Used the MYSQL_PASSWORD provided during advanced setup"; + console.log(passwordSource); + if (publisherProvisionResult) { + console.log( + `Publisher DB ready: created=${publisherProvisionResult.databaseCreated}, walletInserted=${publisherProvisionResult.walletInserted}`, + ); + } + if (publisherProvisionError) { + console.log( + `Publisher provisioning warning: ${publisherProvisionError}. Async publishing was left disabled in apps/agent/.env.`, + ); + } + } } setup() diff --git a/apps/agent/src/server/setupPublisher.ts b/apps/agent/src/server/setupPublisher.ts new file mode 100644 index 00000000..97851594 --- /dev/null +++ b/apps/agent/src/server/setupPublisher.ts @@ -0,0 +1,119 @@ +import path from "path"; +import { promises as fs } from "fs"; +import dotenv from "dotenv"; +import { provisionPublisherDatabase } from "@dkg/plugin-dkg-publisher"; + +export type AsyncPublishingMode = "disabled" | "recommended" | "advanced"; + +export interface EnginePasswordResolution { + envPath: string; + mysqlPassword: string | null; + status: "found" | "missing-file" | "missing-key"; +} + +export interface PublisherAdvancedOptions { + mysqlHost?: string; + mysqlPort?: number; + mysqlUser?: string; + mysqlDatabase?: string; + redisUrl?: string; + workerCount?: number; + pollFrequency?: number; + storagePath?: string; + storageBaseUrl?: string; +} + +const MYSQL_IDENTIFIER_PATTERN = /^[A-Za-z0-9_]+$/; + +export function resolveNodeRoot(currentWorkingDirectory = process.cwd()) { + return path.resolve(currentWorkingDirectory, "../.."); +} + +export async function resolveEngineMysqlPassword(): Promise { + const envPath = + process.env.DKG_ENGINE_ENV_PATH || + path.join(resolveNodeRoot(), "dkg-engine/current/.env"); + + try { + const envFile = await fs.readFile(envPath, "utf8"); + const parsedEnv = dotenv.parse(envFile); + const mysqlPassword = parsedEnv.REPOSITORY_PASSWORD?.trim(); + + if (!mysqlPassword) { + return { + envPath, + mysqlPassword: null, + status: "missing-key", + }; + } + + return { + envPath, + mysqlPassword, + status: "found", + }; + } catch (error: any) { + if (error?.code === "ENOENT") { + return { + envPath, + mysqlPassword: null, + status: "missing-file", + }; + } + + throw error; + } +} + +export function buildPublisherDatabaseUrl( + mysqlPassword: string, + options: PublisherAdvancedOptions = {}, +) { + const mysqlUser = options.mysqlUser || "root"; + const mysqlHost = options.mysqlHost || "localhost"; + const mysqlPort = options.mysqlPort || 3306; + const mysqlDatabase = options.mysqlDatabase || "dkg_publisher_db"; + + if (!MYSQL_IDENTIFIER_PATTERN.test(mysqlDatabase)) { + throw new Error( + "Publisher MySQL database name may contain only letters, numbers, and underscores", + ); + } + + const encodedUser = encodeURIComponent(mysqlUser); + const encodedPassword = encodeURIComponent(mysqlPassword); + + return `mysql://${encodedUser}:${encodedPassword}@${mysqlHost}:${mysqlPort}/${mysqlDatabase}`; +} + +export function isValidMysqlIdentifier(value: string) { + return MYSQL_IDENTIFIER_PATTERN.test(value); +} + +export function buildPublisherDefaults( + appUrl: string, + mysqlPassword: string, + options: PublisherAdvancedOptions = {}, +) { + return { + mysqlPassword, + databaseUrl: buildPublisherDatabaseUrl(mysqlPassword, options), + redisUrl: options.redisUrl || "redis://localhost:6379", + workerCount: options.workerCount || 1, + pollFrequency: options.pollFrequency || 2000, + storagePath: options.storagePath || "./data/publisher", + storageBaseUrl: + options.storageBaseUrl || + new URL("/storage", appUrl).toString().replace(/\/$/, ""), + }; +} + +export async function provisionAsyncPublishing( + databaseUrl: string, + walletSeed?: { + privateKey: string; + blockchain: string; + }, +) { + return provisionPublisherDatabase(databaseUrl, walletSeed); +} diff --git a/apps/agent/tests/integration/dkg-publisher.env b/apps/agent/tests/integration/dkg-publisher.env index a447b702..360ce81e 100644 --- a/apps/agent/tests/integration/dkg-publisher.env +++ b/apps/agent/tests/integration/dkg-publisher.env @@ -5,12 +5,10 @@ DKGP_DATABASE_URL=sqlite://./test-dkg-publisher.db # Redis (test instance - will be started by test environment) -REDIS_HOST=localhost -REDIS_PORT=6379 -# REDIS_PASSWORD= (empty for test Redis) +REDIS_URL=redis://localhost:6379 # DKG Network (testnet) -DKG_ENDPOINT=http://localhost:8900 +DKG_OTNODE_URL=http://localhost:8900 DKG_BLOCKCHAIN=otp:20430 # Storage (test directory) diff --git a/docs/build-a-dkg-node-ai-agent/plugins/epcis-plugin.md b/docs/build-a-dkg-node-ai-agent/plugins/epcis-plugin.md index 6fcdd38f..e2525b06 100644 --- a/docs/build-a-dkg-node-ai-agent/plugins/epcis-plugin.md +++ b/docs/build-a-dkg-node-ai-agent/plugins/epcis-plugin.md @@ -25,11 +25,11 @@ It provides both HTTP endpoints and MCP tools for: 1. Enable EPCIS + publisher plugins in server plugin registration (this is not enabled by default in this repo): - `apps/agent/src/server/index.ts` should include `epcisPlugin` in the `plugins` array. - - `apps/agent/src/server/index.ts` should include `dkgPublisherPlugin` in the `plugins` array. + - `apps/agent/src/server/index.ts` loads `dkgPublisherPlugin` when `ASYNC_PUBLISHING_ENABLED=true`. - If you want route-level EPCIS scope enforcement, apply `applyEpcisHttpScopeGuards(api, authorized)` in the auth middleware plugin. -2. Run publisher plugin setup: - - `cd packages/plugin-dkg-publisher && npm run setup` - - This initializes publisher configuration (including `.env.publisher`) for the publisher flow. +2. Run the main Agent setup and enable async publishing when prompted: + - `cd apps/agent && npm run script:setup` + - This initializes the consolidated Agent + Publisher configuration in `apps/agent/.env`. 3. Configure runtime environment: - `EXPO_PUBLIC_MCP_URL=http://localhost:9200` (local same-host setup) 4. Create a token with EPCIS scopes: diff --git a/package-lock.json b/package-lock.json index ca2e6cc8..648a017b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -52,6 +52,7 @@ "dependencies": { "@dkg/expo-forcegraph": "^0.0.0", "@dkg/plugin-dkg-essentials": "^0.0.3", + "@dkg/plugin-dkg-publisher": "^1.0.0", "@dkg/plugin-example": "^0.0.3", "@dkg/plugin-oauth": "^0.0.2", "@dkg/plugin-swagger": "^0.0.2", diff --git a/packages/plugin-dkg-essentials/src/plugins/dkg-tools.ts b/packages/plugin-dkg-essentials/src/plugins/dkg-tools.ts index f409de48..3b91de92 100644 --- a/packages/plugin-dkg-essentials/src/plugins/dkg-tools.ts +++ b/packages/plugin-dkg-essentials/src/plugins/dkg-tools.ts @@ -1,6 +1,6 @@ import consumers from "stream/consumers"; import { defineDkgPlugin } from "@dkg/plugins"; -import { z } from "@dkg/plugins/helpers"; +import { openAPIRoute, z } from "@dkg/plugin-swagger"; import { CompleteResourceTemplateCallback, ResourceTemplate, @@ -9,8 +9,12 @@ import { import { BLOCKCHAIN_IDS } from "dkg.js/constants"; import { getExplorerUrl, validateSparqlQuery } from "../utils"; -export default defineDkgPlugin((ctx, mcp) => { - +type SupportedQueryType = "SELECT" | "CONSTRUCT"; +type SparqlValidationResult = + | { valid: true; queryType: SupportedQueryType } + | { valid: false; error: string }; + +export default defineDkgPlugin((ctx, mcp, api) => { async function publishJsonLdAsset( jsonldRaw: string, privacy: "private" | "public", @@ -31,6 +35,62 @@ export default defineDkgPlugin((ctx, mcp) => { } } + function validateSparqlInput(query: string): SparqlValidationResult { + const validation = validateSparqlQuery(query); + if (!validation.valid) { + return { + valid: false, + error: validation.error || "Invalid SPARQL query", + }; + } + + if (!validation.queryType) { + throw new Error( + "Internal error: queryType missing after successful validation", + ); + } + + return { + valid: true, + queryType: validation.queryType as SupportedQueryType, + }; + } + + async function runSparqlQuery( + query: string, + queryType: SupportedQueryType, + ) { + return ctx.dkg.graph.query(query, queryType); + } + + function formatSparqlResult(queryResult: unknown, queryType: SupportedQueryType) { + const isConstructQuery = queryType === "CONSTRUCT"; + const hasDataProperty = + typeof queryResult === "object" && + queryResult !== null && + "data" in queryResult; + + if ( + isConstructQuery && + hasDataProperty && + typeof queryResult.data === "string" + ) { + return { + codeBlockLang: "ntriples", + resultText: queryResult.data, + }; + } + + return { + codeBlockLang: "json", + resultText: JSON.stringify(queryResult, null, 2), + }; + } + + async function getAssetByUal(ual: string) { + return ctx.dkg.asset.get(ual); + } + const ualCompleteOptions: Record = { blockchainName: (val) => (Object.values(BLOCKCHAIN_IDS) as string[]).reduce( @@ -176,9 +236,7 @@ export default defineDkgPlugin((ctx, mcp) => { }, }, async ({ query }) => { - // Validate query syntax - const validation = validateSparqlQuery(query); - + const validation = validateSparqlInput(query); if (!validation.valid) { console.error("Invalid SPARQL query:", validation.error); return { @@ -191,34 +249,13 @@ export default defineDkgPlugin((ctx, mcp) => { }; } - // Use validated query type (must be defined after successful validation) - if (!validation.queryType) { - throw new Error("Internal error: queryType missing after successful validation"); - } - const queryType = validation.queryType; - try { - console.log(`Executing SPARQL ${queryType} query...`); - const queryResult = await ctx.dkg.graph.query(query, queryType); - - // Format result based on query type: - // - SELECT returns { data: [...bindings] } - // - CONSTRUCT returns { data: "" } - const isConstructQuery = queryType === "CONSTRUCT"; - const hasDataProperty = typeof queryResult === "object" && queryResult !== null && "data" in queryResult; - - let resultText: string; - let codeBlockLang: string; - - if (isConstructQuery && hasDataProperty && typeof queryResult.data === "string") { - // CONSTRUCT: extract N-triples string directly - resultText = queryResult.data; - codeBlockLang = "ntriples"; - } else { - // SELECT or fallback: JSON format - resultText = JSON.stringify(queryResult, null, 2); - codeBlockLang = "json"; - } + console.log(`Executing SPARQL ${validation.queryType} query...`); + const queryResult = await runSparqlQuery(query, validation.queryType); + const { codeBlockLang, resultText } = formatSparqlResult( + queryResult, + validation.queryType, + ); return { content: [ @@ -250,10 +287,16 @@ export default defineDkgPlugin((ctx, mcp) => { title: "DKG Knowledge Asset get tool", description: "Retrieve a specific Knowledge Asset from the DKG by its UAL (Unique Asset Locator). ", - inputSchema: { ual: z.string().describe("The UAL (Unique Asset Locator) in format: did:dkg:{blockchainName}:{blockchainId}/{blockchainAddress}/{collectionId}/{assetId} or did:dkg:{blockchainName}:{blockchainId}/{blockchainAddress}/{collectionId}") }, + inputSchema: { + ual: z + .string() + .describe( + "The UAL (Unique Asset Locator) in format: did:dkg:{blockchainName}:{blockchainId}/{blockchainAddress}/{collectionId}/{assetId} or did:dkg:{blockchainName}:{blockchainId}/{blockchainAddress}/{collectionId}", + ), + }, }, async ({ ual }) => { - const getAssetResult = await ctx.dkg.asset.get(ual); + const getAssetResult = await getAssetByUal(ual); return { content: [ { type: "text", text: JSON.stringify(getAssetResult, null, 2) }, @@ -262,4 +305,110 @@ export default defineDkgPlugin((ctx, mcp) => { }, ); + api.post( + "/api/dkg/query", + openAPIRoute( + { + tag: "DKG Retrieval", + summary: "Execute SPARQL Query", + description: "Execute a SPARQL query on the DKG network", + body: z.object({ + query: z.string().min(1, "Query cannot be empty"), + queryType: z + .enum(["SELECT", "CONSTRUCT"]) + .optional() + .default("SELECT"), + validate: z.boolean().optional().default(true), + }), + response: { + schema: z.object({ + success: z.boolean(), + data: z.any().optional(), + error: z.string().optional(), + validation: z + .object({ + valid: z.boolean(), + error: z.string().optional(), + }) + .optional(), + }), + }, + finalizeRouteConfig: (config) => ({ + ...config, + security: [], + }), + }, + async (req, res) => { + try { + let queryType: SupportedQueryType; + if (req.body.validate !== false) { + const validation = validateSparqlInput(req.body.query); + if (!validation.valid) { + return res.status(400).json({ + success: false, + error: validation.error, + validation: { + valid: false, + error: validation.error, + }, + }); + } + queryType = validation.queryType; + } else { + queryType = req.body.queryType || "SELECT"; + } + + const queryResult = await runSparqlQuery(req.body.query, queryType); + return res.json({ + success: true, + data: queryResult, + }); + } catch (error: any) { + return res.status(500).json({ + success: false, + error: error.message, + }); + } + }, + ), + ); + + api.get( + "/api/dkg/get", + openAPIRoute( + { + tag: "DKG Retrieval", + summary: "Get DKG Asset", + description: "Retrieve an asset from DKG by UAL", + query: z.object({ + ual: z.string(), + }), + response: { + schema: z.object({ + success: z.boolean(), + data: z.any().optional(), + error: z.string().optional(), + }), + }, + finalizeRouteConfig: (config) => ({ + ...config, + security: [], + }), + }, + async (req, res) => { + try { + const asset = await getAssetByUal(req.query.ual); + return res.json({ + success: true, + data: asset, + }); + } catch (error: any) { + return res.status(500).json({ + success: false, + error: error.message, + }); + } + }, + ), + ); }); diff --git a/packages/plugin-dkg-essentials/tests/dkg-tools.spec.ts b/packages/plugin-dkg-essentials/tests/dkg-tools.spec.ts index d512bb58..5adccff0 100644 --- a/packages/plugin-dkg-essentials/tests/dkg-tools.spec.ts +++ b/packages/plugin-dkg-essentials/tests/dkg-tools.spec.ts @@ -3,6 +3,7 @@ import { describe, it, beforeEach, afterEach } from "mocha"; import { expect } from "chai"; import sinon from "sinon"; +import request from "supertest"; import { dkgToolsPlugin } from "../dist/index.js"; import { getExplorerUrl, @@ -47,6 +48,12 @@ mockDkgContext.dkg.asset = { UAL: "did:dkg:otp:20430/0x123456/12345", }), }; +mockDkgContext.dkg.graph = { + query: (query: string, queryType: string) => + Promise.resolve({ + data: [{ query, queryType, result: "ok" }], + }), +}; describe("@dkg/plugin-dkg-essentials checks", () => { let mockMcpServer: McpServer; @@ -84,10 +91,16 @@ describe("@dkg/plugin-dkg-essentials checks", () => { expect(tools.some((t) => t.name === "dkg-create")).to.equal(true); }); - it("should register exactly 2 tools", async () => { + it("should register the dkg-sparql-query tool", async () => { const tools = await mockMcpClient.listTools().then((t) => t.tools); - expect(tools.length).to.equal(2); + expect(tools.some((t) => t.name === "dkg-sparql-query")).to.equal(true); + }); + + it("should register exactly 3 tools", async () => { + const tools = await mockMcpClient.listTools().then((t) => t.tools); + + expect(tools.length).to.equal(3); }); it("should have correct dkg-get tool configuration", async () => { @@ -96,7 +109,7 @@ describe("@dkg/plugin-dkg-essentials checks", () => { expect(dkgGetTool).to.not.equal(undefined); expect(dkgGetTool!.title).to.equal("DKG Knowledge Asset get tool"); - expect(dkgGetTool!.description).to.include("GET operation"); + expect(dkgGetTool!.description).to.include("Retrieve a specific"); expect(dkgGetTool!.description).to.include("UAL"); expect(dkgGetTool!.inputSchema).to.not.equal(undefined); }); @@ -412,6 +425,79 @@ describe("@dkg/plugin-dkg-essentials checks", () => { // Restore original mock mockDkgContext.dkg.asset.get = originalGet; }); + + it("should handle DKG service errors in SPARQL tool", async () => { + const originalQuery = mockDkgContext.dkg.graph.query; + try { + mockDkgContext.dkg.graph.query = () => { + throw new Error("DKG query unavailable"); + }; + + const result = await mockMcpClient.callTool({ + name: "dkg-sparql-query", + arguments: { query: "SELECT * WHERE { ?s ?p ?o } LIMIT 1" }, + }); + + expect((result.content as any[])[0].text).to.include( + "Error executing SPARQL query", + ); + expect((result.content as any[])[0].text).to.include( + "DKG query unavailable", + ); + } finally { + mockDkgContext.dkg.graph.query = originalQuery; + } + }); + }); + + describe("HTTP API Routes", () => { + it("should expose POST /api/dkg/query from Essentials", async () => { + const response = await request(app) + .post("/api/dkg/query") + .send({ + query: "SELECT * WHERE { ?s ?p ?o } LIMIT 1", + queryType: "SELECT", + }) + .expect(200); + + expect(response.body.success).to.equal(true); + expect(response.body.data.data[0].result).to.equal("ok"); + }); + + it("should return 400 for invalid POST /api/dkg/query input", async () => { + const response = await request(app) + .post("/api/dkg/query") + .send({ + query: "bad query", + queryType: "SELECT", + }) + .expect(400); + + expect(response.body.success).to.equal(false); + expect(response.body.error).to.be.a("string"); + }); + + it("should return 400 for unsupported POST /api/dkg/query queryType", async () => { + await request(app) + .post("/api/dkg/query") + .send({ + query: "SELECT * WHERE { ?s ?p ?o } LIMIT 1", + queryType: "ASK", + }) + .expect(400); + }); + + it("should expose GET /api/dkg/get from Essentials", async () => { + const response = await request(app) + .get("/api/dkg/get") + .query({ ual: "did:dkg:otp:20430/0x123456/12345" }) + .expect(200); + + expect(response.body.success).to.equal(true); + expect(response.body.data.metadata.UAL).to.equal( + "did:dkg:otp:20430/0x123456/12345", + ); + }); }); describe("Utility Functions", () => { @@ -512,7 +598,7 @@ describe("@dkg/plugin-dkg-essentials checks", () => { expect(result.text).to.include( "https://dkg-testnet.origintrail.io/explore?ual=did:dkg:test/123", ); - expect(result.description).to.be.a("string"); + expect((result as any)._meta.description).to.be.a("string"); }); }); diff --git a/packages/plugin-dkg-publisher/README.md b/packages/plugin-dkg-publisher/README.md index 91ff8f2e..4d8d1935 100644 --- a/packages/plugin-dkg-publisher/README.md +++ b/packages/plugin-dkg-publisher/README.md @@ -22,16 +22,16 @@ The DKG Publisher plugin provides a **production-ready, scalable system** for pu ### 1. Setup ```bash -npm run setup +cd apps/agent +npm run script:setup ``` -The setup script will guide you through: +The standard node setup now owns Publisher configuration too. Choose async +publishing during the Agent setup flow to write the consolidated Publisher +runtime env into `apps/agent/.env` and provision the Publisher MySQL database. -- Database configuration (MySQL) -- Redis configuration -- DKG network settings -- Wallet pool setup -- Worker configuration +`npm run setup` in this package remains as a legacy compatibility path for older +workflows that still rely on `.env.publisher`. ### 2. Start the Application diff --git a/packages/plugin-dkg-publisher/[LOCAL] DKG Node - Publisher.postman_collection.json b/packages/plugin-dkg-publisher/[LOCAL] DKG Node - Publisher.postman_collection.json index f8dd0504..58699329 100644 --- a/packages/plugin-dkg-publisher/[LOCAL] DKG Node - Publisher.postman_collection.json +++ b/packages/plugin-dkg-publisher/[LOCAL] DKG Node - Publisher.postman_collection.json @@ -150,9 +150,9 @@ "method": "GET", "header": [], "url": { - "raw": "{{endpoint}}/api/dkg/assets?ual=did:dkg:otp:20430/0xcdb28e93ed340ec10a71bba00a31dbfcf1bd5d37/347048", + "raw": "{{endpoint}}/api/dkg/get?ual=did:dkg:otp:20430/0xcdb28e93ed340ec10a71bba00a31dbfcf1bd5d37/347048", "host": ["{{endpoint}}"], - "path": ["api", "dkg", "assets"], + "path": ["api", "dkg", "get"], "query": [ { "key": "ual", @@ -160,7 +160,7 @@ } ] }, - "description": "Generated from cURL: curl -X GET \"http://localhost:9200/api/dkg/assets/did:dkg:otp:20430/0x1234567890abcdef/123456?contentType=all&includeMetadata=true\"" + "description": "Generated from cURL: curl -X GET \"http://localhost:9200/api/dkg/get?ual=did:dkg:otp:20430/0x1234567890abcdef/123456\"" }, "response": [] } diff --git a/packages/plugin-dkg-publisher/scripts/check-queue-status.js b/packages/plugin-dkg-publisher/scripts/check-queue-status.js index efc900d8..7c2b062f 100644 --- a/packages/plugin-dkg-publisher/scripts/check-queue-status.js +++ b/packages/plugin-dkg-publisher/scripts/check-queue-status.js @@ -6,11 +6,16 @@ const IORedis = require("ioredis"); async function checkQueueStatus() { console.log("๐Ÿ” Checking BullMQ queue status...\n"); + const redisUrl = + process.env.REDIS_URL || + `redis://${process.env.REDIS_HOST || "localhost"}:${process.env.REDIS_PORT || "6379"}`; + const parsedRedisUrl = new URL(redisUrl); + // Connect to Redis (same config as the service) const redis = new IORedis({ - host: process.env.REDIS_HOST || "localhost", - port: process.env.REDIS_PORT || 6379, - password: process.env.REDIS_PASSWORD, + host: parsedRedisUrl.hostname || "localhost", + port: Number(parsedRedisUrl.port || 6379), + password: parsedRedisUrl.password || process.env.REDIS_PASSWORD, maxRetriesPerRequest: 3, retryStrategy: (times) => Math.min(times * 50, 2000), }); diff --git a/packages/plugin-dkg-publisher/scripts/clear-failed-jobs.js b/packages/plugin-dkg-publisher/scripts/clear-failed-jobs.js index e7d99e11..33eb79dd 100644 --- a/packages/plugin-dkg-publisher/scripts/clear-failed-jobs.js +++ b/packages/plugin-dkg-publisher/scripts/clear-failed-jobs.js @@ -6,11 +6,16 @@ const IORedis = require("ioredis"); async function clearFailedJobs() { console.log("๐Ÿงน Clearing failed jobs from BullMQ..."); + const redisUrl = + process.env.REDIS_URL || + `redis://${process.env.REDIS_HOST || "localhost"}:${process.env.REDIS_PORT || "6379"}`; + const parsedRedisUrl = new URL(redisUrl); + // Connect to Redis (same config as the service) const redis = new IORedis({ - host: process.env.REDIS_HOST || "localhost", - port: process.env.REDIS_PORT || 6379, - password: process.env.REDIS_PASSWORD, + host: parsedRedisUrl.hostname || "localhost", + port: Number(parsedRedisUrl.port || 6379), + password: parsedRedisUrl.password || process.env.REDIS_PASSWORD, maxRetriesPerRequest: 3, retryStrategy: (times) => Math.min(times * 50, 2000), }); diff --git a/packages/plugin-dkg-publisher/src/config.ts b/packages/plugin-dkg-publisher/src/config.ts new file mode 100644 index 00000000..aaaacb7b --- /dev/null +++ b/packages/plugin-dkg-publisher/src/config.ts @@ -0,0 +1,144 @@ +import path from "path"; +import { config as dotenvConfig } from "dotenv"; +import type { KnowledgeAssetManagerConfig } from "./types"; + +export interface PublisherRuntimeSettings { + dkgEndpoint: string; + dkgBlockchain: string; + workerCount: number; + pollFrequency: number; + storageType: "filesystem" | "s3"; + storagePath: string; + storageBaseUrl: string; + redisUrl: string; +} + +export interface PublisherConfigResolution { + config: KnowledgeAssetManagerConfig; + runtime: PublisherRuntimeSettings; + legacyEnvPath: string; + loadedLegacyEnv: boolean; +} + +function parsePositiveInt(value: string | undefined, fallback: number) { + const parsed = Number.parseInt(value || "", 10); + return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; +} + +function loadLegacyPublisherEnv(): { + legacyEnvPath: string; + loadedLegacyEnv: boolean; +} { + const legacyEnvPath = path.resolve(__dirname, "..", ".env.publisher"); + const hasPublisherConfig = + Boolean(process.env.DKGP_DATABASE_URL) && + Boolean( + process.env.REDIS_URL || + process.env.REDIS_HOST || + process.env.REDIS_PORT || + process.env.REDIS_PASSWORD, + ); + + if (!hasPublisherConfig) { + dotenvConfig({ path: legacyEnvPath }); + } + + return { + legacyEnvPath, + loadedLegacyEnv: !hasPublisherConfig && Boolean(process.env.DKGP_DATABASE_URL), + }; +} + +function getDefaultStorageBaseUrl(): string { + const baseUrl = + process.env.EXPO_PUBLIC_MCP_URL || `http://localhost:${process.env.PORT || "9200"}`; + return new URL("/storage", baseUrl).toString().replace(/\/$/, ""); +} + +function resolveRedisSettings() { + const redisUrl = + process.env.REDIS_URL || + `redis://${process.env.REDIS_HOST || "localhost"}:${process.env.REDIS_PORT || "6379"}`; + + const parsedRedisUrl = new URL(redisUrl); + + return { + redisUrl, + host: parsedRedisUrl.hostname || "localhost", + port: Number(parsedRedisUrl.port || 6379), + password: + parsedRedisUrl.password || process.env.REDIS_PASSWORD || undefined, + }; +} + +export function resolvePublisherRuntimeConfig(): PublisherConfigResolution | null { + const { legacyEnvPath, loadedLegacyEnv } = loadLegacyPublisherEnv(); + const databaseUrl = process.env.DKGP_DATABASE_URL; + + if (!databaseUrl) { + return null; + } + + const redis = resolveRedisSettings(); + const storageType = + process.env.STORAGE_TYPE === "s3" ? "s3" : "filesystem"; + const storagePath = + process.env.STORAGE_PATH || path.resolve(process.cwd(), "data/publisher"); + const storageBaseUrl = + process.env.STORAGE_BASE_URL || getDefaultStorageBaseUrl(); + const dkgEndpoint = + process.env.DKG_OTNODE_URL || + process.env.DKG_ENDPOINT || + "http://localhost:8900"; + const dkgBlockchain = process.env.DKG_BLOCKCHAIN || "hardhat1:31337"; + const workerCount = parsePositiveInt(process.env.WORKER_COUNT, 1); + const pollFrequency = parsePositiveInt(process.env.POLL_FREQUENCY, 2000); + + return { + legacyEnvPath, + loadedLegacyEnv, + runtime: { + dkgEndpoint, + dkgBlockchain, + workerCount, + pollFrequency, + storageType, + storagePath, + storageBaseUrl, + redisUrl: redis.redisUrl, + }, + config: { + database: { + connectionString: databaseUrl, + }, + redis: { + host: redis.host, + port: redis.port, + password: redis.password, + }, + wallets: [], + dkg: { + endpoint: dkgEndpoint, + blockchain: dkgBlockchain, + }, + storage: { + type: storageType, + path: storagePath, + }, + }, + }; +} + +export function applyPublisherRuntimeDefaults( + resolution: PublisherConfigResolution, +) { + process.env.DKGP_DATABASE_URL ||= resolution.config.database.connectionString; + process.env.REDIS_URL ||= resolution.runtime.redisUrl; + process.env.DKG_OTNODE_URL ||= resolution.runtime.dkgEndpoint; + process.env.DKG_BLOCKCHAIN ||= resolution.runtime.dkgBlockchain; + process.env.WORKER_COUNT ||= resolution.runtime.workerCount.toString(); + process.env.POLL_FREQUENCY ||= resolution.runtime.pollFrequency.toString(); + process.env.STORAGE_TYPE ||= resolution.runtime.storageType; + process.env.STORAGE_PATH ||= resolution.runtime.storagePath; + process.env.STORAGE_BASE_URL ||= resolution.runtime.storageBaseUrl; +} diff --git a/packages/plugin-dkg-publisher/src/database/bootstrap.ts b/packages/plugin-dkg-publisher/src/database/bootstrap.ts index f6191d24..6474b7ee 100644 --- a/packages/plugin-dkg-publisher/src/database/bootstrap.ts +++ b/packages/plugin-dkg-publisher/src/database/bootstrap.ts @@ -114,7 +114,7 @@ async function tableExists(db: Database, tableName: string): Promise { const result = await db.execute( sql`SELECT COUNT(*) as cnt FROM information_schema.tables WHERE table_schema = DATABASE() AND table_name = ${tableName}`, ); - const rows = result[0] as Array<{ cnt: number | bigint }>; + const rows = result[0] as unknown as Array<{ cnt: number | bigint }>; return Number(rows[0]?.cnt) > 0; } @@ -126,6 +126,6 @@ async function columnExists( const result = await db.execute( sql`SELECT COUNT(*) as cnt FROM information_schema.columns WHERE table_schema = DATABASE() AND table_name = ${tableName} AND column_name = ${columnName}`, ); - const rows = result[0] as Array<{ cnt: number | bigint }>; + const rows = result[0] as unknown as Array<{ cnt: number | bigint }>; return Number(rows[0]?.cnt) > 0; } diff --git a/packages/plugin-dkg-publisher/src/index.ts b/packages/plugin-dkg-publisher/src/index.ts index 0f8a95d9..7c29e025 100644 --- a/packages/plugin-dkg-publisher/src/index.ts +++ b/packages/plugin-dkg-publisher/src/index.ts @@ -1,7 +1,6 @@ import { defineDkgPlugin } from "@dkg/plugins"; import { z } from "@dkg/plugin-swagger"; import type { KnowledgeAssetManagerConfig, AssetInput } from "./types"; -import { config as dotenvConfig } from "dotenv"; import path from "path"; import { initializeServices, @@ -10,11 +9,14 @@ import { AssetService, WalletService, QueueService, - DkgService, } from "./services"; import { openAPIRoute } from "@dkg/plugin-swagger"; import express from "express"; import { registerMcpTools } from "./mcp/tools"; +import { + applyPublisherRuntimeDefaults, + resolvePublisherRuntimeConfig, +} from "./config"; /** * DKG Publisher Plugin @@ -38,38 +40,16 @@ export default defineDkgPlugin((ctx, mcp, api) => { console.log( `๐Ÿ” DKG Publisher Plugin executing at ${new Date().toISOString()} (${pluginInitTime})`, ); - // Load configuration from package root .env file - const envPath = path.resolve(__dirname, "..", ".env.publisher"); - - console.log(`๐Ÿ”ง Loading DKG Publisher config from: ${envPath}`); - dotenvConfig({ path: envPath }); - - console.log(`๐Ÿ“Š DKGP_DATABASE_URL found: ${!!process.env.DKGP_DATABASE_URL}`); + const resolution = resolvePublisherRuntimeConfig(); + console.log(`๐Ÿ“Š DKGP_DATABASE_URL found: ${Boolean(resolution)}`); - // Initialize services if configuration is provided via environment - if (process.env.DKGP_DATABASE_URL) { - const config: KnowledgeAssetManagerConfig = { - database: { - connectionString: process.env.DKGP_DATABASE_URL, - }, - redis: { - host: process.env.REDIS_HOST || "localhost", - port: process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : 6379, - password: process.env.REDIS_PASSWORD, - }, - wallets: [], // Should be loaded from config or setup - dkg: { - endpoint: process.env.DKG_ENDPOINT, - blockchain: process.env.DKG_BLOCKCHAIN, - }, - encryptionKey: process.env.ENCRYPTION_KEY, - }; + if (resolution) { + const config: KnowledgeAssetManagerConfig = resolution.config; + applyPublisherRuntimeDefaults(resolution); console.log(`๐Ÿš€ Initializing DKG Publisher services... (${Date.now()})`); // Mount storage directory immediately (before services initialize) - const storageType = process.env.STORAGE_TYPE || "filesystem"; - const storagePath = - process.env.STORAGE_PATH || path.resolve(__dirname, "../storage"); + const { storageType, storagePath } = resolution.runtime; if (storageType === "filesystem") { try { @@ -97,6 +77,11 @@ export default defineDkgPlugin((ctx, mcp, api) => { console.log(` - DKG Endpoint: ${config.dkg?.endpoint}`); console.log(` - Blockchain: ${config.dkg?.blockchain}`); console.log(`๐Ÿ“ Storage configured for: ${storageType}`); + if (resolution.loadedLegacyEnv) { + console.log( + ` - Legacy env imported from: ${resolution.legacyEnvPath}`, + ); + } }) .catch((error) => { console.error("โŒ DKG Publisher Plugin initialization failed:", error); @@ -105,7 +90,9 @@ export default defineDkgPlugin((ctx, mcp, api) => { console.log( "โš ๏ธ DKG Publisher Plugin not configured - DKGP_DATABASE_URL not found", ); - console.log(` Looked for config in: ${envPath}`); + console.log( + ` Looked for config in: ${path.resolve(__dirname, "..", ".env.publisher")}`, + ); } // Mount admin dashboard route immediately - handle service readiness internally @@ -292,114 +279,6 @@ export default defineDkgPlugin((ctx, mcp, api) => { } }); - // Add SPARQL query endpoint - api.post( - "/api/dkg/query", - openAPIRoute( - { - tag: "DKG Queries", - summary: "Execute SPARQL Query", - description: "Execute a SPARQL query on the DKG network", - body: z.object({ - query: z.string().min(1, "Query cannot be empty"), - queryType: z - .enum(["SELECT", "CONSTRUCT", "ASK", "DESCRIBE"]) - .optional() - .default("SELECT"), - validate: z.boolean().optional().default(true), - }), - response: { - schema: z.object({ - success: z.boolean(), - data: z.any().optional(), - error: z.string().optional(), - validation: z - .object({ - valid: z.boolean(), - error: z.string().optional(), - }) - .optional(), - }), - }, - finalizeRouteConfig: (config) => ({ - ...config, - security: [], - }), - }, - async (req, res) => { - if (!serviceContainer) { - return res.status(503).json({ - success: false, - error: "DKG service is starting up", - }); - } - - try { - const { query, queryType = "SELECT" } = req.body; - const dkgService = serviceContainer.get("dkgService"); - - // Execute SPARQL query - const result = await dkgService.executeSparqlQuery(query, queryType); - - res.json(result); - } catch (error: any) { - res.status(500).json({ - success: false, - error: error.message, - }); - } - }, - ), - ); - - // Add DKG asset get endpoint - api.get( - "/api/dkg/assets", - openAPIRoute( - { - tag: "DKG Queries", - summary: "Get DKG Asset", - description: "Retrieve an asset from DKG by UAL", - query: z.object({ - ual: z.string(), - }), - response: { - schema: z.object({ - success: z.boolean(), - data: z.any().optional(), - error: z.string().optional(), - }), - }, - finalizeRouteConfig: (config) => ({ - ...config, - security: [], - }), - }, - async (req, res) => { - if (!serviceContainer) { - return res.status(503).json({ - success: false, - error: "DKG service is starting up", - }); - } - - try { - const { ual } = req.query; - const dkgService = serviceContainer.get("dkgService"); - - const result = await dkgService.getAsset(ual); - - res.json(result); - } catch (error: any) { - res.status(500).json({ - success: false, - error: error.message, - }); - } - }, - ), - ); - // Register all MCP tools for publisher plugin registerMcpTools(mcp, serviceContainer, ctx); }); @@ -430,3 +309,8 @@ export type { AssetStatus, KnowledgeAssetManagerConfig, } from "./types"; +export { + applyPublisherRuntimeDefaults, + resolvePublisherRuntimeConfig, +} from "./config"; +export { provisionPublisherDatabase } from "./provision"; diff --git a/packages/plugin-dkg-publisher/src/provision.ts b/packages/plugin-dkg-publisher/src/provision.ts new file mode 100644 index 00000000..f3ef341a --- /dev/null +++ b/packages/plugin-dkg-publisher/src/provision.ts @@ -0,0 +1,93 @@ +import mysql from "mysql2/promise"; +import { Wallet } from "ethers"; +import { runMigrations } from "./database"; + +export interface PublisherWalletSeed { + privateKey: string; + blockchain: string; +} + +export interface PublisherProvisionResult { + databaseCreated: boolean; + walletInserted: boolean; +} + +const MYSQL_IDENTIFIER_PATTERN = /^[A-Za-z0-9_]+$/; + +function getDatabaseName(connectionString: string) { + const databaseUrl = new URL(connectionString); + const databaseName = databaseUrl.pathname.replace(/^\//, ""); + + if (!databaseName) { + throw new Error("DKGP_DATABASE_URL must include a database name"); + } + + if (!MYSQL_IDENTIFIER_PATTERN.test(databaseName)) { + throw new Error( + "DKGP_DATABASE_URL contains an invalid database name. Use letters, numbers, and underscores only.", + ); + } + + return { databaseUrl, databaseName }; +} + +function getAdminConnectionString(connectionString: string) { + const { databaseUrl } = getDatabaseName(connectionString); + databaseUrl.pathname = "/"; + return databaseUrl.toString(); +} + +export async function provisionPublisherDatabase( + connectionString: string, + walletSeed?: PublisherWalletSeed, +): Promise { + const { databaseName } = getDatabaseName(connectionString); + let databaseCreated = false; + let walletInserted = false; + + const adminConnection = await mysql.createConnection( + getAdminConnectionString(connectionString), + ); + + try { + const [existingDatabases] = await adminConnection.execute( + "SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME = ?", + [databaseName], + ); + databaseCreated = (existingDatabases as unknown[]).length === 0; + + await adminConnection.query( + `CREATE DATABASE IF NOT EXISTS \`${databaseName}\``, + ); + } finally { + await adminConnection.end(); + } + + await runMigrations(connectionString); + + if (!walletSeed) { + return { databaseCreated, walletInserted }; + } + + const address = new Wallet(walletSeed.privateKey).address; + const databaseConnection = await mysql.createConnection(connectionString); + + try { + const [existingWallets] = await databaseConnection.execute( + "SELECT id FROM wallets WHERE address = ? LIMIT 1", + [address], + ); + + if ((existingWallets as unknown[]).length === 0) { + await databaseConnection.execute( + "INSERT INTO wallets (address, private_key, blockchain) VALUES (?, ?, ?)", + [address, walletSeed.privateKey, walletSeed.blockchain], + ); + walletInserted = true; + } + } finally { + await databaseConnection.end(); + } + + return { databaseCreated, walletInserted }; +} diff --git a/packages/plugin-dkg-publisher/src/services/AssetService.ts b/packages/plugin-dkg-publisher/src/services/AssetService.ts index acf0c2c3..cb16b45c 100644 --- a/packages/plugin-dkg-publisher/src/services/AssetService.ts +++ b/packages/plugin-dkg-publisher/src/services/AssetService.ts @@ -367,7 +367,10 @@ export class AssetService extends EventEmitter { workerId: process.pid.toString(), walletAddress: wallet.address, walletId: wallet.id, - otnodeUrl: process.env.DKG_ENDPOINT, + otnodeUrl: + process.env.DKG_OTNODE_URL || + process.env.DKG_ENDPOINT || + "http://localhost:8900", blockchain: wallet.blockchain, status: "started", startedAt: sql`NOW()`, diff --git a/packages/plugin-dkg-publisher/src/services/DkgService.ts b/packages/plugin-dkg-publisher/src/services/DkgService.ts index 0b61c8de..f0f2aed9 100644 --- a/packages/plugin-dkg-publisher/src/services/DkgService.ts +++ b/packages/plugin-dkg-publisher/src/services/DkgService.ts @@ -20,7 +20,10 @@ export class DkgService { private walletService: WalletService; constructor(walletService: WalletService) { - this.dkgEndpoint = process.env.DKG_ENDPOINT || "http://localhost:8900"; + this.dkgEndpoint = + process.env.DKG_OTNODE_URL || + process.env.DKG_ENDPOINT || + "http://localhost:8900"; this.dkgBlockchain = process.env.DKG_BLOCKCHAIN || "hardhat1:31337"; this.walletService = walletService; this.initializeQueryClient(); @@ -319,7 +322,14 @@ export class DkgService { */ async getNodeInfo(): Promise { try { - const dkgClient = this.createQueryDKGClient(); + if (!this.queryClient) { + await this.initializeQueryClient(); + if (!this.queryClient) { + throw new Error("Failed to initialize DKG query client"); + } + } + + const dkgClient = this.queryClient; const nodeInfo = await dkgClient.node.info(); return { diff --git a/packages/plugin-dkg-publisher/src/services/PublishingService.ts b/packages/plugin-dkg-publisher/src/services/PublishingService.ts index 797c9178..87410d4d 100644 --- a/packages/plugin-dkg-publisher/src/services/PublishingService.ts +++ b/packages/plugin-dkg-publisher/src/services/PublishingService.ts @@ -17,11 +17,8 @@ export interface PublishResult { export class PublishingService { private dkgService: DkgService; - constructor( - private db: Database, - dkgService?: DkgService, - ) { - this.dkgService = dkgService || new DkgService(); + constructor(private db: Database, dkgService: DkgService) { + this.dkgService = dkgService; } /** @@ -89,7 +86,7 @@ export class PublishingService { // Try direct filesystem access const storagePath = process.env.STORAGE_PATH || - path.resolve(__dirname, "../../storage"); + path.resolve(process.cwd(), "data/publisher"); const filePath = require("path").resolve(storagePath, filename); console.log(`๐Ÿ”„ Trying direct file access: ${filePath}`); diff --git a/packages/plugin-dkg-publisher/src/services/StorageService.ts b/packages/plugin-dkg-publisher/src/services/StorageService.ts index 398483cf..9654a590 100644 --- a/packages/plugin-dkg-publisher/src/services/StorageService.ts +++ b/packages/plugin-dkg-publisher/src/services/StorageService.ts @@ -8,10 +8,13 @@ export class StorageService { constructor() { this.storagePath = - process.env.STORAGE_PATH || path.resolve(__dirname, "../storage"); - const serverPort = process.env.PORT || "9200"; + process.env.STORAGE_PATH || path.resolve(process.cwd(), "data/publisher"); + const baseUrl = + process.env.EXPO_PUBLIC_MCP_URL || + `http://localhost:${process.env.PORT || "9200"}`; this.baseUrl = - process.env.STORAGE_BASE_URL || `http://localhost:${serverPort}/storage`; + process.env.STORAGE_BASE_URL || + new URL("/storage", baseUrl).toString().replace(/\/$/, ""); console.log(`๐Ÿ“ StorageService initialized:`); console.log(` - storagePath: ${this.storagePath}`); console.log(` - resolved path: ${path.resolve(this.storagePath)}`); From 1a5627f3835ed8c48ad1e9a60e7ab1c671959fb3 Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Wed, 4 Mar 2026 19:50:13 +0100 Subject: [PATCH 02/10] [feat] unify Agent setup and async Publisher integration --- README.md | 5 +- apps/agent/env.d.ts | 1 - apps/agent/src/server/index.ts | 8 +- apps/agent/src/server/scripts/setup.ts | 405 ++++++++++++------ apps/agent/src/server/setupPublisher.ts | 34 +- .../tests/integration/setup/test-server.ts | 2 +- packages/plugin-dkg-publisher/README.md | 8 +- packages/plugin-dkg-publisher/package.json | 17 + .../plugin-dkg-publisher/src/privateKey.ts | 27 ++ .../plugin-dkg-publisher/src/provision.ts | 39 +- 10 files changed, 395 insertions(+), 151 deletions(-) create mode 100644 packages/plugin-dkg-publisher/src/privateKey.ts diff --git a/README.md b/README.md index b2f9149e..6f3c30a6 100644 --- a/README.md +++ b/README.md @@ -77,10 +77,13 @@ The setup script will: If you enable async publishing during setup, the Agent writes the consolidated Publisher settings into `apps/agent/.env`: - `ASYNC_PUBLISHING_ENABLED=true` -- `MYSQL_PASSWORD` - `DKGP_DATABASE_URL` - `REDIS_URL` +The setup stores `DKG_PUBLISH_WALLET` in `apps/agent/.env` without a `0x` +prefix for compatibility with existing node env files, and Publisher wallet +records are seeded into MySQL using the same bare 64-hex format. + The Agent server only loads `@dkg/plugin-dkg-publisher` when `ASYNC_PUBLISHING_ENABLED=true`. ### 4. Start Development diff --git a/apps/agent/env.d.ts b/apps/agent/env.d.ts index 377522e3..7346990a 100644 --- a/apps/agent/env.d.ts +++ b/apps/agent/env.d.ts @@ -15,7 +15,6 @@ declare global { DKG_BLOCKCHAIN: string; DKG_OTNODE_URL: string; ASYNC_PUBLISHING_ENABLED?: string; - MYSQL_PASSWORD?: string; DKGP_DATABASE_URL?: string; REDIS_URL?: string; WORKER_COUNT?: string; diff --git a/apps/agent/src/server/index.ts b/apps/agent/src/server/index.ts index 550a9bda..17873de7 100644 --- a/apps/agent/src/server/index.ts +++ b/apps/agent/src/server/index.ts @@ -1,5 +1,9 @@ import path from "path"; -import { createPluginServer, defaultPlugin } from "@dkg/plugins"; +import { + createPluginServer, + defaultPlugin, + type DkgPlugin, +} from "@dkg/plugins"; import { authorized, createOAuthPlugin } from "@dkg/plugin-oauth"; import dkgEssentialsPlugin from "@dkg/plugin-dkg-essentials"; import createFsBlobStorage from "@dkg/plugin-dkg-essentials/createFsBlobStorage"; @@ -83,7 +87,7 @@ async function main() { const blobStorage = createFsBlobStorage(path.join(__dirname, "../data")); const otnodeUrl = new URL(process.env.DKG_OTNODE_URL); - const plugins = [ + const plugins: DkgPlugin[] = [ defaultPlugin, oauthPlugin, (_, __, api) => { diff --git a/apps/agent/src/server/scripts/setup.ts b/apps/agent/src/server/scripts/setup.ts index fcb938e8..cb6d4f3d 100644 --- a/apps/agent/src/server/scripts/setup.ts +++ b/apps/agent/src/server/scripts/setup.ts @@ -13,42 +13,223 @@ import { } from "../helpers"; import { buildPublisherDefaults, + isValidPrivateKey, isValidMysqlIdentifier, - resolveEngineMysqlPassword, provisionAsyncPublishing, + resolveEngineMysqlPassword, + stripPrivateKeyPrefix, } from "../setupPublisher"; function formatEnvValue(value: string) { return `"${value.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`; } -function isValidPrivateKey(value: string) { - return /^0x[0-9a-fA-F]{64}$/.test(value); +const promptOptions = { + onCancel: () => { + throw new Error("Setup cancelled by user"); + }, +}; + +const styles = { + reset: "\x1b[0m", + bold: "\x1b[1m", + cyan: "\x1b[36m", + blue: "\x1b[34m", + yellow: "\x1b[33m", +}; + +function printBanner() { + const lines = [ + "+------------------------------------------------------------+", + "| DKG Agent Setup |", + "| |", + "| This script will help you configure your DKG Agent. |", + "+------------------------------------------------------------+", + ]; + + console.log( + `\n${styles.bold}${styles.blue}${lines.join("\n")}${styles.reset}\n`, + ); } -async function setup() { - const enginePassword = await resolveEngineMysqlPassword(); +function printSection(title: string, description?: string) { + console.log( + `\n${styles.bold}${styles.cyan}=== ${title} ===${styles.reset}`, + ); + if (description) { + console.log(`${styles.yellow}${description}${styles.reset}`); + } +} - if (enginePassword.status === "found") { - console.log( - `Detected MySQL password from ${enginePassword.envPath}. Async publishing can reuse it.`, - ); - } else if (enginePassword.status === "missing-file") { - console.log( - `Engine config not found at ${enginePassword.envPath}. Async publishing defaults will require advanced setup.`, +function getRecommendedWorkerCount(walletCount: number) { + return Math.max(1, Math.min(Math.ceil(walletCount / 10), 5)); +} + +async function collectAdditionalPublisherWallets() { + const additionalWallets: string[] = []; + const addWalletsResponse = await prompts( + { + type: "confirm", + name: "addMoreWallets", + message: + "Add more publishing wallets now? The primary wallet will already be included.", + initial: false, + }, + promptOptions, + ); + + let shouldAddWallet = addWalletsResponse.addMoreWallets === true; + while (shouldAddWallet) { + const walletResponse = await prompts( + { + type: "text", + name: "privateKey", + message: `Additional publish wallet private key #${additionalWallets.length + 1}`, + validate: (value) => { + if (!value.length) return "Required"; + return ( + isValidPrivateKey(value) || + "Private key must be 64 hexadecimal characters, with or without a 0x prefix" + ); + }, + }, + promptOptions, ); - } else { - console.log( - `Engine config found at ${enginePassword.envPath}, but REPOSITORY_PASSWORD is missing.`, + + additionalWallets.push(walletResponse.privateKey); + + const continueResponse = await prompts( + { + type: "confirm", + name: "addAnotherWallet", + message: "Add another publishing wallet?", + initial: false, + }, + promptOptions, ); + shouldAddWallet = continueResponse.addAnotherWallet === true; } + return additionalWallets; +} + +async function collectAdvancedPublisherOptions( + hasEngineDefaultMysqlPassword: boolean, +) { + const connectionResponse = await prompts( + [ + { + type: "text", + name: "mysqlHost", + message: "Publisher MySQL host", + initial: "localhost", + }, + { + type: "number", + name: "mysqlPort", + message: "Publisher MySQL port", + initial: 3306, + min: 0, + }, + { + type: "text", + name: "mysqlUser", + message: "Publisher MySQL username", + initial: "root", + }, + { + type: "password", + name: "mysqlPassword", + message: hasEngineDefaultMysqlPassword + ? "Publisher MySQL password (leave blank to use the password set during dkg-node install)" + : "Publisher MySQL password", + validate: (value) => + hasEngineDefaultMysqlPassword || value.length > 0 + ? true + : "Required for async publishing", + }, + { + type: "text", + name: "mysqlDatabase", + message: "Publisher MySQL database name", + initial: "dkg_publisher_db", + validate: (value) => + isValidMysqlIdentifier(value) || + "Use letters, numbers, and underscores only", + }, + { + type: "text", + name: "redisHost", + message: "Publisher Redis host", + initial: "localhost", + }, + { + type: "number", + name: "redisPort", + message: "Publisher Redis port", + initial: 6379, + min: 0, + }, + { + type: "password", + name: "redisPassword", + message: "Publisher Redis password (leave blank if no password)", + }, + ], + promptOptions, + ); + + const additionalWallets = await collectAdditionalPublisherWallets(); + const totalWalletCount = 1 + additionalWallets.length; + const recommendedWorkerCount = getRecommendedWorkerCount(totalWalletCount); + const workerDefaultsResponse = await prompts( + [ + { + type: "number", + name: "workerCount", + message: `Publisher worker count (default: ${recommendedWorkerCount}, based on wallet count; concurrency auto-balances)`, + initial: recommendedWorkerCount, + min: 1, + }, + { + type: "number", + name: "pollFrequency", + message: "Publisher poll frequency (ms)", + initial: 2000, + min: 100, + }, + { + type: "text", + name: "storagePath", + message: "Publisher storage path", + initial: "./data/publisher", + }, + { + type: "text", + name: "storageBaseUrl", + message: "Publisher storage base URL", + initial: "http://localhost:9200/storage", + }, + ], + promptOptions, + ); + + return { + ...connectionResponse, + ...workerDefaultsResponse, + additionalWallets, + }; +} + +async function setup() { + const enginePassword = await resolveEngineMysqlPassword(); + printBanner(); + const asyncPublishingChoices = enginePassword.status === "found" ? [ { - title: - "Yes (Recommended) - more seamless publishing and tracking", + title: "Yes (Recommended)", value: "recommended", }, { title: "No", value: "disabled" }, @@ -59,7 +240,11 @@ async function setup() { { title: "Yes, with advanced configuration", value: "advanced" }, ]; - const response = await prompts([ + printSection( + "DKG Agent LLM Configuration", + "Choose the language model provider, credentials, model, and default system prompt for your DKG Agent.", + ); + const llmResponse = await prompts([ { type: "select", name: "llmProvider", @@ -96,6 +281,13 @@ async function setup() { initial: DEFAULT_SYSTEM_PROMPT, format: (value) => (value === DEFAULT_SYSTEM_PROMPT ? "" : value.trim()), }, + ], promptOptions); + + printSection( + "Document Processing", + "Choose how the agent should convert uploaded documents for downstream use.", + ); + const documentResponse = await prompts([ { type: "select", name: "docConversionProvider", @@ -109,13 +301,30 @@ async function setup() { { type: (_, answers) => answers.docConversionProvider === "mistral" && - answers.llmProvider !== "mistralai" + llmResponse.llmProvider !== "mistralai" ? "text" : null, name: "mistralApiKey", message: "MISTRAL_API_KEY", validate: (value) => value.length || "Required for Mistral OCR provider", }, + ], promptOptions); + + printSection( + "DKG Interaction", + "Configure the DKG network, blockchain, and publishing setup for your DKG Agent.", + ); + if (enginePassword.status === "missing-file") { + console.log( + `Publisher MySQL defaults were not found at ${enginePassword.envPath}. Use advanced async setup if you want to provide them manually.`, + ); + } else if (enginePassword.status === "missing-key") { + console.log( + `Publisher MySQL defaults are incomplete in ${enginePassword.envPath} because REPOSITORY_PASSWORD is missing. Use advanced async setup if you want to provide the password manually.`, + ); + } + + const publishingResponse = await prompts([ { type: "select", name: "dkgEnv", @@ -151,13 +360,13 @@ async function setup() { message: "Publish wallet private key", initial: (_, answers) => answers.dkgEnv === "development" - ? "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" + ? "ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" : "", validate: (value) => { if (!value.length) return "Required"; return ( isValidPrivateKey(value) || - "Private key must be a 0x-prefixed 32-byte hex string" + "Private key must be 64 hexadecimal characters, with or without a 0x prefix" ); }, }, @@ -165,89 +374,26 @@ async function setup() { type: "select", name: "asyncPublishingMode", message: - "Enable async publishing for more seamless publishing and tracking of published knowledge assets?", + "Enable async publishing on the DKG for smoother publishing, easier status tracking, and queue management?", choices: asyncPublishingChoices, initial: 0, }, - { - type: (_, answers) => - answers.asyncPublishingMode === "advanced" && - enginePassword.status !== "found" - ? "password" - : null, - name: "mysqlPassword", - message: "MYSQL_PASSWORD", - validate: (value) => value.length || "Required for async publishing", - }, - { - type: (_, answers) => - answers.asyncPublishingMode === "advanced" ? "text" : null, - name: "mysqlHost", - message: "Publisher MySQL host", - initial: "localhost", - }, - { - type: (_, answers) => - answers.asyncPublishingMode === "advanced" ? "number" : null, - name: "mysqlPort", - message: "Publisher MySQL port", - initial: 3306, - min: 0, - }, - { - type: (_, answers) => - answers.asyncPublishingMode === "advanced" ? "text" : null, - name: "mysqlUser", - message: "Publisher MySQL username", - initial: "root", - }, - { - type: (_, answers) => - answers.asyncPublishingMode === "advanced" ? "text" : null, - name: "mysqlDatabase", - message: "Publisher MySQL database name", - initial: "dkg_publisher_db", - validate: (value) => - isValidMysqlIdentifier(value) || - "Use letters, numbers, and underscores only", - }, - { - type: (_, answers) => - answers.asyncPublishingMode === "advanced" ? "text" : null, - name: "redisUrl", - message: "Publisher Redis URL", - initial: "redis://localhost:6379", - }, - { - type: (_, answers) => - answers.asyncPublishingMode === "advanced" ? "number" : null, - name: "workerCount", - message: "Publisher worker count", - initial: 1, - min: 1, - }, - { - type: (_, answers) => - answers.asyncPublishingMode === "advanced" ? "number" : null, - name: "pollFrequency", - message: "Publisher poll frequency (ms)", - initial: 2000, - min: 100, - }, - { - type: (_, answers) => - answers.asyncPublishingMode === "advanced" ? "text" : null, - name: "storagePath", - message: "Publisher storage path", - initial: "./data/publisher", - }, - { - type: (_, answers) => - answers.asyncPublishingMode === "advanced" ? "text" : null, - name: "storageBaseUrl", - message: "Publisher storage base URL", - initial: "http://localhost:9200/storage", - }, + ], promptOptions); + + const advancedPublisherResponse = + publishingResponse.asyncPublishingMode === "advanced" + ? (printSection( + "Advanced Async Publishing", + "Provide MySQL, Redis, wallet, worker, and storage overrides for the DKG Publisher Plugin.", + ), + await collectAdvancedPublisherOptions(enginePassword.status === "found")) + : null; + + printSection( + "Email Configuration", + "Configure SMTP settings for password reset emails and other notifications.", + ); + const emailResponse = await prompts([ { type: "confirm", name: "smtpEnabled", @@ -289,6 +435,13 @@ async function setup() { message: "SMTP Sender email", initial: "noreply@example.com", }, + ], promptOptions); + + printSection( + "DKG Agent Database", + "Choose the local SQLite database file used by the DKG Agent.", + ); + const finalResponse = await prompts([ { type: "text", name: "dbFilename", @@ -296,16 +449,22 @@ async function setup() { validate: (value) => value.length || "Required", format: (value) => (value.endsWith(".db") ? value : `${value}.db`), }, - ], { - onCancel: () => { - throw new Error("Setup cancelled by user"); - }, - }); + ], promptOptions); + + const response = { + ...llmResponse, + ...documentResponse, + ...publishingResponse, + ...(advancedPublisherResponse || {}), + ...emailResponse, + ...finalResponse, + }; const requestedAsyncPublishing = response.asyncPublishingMode !== "disabled"; const appUrl = "http://localhost:9200"; + const envPublishWallet = stripPrivateKeyPrefix(response.dkgPublishWallet); const mysqlPassword = - enginePassword.mysqlPassword || response.mysqlPassword || ""; + response.mysqlPassword?.trim() || enginePassword.mysqlPassword || ""; const publisherDefaults = requestedAsyncPublishing && mysqlPassword ? buildPublisherDefaults(appUrl, mysqlPassword, { @@ -313,7 +472,9 @@ async function setup() { mysqlPort: response.mysqlPort, mysqlUser: response.mysqlUser, mysqlDatabase: response.mysqlDatabase, - redisUrl: response.redisUrl, + redisHost: response.redisHost, + redisPort: response.redisPort, + redisPassword: response.redisPassword, workerCount: response.workerCount, pollFrequency: response.pollFrequency, storagePath: response.storagePath, @@ -325,8 +486,10 @@ async function setup() { throw new Error("Async publishing requires a MySQL password"); } + const additionalPublisherWallets = advancedPublisherResponse?.additionalWallets || []; + let publisherProvisionResult: - | { databaseCreated: boolean; walletInserted: boolean } + | { databaseCreated: boolean; walletsInserted: number } | null = null; let publisherProvisionError: string | null = null; @@ -334,10 +497,16 @@ async function setup() { try { publisherProvisionResult = await provisionAsyncPublishing( publisherDefaults.databaseUrl, - { - privateKey: response.dkgPublishWallet, - blockchain: response.dkgBlockchain, - }, + [ + { + privateKey: response.dkgPublishWallet, + blockchain: response.dkgBlockchain, + }, + ...additionalPublisherWallets.map((privateKey) => ({ + privateKey, + blockchain: response.dkgBlockchain, + })), + ], ); } catch (error: any) { publisherProvisionError = error.message; @@ -360,7 +529,7 @@ async function setup() { `LLM_TEMPERATURE=${formatEnvValue(String(response.llmTemperature))}`, `LLM_SYSTEM_PROMPT=${formatEnvValue(response.llmSystemPrompt)}`, `${getLLMProviderApiKeyEnvName(response.llmProvider)}=${formatEnvValue(response.llmApiKey)}`, - `DKG_PUBLISH_WALLET=${formatEnvValue(response.dkgPublishWallet)}`, + `DKG_PUBLISH_WALLET=${formatEnvValue(envPublishWallet)}`, `DKG_BLOCKCHAIN=${formatEnvValue(response.dkgBlockchain)}`, 'DKG_OTNODE_URL="http://localhost:8900"', `ASYNC_PUBLISHING_ENABLED=${asyncPublishingEnabled ? "true" : "false"}`, @@ -381,7 +550,6 @@ async function setup() { } if (publisherDefaults) { - envLines.push(`MYSQL_PASSWORD=${formatEnvValue(publisherDefaults.mysqlPassword)}`); envLines.push(`DKGP_DATABASE_URL=${formatEnvValue(publisherDefaults.databaseUrl)}`); envLines.push(`REDIS_URL=${formatEnvValue(publisherDefaults.redisUrl)}`); envLines.push(`WORKER_COUNT=${formatEnvValue(String(publisherDefaults.workerCount))}`); @@ -450,14 +618,9 @@ To create new users, run 'npm run script:createUser' inside of the agent directo console.log(`Async publishing: ${asyncPublishingEnabled ? "enabled" : "disabled"}`); if (publisherDefaults) { - const passwordSource = - enginePassword.status === "found" - ? `Detected and copied MYSQL_PASSWORD from ${enginePassword.envPath}` - : "Used the MYSQL_PASSWORD provided during advanced setup"; - console.log(passwordSource); if (publisherProvisionResult) { console.log( - `Publisher DB ready: created=${publisherProvisionResult.databaseCreated}, walletInserted=${publisherProvisionResult.walletInserted}`, + `Publisher DB ready: created=${publisherProvisionResult.databaseCreated}, walletsInserted=${publisherProvisionResult.walletsInserted}`, ); } if (publisherProvisionError) { diff --git a/apps/agent/src/server/setupPublisher.ts b/apps/agent/src/server/setupPublisher.ts index 97851594..946adc2a 100644 --- a/apps/agent/src/server/setupPublisher.ts +++ b/apps/agent/src/server/setupPublisher.ts @@ -1,7 +1,12 @@ import path from "path"; import { promises as fs } from "fs"; import dotenv from "dotenv"; -import { provisionPublisherDatabase } from "@dkg/plugin-dkg-publisher"; +import { provisionPublisherDatabase } from "@dkg/plugin-dkg-publisher/provision"; +import { + isValidPrivateKey, + normalizePrivateKey, + stripPrivateKeyPrefix, +} from "@dkg/plugin-dkg-publisher/privateKey"; export type AsyncPublishingMode = "disabled" | "recommended" | "advanced"; @@ -16,6 +21,9 @@ export interface PublisherAdvancedOptions { mysqlPort?: number; mysqlUser?: string; mysqlDatabase?: string; + redisHost?: string; + redisPort?: number; + redisPassword?: string; redisUrl?: string; workerCount?: number; pollFrequency?: number; @@ -90,15 +98,29 @@ export function isValidMysqlIdentifier(value: string) { return MYSQL_IDENTIFIER_PATTERN.test(value); } +export { isValidPrivateKey, normalizePrivateKey, stripPrivateKeyPrefix }; + +export function buildRedisUrl(options: PublisherAdvancedOptions = {}) { + if (options.redisUrl) { + return options.redisUrl; + } + + const redisHost = options.redisHost || "localhost"; + const redisPort = options.redisPort || 6379; + const redisPassword = options.redisPassword?.trim(); + const auth = redisPassword ? `:${encodeURIComponent(redisPassword)}@` : ""; + + return `redis://${auth}${redisHost}:${redisPort}`; +} + export function buildPublisherDefaults( appUrl: string, mysqlPassword: string, options: PublisherAdvancedOptions = {}, ) { return { - mysqlPassword, databaseUrl: buildPublisherDatabaseUrl(mysqlPassword, options), - redisUrl: options.redisUrl || "redis://localhost:6379", + redisUrl: buildRedisUrl(options), workerCount: options.workerCount || 1, pollFrequency: options.pollFrequency || 2000, storagePath: options.storagePath || "./data/publisher", @@ -110,10 +132,10 @@ export function buildPublisherDefaults( export async function provisionAsyncPublishing( databaseUrl: string, - walletSeed?: { + walletSeeds: Array<{ privateKey: string; blockchain: string; - }, + }> = [], ) { - return provisionPublisherDatabase(databaseUrl, walletSeed); + return provisionPublisherDatabase(databaseUrl, walletSeeds); } diff --git a/apps/agent/tests/integration/setup/test-server.ts b/apps/agent/tests/integration/setup/test-server.ts index 0d79917d..aad91b28 100644 --- a/apps/agent/tests/integration/setup/test-server.ts +++ b/apps/agent/tests/integration/setup/test-server.ts @@ -133,7 +133,7 @@ export async function createTestServer(config: TestServerConfig = {}): Promise<{ port: "8900", blockchain: { name: process.env.DKG_BLOCKCHAIN || "hardhat1:31337", - privateKey: process.env.DKG_PUBLISH_WALLET || "0x" + "a".repeat(64), + privateKey: process.env.DKG_PUBLISH_WALLET || "a".repeat(64), }, maxNumberOfRetries: 300, frequency: 2, diff --git a/packages/plugin-dkg-publisher/README.md b/packages/plugin-dkg-publisher/README.md index 4d8d1935..f9f585c0 100644 --- a/packages/plugin-dkg-publisher/README.md +++ b/packages/plugin-dkg-publisher/README.md @@ -29,6 +29,8 @@ npm run script:setup The standard node setup now owns Publisher configuration too. Choose async publishing during the Agent setup flow to write the consolidated Publisher runtime env into `apps/agent/.env` and provision the Publisher MySQL database. +In advanced mode, setup can also seed additional publishing wallets into the +Publisher wallet pool. `npm run setup` in this package remains as a legacy compatibility path for older workflows that still rely on `.env.publisher`. @@ -99,8 +101,10 @@ pending โ†’ queued โ†’ assigned โ†’ publishing โ†’ published โœ… **During setup:** ```bash -npm run setup -# Choose option 3: "Add wallets only" +cd apps/agent +npm run script:setup +# Choose "Yes, with advanced configuration" for async publishing +# The primary wallet is seeded automatically and setup can add more wallets ``` **After 5 minutes**, workers automatically restart with new concurrency. No application restart needed! diff --git a/packages/plugin-dkg-publisher/package.json b/packages/plugin-dkg-publisher/package.json index 9950e434..79037921 100644 --- a/packages/plugin-dkg-publisher/package.json +++ b/packages/plugin-dkg-publisher/package.json @@ -5,6 +5,23 @@ "main": "./dist/index.js", "module": "./dist/index.mjs", "types": "./dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.mjs", + "require": "./dist/index.js" + }, + "./provision": { + "types": "./dist/provision.d.ts", + "import": "./dist/provision.mjs", + "require": "./dist/provision.js" + }, + "./privateKey": { + "types": "./dist/privateKey.d.ts", + "import": "./dist/privateKey.mjs", + "require": "./dist/privateKey.js" + } + }, "scripts": { "dev": "tsup src/*.ts --format cjs,esm --dts --watch", "build": "tsup src/*.ts --format cjs,esm --dts", diff --git a/packages/plugin-dkg-publisher/src/privateKey.ts b/packages/plugin-dkg-publisher/src/privateKey.ts new file mode 100644 index 00000000..bbf089a0 --- /dev/null +++ b/packages/plugin-dkg-publisher/src/privateKey.ts @@ -0,0 +1,27 @@ +const PRIVATE_KEY_BODY_PATTERN = /^[0-9a-fA-F]{64}$/; + +export function normalizePrivateKey(privateKey: string) { + const trimmed = privateKey.trim(); + const body = trimmed.startsWith("0x") ? trimmed.slice(2) : trimmed; + + if (!PRIVATE_KEY_BODY_PATTERN.test(body)) { + throw new Error( + "Private key must be 64 hexadecimal characters, with or without a 0x prefix", + ); + } + + return `0x${body}`; +} + +export function stripPrivateKeyPrefix(privateKey: string) { + return normalizePrivateKey(privateKey).slice(2); +} + +export function isValidPrivateKey(privateKey: string) { + try { + normalizePrivateKey(privateKey); + return true; + } catch { + return false; + } +} diff --git a/packages/plugin-dkg-publisher/src/provision.ts b/packages/plugin-dkg-publisher/src/provision.ts index f3ef341a..0ecfff29 100644 --- a/packages/plugin-dkg-publisher/src/provision.ts +++ b/packages/plugin-dkg-publisher/src/provision.ts @@ -1,6 +1,7 @@ import mysql from "mysql2/promise"; import { Wallet } from "ethers"; import { runMigrations } from "./database"; +import { normalizePrivateKey, stripPrivateKeyPrefix } from "./privateKey"; export interface PublisherWalletSeed { privateKey: string; @@ -9,7 +10,7 @@ export interface PublisherWalletSeed { export interface PublisherProvisionResult { databaseCreated: boolean; - walletInserted: boolean; + walletsInserted: number; } const MYSQL_IDENTIFIER_PATTERN = /^[A-Za-z0-9_]+$/; @@ -39,11 +40,11 @@ function getAdminConnectionString(connectionString: string) { export async function provisionPublisherDatabase( connectionString: string, - walletSeed?: PublisherWalletSeed, + walletSeeds: PublisherWalletSeed[] = [], ): Promise { const { databaseName } = getDatabaseName(connectionString); let databaseCreated = false; - let walletInserted = false; + let walletsInserted = 0; const adminConnection = await mysql.createConnection( getAdminConnectionString(connectionString), @@ -65,29 +66,33 @@ export async function provisionPublisherDatabase( await runMigrations(connectionString); - if (!walletSeed) { - return { databaseCreated, walletInserted }; + if (!walletSeeds.length) { + return { databaseCreated, walletsInserted }; } - const address = new Wallet(walletSeed.privateKey).address; const databaseConnection = await mysql.createConnection(connectionString); try { - const [existingWallets] = await databaseConnection.execute( - "SELECT id FROM wallets WHERE address = ? LIMIT 1", - [address], - ); - - if ((existingWallets as unknown[]).length === 0) { - await databaseConnection.execute( - "INSERT INTO wallets (address, private_key, blockchain) VALUES (?, ?, ?)", - [address, walletSeed.privateKey, walletSeed.blockchain], + for (const walletSeed of walletSeeds) { + const normalizedPrivateKey = normalizePrivateKey(walletSeed.privateKey); + const storedPrivateKey = stripPrivateKeyPrefix(walletSeed.privateKey); + const address = new Wallet(normalizedPrivateKey).address; + const [existingWallets] = await databaseConnection.execute( + "SELECT id FROM wallets WHERE address = ? LIMIT 1", + [address], ); - walletInserted = true; + + if ((existingWallets as unknown[]).length === 0) { + await databaseConnection.execute( + "INSERT INTO wallets (address, private_key, blockchain) VALUES (?, ?, ?)", + [address, storedPrivateKey, walletSeed.blockchain], + ); + walletsInserted += 1; + } } } finally { await databaseConnection.end(); } - return { databaseCreated, walletInserted }; + return { databaseCreated, walletsInserted }; } From 7a7354f82704ddc512cbfacec60f95d8d411b890 Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Wed, 4 Mar 2026 21:06:37 +0100 Subject: [PATCH 03/10] [feat] add optional custom DKG RPC support --- README.md | 1 + apps/agent/env.d.ts | 1 + apps/agent/src/server/index.ts | 2 ++ apps/agent/src/server/scripts/setup.ts | 24 +++++++++++++++ .../src/blockchainConfig.ts | 16 ++++++++++ .../src/services/DkgService.ts | 11 ++++--- .../tests/blockchain-config.spec.ts | 30 +++++++++++++++++++ 7 files changed, 81 insertions(+), 4 deletions(-) create mode 100644 packages/plugin-dkg-publisher/src/blockchainConfig.ts create mode 100644 packages/plugin-dkg-publisher/tests/blockchain-config.spec.ts diff --git a/README.md b/README.md index 6f3c30a6..6a2f1523 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,7 @@ Before running the project setup, you'll need to configure the following environ - **Testnet** (safe testing with mock tokens): `https://v6-pegasus-node-02.origin-trail.network:8900` - **Mainnet** (production DKG interactions): `https://positron.origin-trail.network` - **Local development**: `http://localhost:8900` (default) +- **`DKG_NODE_CUSTOM_RPC`**: Optional custom blockchain RPC URL. Leave unset to use the default RPC behavior. - **`PORT`**: Server port (default: `9200`) - **`EXPO_PUBLIC_APP_URL`**: Public app URL (default: `http://localhost:9200`) - **`EXPO_PUBLIC_MCP_URL`**: MCP server URL (default: `http://localhost:9200`) diff --git a/apps/agent/env.d.ts b/apps/agent/env.d.ts index 7346990a..14a94bb5 100644 --- a/apps/agent/env.d.ts +++ b/apps/agent/env.d.ts @@ -14,6 +14,7 @@ declare global { DKG_PUBLISH_WALLET: string; DKG_BLOCKCHAIN: string; DKG_OTNODE_URL: string; + DKG_NODE_CUSTOM_RPC?: string; ASYNC_PUBLISHING_ENABLED?: string; DKGP_DATABASE_URL?: string; REDIS_URL?: string; diff --git a/apps/agent/src/server/index.ts b/apps/agent/src/server/index.ts index 17873de7..def22735 100644 --- a/apps/agent/src/server/index.ts +++ b/apps/agent/src/server/index.ts @@ -87,6 +87,7 @@ async function main() { const blobStorage = createFsBlobStorage(path.join(__dirname, "../data")); const otnodeUrl = new URL(process.env.DKG_OTNODE_URL); + const dkgCustomRpc = process.env.DKG_NODE_CUSTOM_RPC?.trim(); const plugins: DkgPlugin[] = [ defaultPlugin, oauthPlugin, @@ -151,6 +152,7 @@ async function main() { blockchain: { name: process.env.DKG_BLOCKCHAIN, privateKey: process.env.DKG_PUBLISH_WALLET, + ...(dkgCustomRpc && { rpc: dkgCustomRpc }), }, maxNumberOfRetries: 300, frequency: 2, diff --git a/apps/agent/src/server/scripts/setup.ts b/apps/agent/src/server/scripts/setup.ts index cb6d4f3d..4715e662 100644 --- a/apps/agent/src/server/scripts/setup.ts +++ b/apps/agent/src/server/scripts/setup.ts @@ -65,6 +65,15 @@ function getRecommendedWorkerCount(walletCount: number) { return Math.max(1, Math.min(Math.ceil(walletCount / 10), 5)); } +function isValidUrl(value: string) { + try { + new URL(value); + return true; + } catch { + return false; + } +} + async function collectAdditionalPublisherWallets() { const additionalWallets: string[] = []; const addWalletsResponse = await prompts( @@ -354,6 +363,16 @@ async function setup() { { title: "Gnosis Chiado", value: "gnosis:10200" }, ], }, + { + type: "text", + name: "dkgCustomRpc", + message: "Custom blockchain RPC (leave blank to use default RPC)", + format: (value) => value.trim(), + validate: (value) => + !value.trim() || + isValidUrl(value.trim()) || + "Provide a valid URL or leave blank to use the default RPC", + }, { type: "text", name: "dkgPublishWallet", @@ -462,6 +481,7 @@ async function setup() { const requestedAsyncPublishing = response.asyncPublishingMode !== "disabled"; const appUrl = "http://localhost:9200"; + const customRpc = response.dkgCustomRpc?.trim() || ""; const envPublishWallet = stripPrivateKeyPrefix(response.dkgPublishWallet); const mysqlPassword = response.mysqlPassword?.trim() || enginePassword.mysqlPassword || ""; @@ -542,6 +562,10 @@ async function setup() { `DOCUMENT_CONVERSION_PROVIDER=${formatEnvValue(response.docConversionProvider)}`, ]; + if (customRpc) { + envLines.push(`DKG_NODE_CUSTOM_RPC=${formatEnvValue(customRpc)}`); + } + if ( response.docConversionProvider === "mistral" && response.llmProvider !== "mistralai" diff --git a/packages/plugin-dkg-publisher/src/blockchainConfig.ts b/packages/plugin-dkg-publisher/src/blockchainConfig.ts new file mode 100644 index 00000000..7c19668b --- /dev/null +++ b/packages/plugin-dkg-publisher/src/blockchainConfig.ts @@ -0,0 +1,16 @@ +export interface DkgBlockchainConfigBase { + name: string; + privateKey: string; + publicKey?: string; +} + +export function buildDkgBlockchainConfig( + base: DkgBlockchainConfigBase, + customRpc?: string, +) { + const rpc = customRpc?.trim(); + return { + ...base, + ...(rpc ? { rpc } : {}), + }; +} diff --git a/packages/plugin-dkg-publisher/src/services/DkgService.ts b/packages/plugin-dkg-publisher/src/services/DkgService.ts index f0f2aed9..8aa7547a 100644 --- a/packages/plugin-dkg-publisher/src/services/DkgService.ts +++ b/packages/plugin-dkg-publisher/src/services/DkgService.ts @@ -1,5 +1,6 @@ import DKG from "dkg.js"; import { WalletService } from "./WalletService"; +import { buildDkgBlockchainConfig } from "../blockchainConfig"; export interface SparqlQueryResult { success: boolean; @@ -16,6 +17,7 @@ export interface DkgGetResult { export class DkgService { private dkgEndpoint: string; private dkgBlockchain: string; + private dkgCustomRpc?: string; private queryClient: any | null = null; private walletService: WalletService; @@ -25,6 +27,7 @@ export class DkgService { process.env.DKG_ENDPOINT || "http://localhost:8900"; this.dkgBlockchain = process.env.DKG_BLOCKCHAIN || "hardhat1:31337"; + this.dkgCustomRpc = process.env.DKG_NODE_CUSTOM_RPC?.trim() || undefined; this.walletService = walletService; this.initializeQueryClient(); } @@ -53,11 +56,11 @@ export class DkgService { this.queryClient = new DKG({ endpoint: `${endpointUrl.protocol}//${endpointUrl.hostname}`, port: endpointUrl.port, - blockchain: { + blockchain: buildDkgBlockchainConfig({ name: this.dkgBlockchain, publicKey: wallet.address, privateKey: wallet.privateKey, - }, + }, this.dkgCustomRpc), maxNumberOfRetries: 100, frequency: 2, contentType: "all", @@ -185,11 +188,11 @@ export class DkgService { const walletDkgClient = new DKG({ endpoint: `${endpointUrl.protocol}//${endpointUrl.hostname}`, port: endpointUrl.port, - blockchain: { + blockchain: buildDkgBlockchainConfig({ name: this.dkgBlockchain, publicKey: wallet.address, privateKey: privateKey, - }, + }, this.dkgCustomRpc), maxNumberOfRetries: 600, frequency: 2, contentType: "all", diff --git a/packages/plugin-dkg-publisher/tests/blockchain-config.spec.ts b/packages/plugin-dkg-publisher/tests/blockchain-config.spec.ts new file mode 100644 index 00000000..1d7c10d0 --- /dev/null +++ b/packages/plugin-dkg-publisher/tests/blockchain-config.spec.ts @@ -0,0 +1,30 @@ +import { expect } from "chai"; +import { buildDkgBlockchainConfig } from "../dist/blockchainConfig.mjs"; + +describe("buildDkgBlockchainConfig", () => { + it("omits rpc when custom rpc is missing or blank", () => { + const base = { + name: "otp:20430", + privateKey: "a".repeat(64), + publicKey: "0x123", + }; + + const withoutRpc = buildDkgBlockchainConfig(base); + const blankRpc = buildDkgBlockchainConfig(base, " "); + + expect(withoutRpc).to.not.have.property("rpc"); + expect(blankRpc).to.not.have.property("rpc"); + }); + + it("includes trimmed rpc when provided", () => { + const base = { + name: "otp:20430", + privateKey: "a".repeat(64), + publicKey: "0x123", + }; + + const config = buildDkgBlockchainConfig(base, " https://rpc.example "); + + expect(config).to.have.property("rpc", "https://rpc.example"); + }); +}); From 431a1d3b52e55322aa544d176834bf89e28266ca Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Thu, 5 Mar 2026 10:32:53 +0100 Subject: [PATCH 04/10] [feat] Add DKG Publisher plugin management flow Introduce script:publisher for DKG Publisher plugin config updates, wallet management, and fresh setup reset. Unify setup messaging around async publishing + DKG Publisher plugin and add helper/unit-test coverage for env/config handling. --- apps/agent/package.json | 1 + apps/agent/src/server/scripts/publisher.ts | 733 +++++++++++++++++++ apps/agent/src/server/scripts/setup.ts | 14 +- apps/agent/src/server/setupPublisher.ts | 412 +++++++++++ apps/agent/tests/unit/setupPublisher.spec.ts | 140 ++++ packages/plugin-dkg-publisher/README.md | 20 +- 6 files changed, 1311 insertions(+), 9 deletions(-) create mode 100644 apps/agent/src/server/scripts/publisher.ts create mode 100644 apps/agent/tests/unit/setupPublisher.spec.ts diff --git a/apps/agent/package.json b/apps/agent/package.json index cd932d92..161a0b5f 100644 --- a/apps/agent/package.json +++ b/apps/agent/package.json @@ -17,6 +17,7 @@ "dev:server": "tsup src/server/*.ts --format esm,cjs --watch --onSuccess=\"node dist/index.js --dev\"", "drizzle:studio": "drizzle-kit studio", "script:setup": "node dist/scripts/setup.js", + "script:publisher": "node dist/scripts/publisher.js", "script:createUser": "node dist/scripts/createUser.js", "script:createToken": "node dist/scripts/createToken.js", "test": "PLAYWRIGHT_JUNIT_OUTPUT_NAME=DKG_Node_UI_Tests.xml npx playwright test spec/testUI.spec.js --grep '@gh_actions' --reporter=list,html,junit", diff --git a/apps/agent/src/server/scripts/publisher.ts b/apps/agent/src/server/scripts/publisher.ts new file mode 100644 index 00000000..4298577d --- /dev/null +++ b/apps/agent/src/server/scripts/publisher.ts @@ -0,0 +1,733 @@ +import prompts from "prompts"; +import { + addPublisherWallets, + buildPublisherDefaults, + isValidMysqlIdentifier, + isValidPrivateKey, + listPublisherWallets, + readAgentEnv, + resolveEngineMysqlPassword, + resolvePublisherConfigFromAgentEnv, + resetPublisherDatabase, + setPublisherWalletActive, + stripPrivateKeyPrefix, + type AgentEnvState, + upsertAgentEnvValues, +} from "../setupPublisher"; + +const promptOptions = { + onCancel: () => { + throw new Error("DKG Publisher plugin setup cancelled by user"); + }, +}; + +const styles = { + reset: "\x1b[0m", + bold: "\x1b[1m", + cyan: "\x1b[36m", + blue: "\x1b[34m", + yellow: "\x1b[33m", +}; + +const LOCK_STALE_THRESHOLD_MS = 30 * 60 * 1000; + +function printBanner() { + const lines = [ + "+------------------------------------------------------------+", + "| DKG Publisher Plugin Management |", + "| |", + "| Configure DKG Publisher plugin for async publishing. |", + "+------------------------------------------------------------+", + ]; + + console.log( + `\n${styles.bold}${styles.blue}${lines.join("\n")}${styles.reset}\n`, + ); +} + +function printSection(title: string, description?: string) { + console.log( + `\n${styles.bold}${styles.cyan}=== ${title} ===${styles.reset}`, + ); + if (description) { + console.log(`${styles.yellow}${description}${styles.reset}`); + } +} + +function getRecommendedWorkerCount(walletCount: number) { + return Math.max(1, Math.min(Math.ceil(walletCount / 10), 5)); +} + +function isValidUrl(value: string) { + try { + new URL(value); + return true; + } catch { + return false; + } +} + +function isStaleLock(lockedAt: Date | null) { + if (!lockedAt) { + return false; + } + return Date.now() - lockedAt.getTime() >= LOCK_STALE_THRESHOLD_MS; +} + +function resolvePublisherConfigSafely( + envValues: Record, + appUrl: string, + options: { silent?: boolean } = {}, +) { + try { + return resolvePublisherConfigFromAgentEnv(envValues, appUrl); + } catch (error: any) { + if (!options.silent) { + console.warn( + `Existing DKG Publisher plugin configuration could not be parsed (${error.message}).`, + ); + } + return null; + } +} + +async function collectAdditionalWallets(options?: { + promptForFirstConfirmation?: boolean; +}) { + const wallets: string[] = []; + + let shouldAdd = options?.promptForFirstConfirmation === false; + if (!shouldAdd) { + const addWalletsResponse = await prompts( + { + type: "confirm", + name: "addWallets", + message: "Add additional publishing wallets?", + initial: false, + }, + promptOptions, + ); + shouldAdd = addWalletsResponse.addWallets === true; + } + + while (shouldAdd) { + const walletResponse = await prompts( + { + type: "text", + name: "privateKey", + message: `Additional publish wallet private key #${wallets.length + 1}`, + validate: (value) => { + if (!value.length) return "Required"; + return ( + isValidPrivateKey(value) || + "Private key must be 64 hexadecimal characters, with or without a 0x prefix" + ); + }, + }, + promptOptions, + ); + + wallets.push(walletResponse.privateKey); + + const continueResponse = await prompts( + { + type: "confirm", + name: "addAnother", + message: "Add another publishing wallet?", + initial: false, + }, + promptOptions, + ); + shouldAdd = continueResponse.addAnother === true; + } + + return wallets; +} + +async function ensurePrimaryWallet(envPublishWallet: string | undefined) { + if (envPublishWallet && isValidPrivateKey(envPublishWallet)) { + return envPublishWallet; + } + + const response = await prompts( + { + type: "text", + name: "primaryWallet", + message: + "Primary publish wallet private key (required for publisher provisioning)", + validate: (value) => + isValidPrivateKey(value) || + "Private key must be 64 hexadecimal characters, with or without a 0x prefix", + }, + promptOptions, + ); + + return response.primaryWallet as string; +} + +async function runUpdateConfigMode() { + const envState = await readAgentEnv(); + const enginePassword = await resolveEngineMysqlPassword(); + const appUrl = envState.values.EXPO_PUBLIC_MCP_URL || "http://localhost:9200"; + const publisherConfig = resolvePublisherConfigSafely( + envState.values, + appUrl, + { silent: true }, + ); + + printSection( + "Update DKG Publisher Plugin Config", + "Update MySQL/Redis settings and seed wallets for async publishing.", + ); + + const mysqlPasswordFromEnv = publisherConfig?.mysql.password || ""; + const mysqlPasswordFromEngine = enginePassword.mysqlPassword || ""; + const mysqlPasswordDefault = mysqlPasswordFromEnv || mysqlPasswordFromEngine; + const hasDefaultMysqlPassword = mysqlPasswordDefault.length > 0; + + const connectionResponse = await prompts( + [ + { + type: "text", + name: "mysqlHost", + message: "Publisher MySQL host", + initial: publisherConfig?.mysql.host || "localhost", + }, + { + type: "number", + name: "mysqlPort", + message: "Publisher MySQL port", + initial: publisherConfig?.mysql.port || 3306, + min: 0, + }, + { + type: "text", + name: "mysqlUser", + message: "Publisher MySQL username", + initial: publisherConfig?.mysql.user || "root", + }, + { + type: "password", + name: "mysqlPassword", + message: hasDefaultMysqlPassword + ? "Publisher MySQL password (leave blank to use the password set during dkg-node install)" + : "Publisher MySQL password", + validate: (value) => + hasDefaultMysqlPassword || value.length > 0 + ? true + : "Required for async publishing", + }, + { + type: "text", + name: "mysqlDatabase", + message: "Publisher MySQL database name", + initial: publisherConfig?.mysql.database || "dkg_publisher_db", + validate: (value) => + isValidMysqlIdentifier(value) || + "Use letters, numbers, and underscores only", + }, + { + type: "text", + name: "redisHost", + message: "Publisher Redis host", + initial: publisherConfig?.redis.host || "localhost", + }, + { + type: "number", + name: "redisPort", + message: "Publisher Redis port", + initial: publisherConfig?.redis.port || 6379, + min: 0, + }, + { + type: "password", + name: "redisPassword", + message: "Publisher Redis password (leave blank if no password)", + initial: publisherConfig?.redis.password || "", + }, + ], + promptOptions, + ); + + const primaryWallet = await ensurePrimaryWallet(envState.values.DKG_PUBLISH_WALLET); + const normalizedPrimaryWalletForEnv = stripPrivateKeyPrefix(primaryWallet); + const additionalWallets = await collectAdditionalWallets(); + const walletCount = 1 + additionalWallets.length; + + const advancedResponse = await prompts( + { + type: "confirm", + name: "advanced", + message: "Configure advanced worker and storage overrides?", + initial: false, + }, + promptOptions, + ); + + const recommendedWorkerCount = getRecommendedWorkerCount(walletCount); + const workerDefaultsResponse: { + workerCount: number; + pollFrequency?: number; + storagePath?: string; + storageBaseUrl?: string; + } = advancedResponse.advanced + ? await prompts( + [ + { + type: "number", + name: "workerCount", + message: `Publisher worker count (default: ${recommendedWorkerCount}, based on wallet count; concurrency auto-balances)`, + initial: + publisherConfig?.workerCount || + recommendedWorkerCount, + min: 1, + }, + { + type: "number", + name: "pollFrequency", + message: "Publisher poll frequency (ms)", + initial: publisherConfig?.pollFrequency || 2000, + min: 100, + }, + { + type: "text", + name: "storagePath", + message: "Publisher storage path", + initial: publisherConfig?.storagePath || "./data/publisher", + }, + { + type: "text", + name: "storageBaseUrl", + message: "Publisher storage base URL", + initial: + publisherConfig?.storageBaseUrl || + new URL("/storage", appUrl).toString().replace(/\/$/, ""), + validate: (value) => + isValidUrl(value) || "Provide a valid URL", + }, + ], + promptOptions, + ) + : { + workerCount: publisherConfig?.workerCount || recommendedWorkerCount, + }; + + const mysqlPassword = + connectionResponse.mysqlPassword?.trim() || mysqlPasswordDefault; + if (!mysqlPassword) { + throw new Error("Async publishing requires a MySQL password"); + } + + const publisherDefaults = buildPublisherDefaults(appUrl, mysqlPassword, { + mysqlHost: connectionResponse.mysqlHost, + mysqlPort: connectionResponse.mysqlPort, + mysqlUser: connectionResponse.mysqlUser, + mysqlDatabase: connectionResponse.mysqlDatabase, + redisHost: connectionResponse.redisHost, + redisPort: connectionResponse.redisPort, + redisPassword: connectionResponse.redisPassword, + workerCount: workerDefaultsResponse.workerCount, + pollFrequency: workerDefaultsResponse.pollFrequency, + storagePath: workerDefaultsResponse.storagePath, + storageBaseUrl: workerDefaultsResponse.storageBaseUrl, + }); + + const blockchain = envState.values.DKG_BLOCKCHAIN || "hardhat1:31337"; + const walletSeeds = [ + { privateKey: primaryWallet, blockchain }, + ...additionalWallets.map((privateKey) => ({ privateKey, blockchain })), + ]; + + const provisionResult = await addPublisherWallets( + publisherDefaults.databaseUrl, + walletSeeds, + ); + + await upsertAgentEnvValues( + { + DKGP_DATABASE_URL: publisherDefaults.databaseUrl, + REDIS_URL: publisherDefaults.redisUrl, + WORKER_COUNT: workerDefaultsResponse.workerCount, + ASYNC_PUBLISHING_ENABLED: true, + DKG_PUBLISH_WALLET: normalizedPrimaryWalletForEnv, + ...(advancedResponse.advanced + ? { + POLL_FREQUENCY: publisherDefaults.pollFrequency, + STORAGE_TYPE: "filesystem", + STORAGE_PATH: publisherDefaults.storagePath, + STORAGE_BASE_URL: publisherDefaults.storageBaseUrl, + } + : { + POLL_FREQUENCY: null, + STORAGE_TYPE: null, + STORAGE_PATH: null, + STORAGE_BASE_URL: null, + }), + }, + envState.envPath, + ); + + console.log( + `DKG Publisher plugin configuration updated. walletsInserted=${provisionResult.walletsInserted}`, + ); +} + +function printWalletTable( + wallets: Array<{ + id: number; + address: string; + blockchain: string; + isActive: boolean; + isLocked: boolean; + totalUses: number; + }>, +) { + if (!wallets.length) { + console.log("No wallets found in the DKG Publisher plugin database."); + return; + } + + console.log(""); + console.log("ID Active Locked Uses Blockchain Address"); + for (const wallet of wallets) { + const id = wallet.id.toString().padEnd(3, " "); + const active = (wallet.isActive ? "yes" : "no").padEnd(7, " "); + const locked = (wallet.isLocked ? "yes" : "no").padEnd(7, " "); + const uses = wallet.totalUses.toString().padEnd(5, " "); + const chain = wallet.blockchain.padEnd(15, " "); + console.log( + `${id}${active}${locked}${uses}${chain}${wallet.address}`, + ); + } +} + +async function runWalletManagementMode() { + printSection( + "Manage DKG Publisher Plugin Wallets", + "List, add, deactivate, or reactivate wallets in the DKG Publisher plugin database.", + ); + + const envState = await readAgentEnv(); + const appUrl = envState.values.EXPO_PUBLIC_MCP_URL || "http://localhost:9200"; + const publisherConfig = resolvePublisherConfigSafely( + envState.values, + appUrl, + ); + + if (!publisherConfig) { + throw new Error( + "DKGP_DATABASE_URL is missing in apps/agent/.env. Run DKG Publisher plugin config update first.", + ); + } + + const blockchain = envState.values.DKG_BLOCKCHAIN || "hardhat1:31337"; + let keepRunning = true; + + while (keepRunning) { + const wallets = await listPublisherWallets(publisherConfig.databaseUrl); + + const actionResponse = await prompts( + { + type: "select", + name: "action", + message: "Wallet management action", + choices: [ + { title: "List wallets", value: "list" }, + { title: "Add wallet(s)", value: "add" }, + { title: "Deactivate wallet", value: "deactivate" }, + { title: "Reactivate wallet", value: "reactivate" }, + { title: "Back", value: "back" }, + ], + initial: 0, + }, + promptOptions, + ); + + if (actionResponse.action === "back") { + keepRunning = false; + continue; + } + + if (actionResponse.action === "list") { + printWalletTable(wallets); + continue; + } + + if (actionResponse.action === "add") { + const additionalWallets = await collectAdditionalWallets({ + promptForFirstConfirmation: false, + }); + if (!additionalWallets.length) { + console.log("No wallets were added."); + continue; + } + + const result = await addPublisherWallets( + publisherConfig.databaseUrl, + additionalWallets.map((privateKey) => ({ privateKey, blockchain })), + ); + console.log(`Wallet add complete. walletsInserted=${result.walletsInserted}`); + continue; + } + + if (actionResponse.action === "deactivate") { + const activeWallets = wallets.filter((wallet) => wallet.isActive); + if (!activeWallets.length) { + console.log("No active wallets available to deactivate."); + continue; + } + + const targetResponse = await prompts( + { + type: "select", + name: "walletId", + message: "Choose wallet to deactivate", + choices: activeWallets.map((wallet) => ({ + title: `#${wallet.id} ${wallet.address} (${wallet.isLocked ? "locked" : "available"})`, + value: wallet.id, + })), + }, + promptOptions, + ); + + const selected = wallets.find((wallet) => wallet.id === targetResponse.walletId); + if (!selected) { + console.log("Wallet not found."); + continue; + } + + let forceUnlock = false; + if (selected.isLocked) { + if (!isStaleLock(selected.lockedAt)) { + console.log( + `Wallet #${selected.id} is currently locked. Wait for the active publish to finish before deactivating.`, + ); + continue; + } + + const unlockResponse = await prompts( + { + type: "confirm", + name: "forceUnlock", + message: + "Wallet appears locked for over 30 minutes. Force unlock and deactivate?", + initial: false, + }, + promptOptions, + ); + forceUnlock = unlockResponse.forceUnlock === true; + if (!forceUnlock) { + console.log("Deactivation cancelled."); + continue; + } + } + + const result = await setPublisherWalletActive( + publisherConfig.databaseUrl, + selected.id, + false, + { forceUnlock }, + ); + console.log( + `Wallet #${result.id} deactivated${result.forcedUnlock ? " (forced unlock applied)" : ""}.`, + ); + continue; + } + + if (actionResponse.action === "reactivate") { + const inactiveWallets = wallets.filter((wallet) => !wallet.isActive); + if (!inactiveWallets.length) { + console.log("No inactive wallets available to reactivate."); + continue; + } + + const targetResponse = await prompts( + { + type: "select", + name: "walletId", + message: "Choose wallet to reactivate", + choices: inactiveWallets.map((wallet) => ({ + title: `#${wallet.id} ${wallet.address}`, + value: wallet.id, + })), + }, + promptOptions, + ); + + const result = await setPublisherWalletActive( + publisherConfig.databaseUrl, + targetResponse.walletId, + true, + ); + console.log(`Wallet #${result.id} reactivated.`); + } + } +} + +async function runFreshResetMode() { + const envState = await readAgentEnv(); + const appUrl = envState.values.EXPO_PUBLIC_MCP_URL || "http://localhost:9200"; + const publisherConfig = resolvePublisherConfigSafely( + envState.values, + appUrl, + ); + + if (!publisherConfig) { + throw new Error( + "DKGP_DATABASE_URL is missing in apps/agent/.env. Run DKG Publisher plugin config update first.", + ); + } + + printSection( + "Fresh Setup (Will Delete Existing Data)", + "This will drop DKG Publisher plugin tables and recreate them. Existing async publishing queue/status history will be removed.", + ); + + const confirmationResponse = await prompts( + { + type: "text", + name: "confirmation", + message: 'Type "RESET DKG PUBLISHER DATA" to continue', + validate: (value) => + value === "RESET DKG PUBLISHER DATA" || + 'Type exactly: RESET DKG PUBLISHER DATA', + }, + promptOptions, + ); + + if (confirmationResponse.confirmation !== "RESET DKG PUBLISHER DATA") { + console.log("DKG Publisher plugin fresh setup cancelled."); + return; + } + + const primaryWallet = await ensurePrimaryWallet(envState.values.DKG_PUBLISH_WALLET); + const normalizedPrimaryWalletForEnv = stripPrivateKeyPrefix(primaryWallet); + const additionalWallets = await collectAdditionalWallets(); + const blockchain = envState.values.DKG_BLOCKCHAIN || "hardhat1:31337"; + const walletSeeds = [ + { privateKey: primaryWallet, blockchain }, + ...additionalWallets.map((privateKey) => ({ privateKey, blockchain })), + ]; + + const result = await resetPublisherDatabase( + publisherConfig.databaseUrl, + walletSeeds, + ); + + await upsertAgentEnvValues( + { + ASYNC_PUBLISHING_ENABLED: true, + DKG_PUBLISH_WALLET: normalizedPrimaryWalletForEnv, + }, + envState.envPath, + ); + + console.log( + `DKG Publisher plugin fresh setup complete. droppedTables=${result.droppedTables.length}, walletsInserted=${result.walletsInserted}`, + ); +} + +async function shouldContinueManaging() { + const response = await prompts( + { + type: "confirm", + name: "continueManaging", + message: "Run another DKG Publisher plugin action?", + initial: false, + }, + promptOptions, + ); + + return response.continueManaging === true; +} + +async function main() { + printBanner(); + let envState: AgentEnvState; + try { + envState = await readAgentEnv(); + } catch (error: any) { + if (error?.code === "ENOENT") { + throw new Error( + "apps/agent/.env was not found. Run npm run script:setup first.", + ); + } + throw error; + } + const appUrl = envState.values.EXPO_PUBLIC_MCP_URL || "http://localhost:9200"; + let publisherConfig = resolvePublisherConfigSafely( + envState.values, + appUrl, + ); + + if (!publisherConfig) { + console.log( + "DKG Publisher plugin config was not found in apps/agent/.env. Starting config update mode.", + ); + await runUpdateConfigMode(); + const continueAfterAutoSetup = await shouldContinueManaging(); + if (!continueAfterAutoSetup) { + console.log("DKG Publisher plugin management finished."); + return; + } + envState = await readAgentEnv(); + publisherConfig = resolvePublisherConfigSafely( + envState.values, + envState.values.EXPO_PUBLIC_MCP_URL || "http://localhost:9200", + ); + } + + if (!publisherConfig) { + throw new Error( + "DKG Publisher plugin configuration is still missing after setup. Verify DKGP_DATABASE_URL in apps/agent/.env.", + ); + } + + let done = false; + while (!done) { + const response = await prompts( + { + type: "select", + name: "mode", + message: "Choose DKG Publisher plugin management mode", + choices: [ + { title: "Update DKG Publisher plugin config", value: "config" }, + { title: "Manage wallets", value: "wallets" }, + { title: "Fresh setup (will delete existing data)", value: "reset" }, + { title: "Exit", value: "exit" }, + ], + initial: 0, + }, + promptOptions, + ); + + if (response.mode === "exit") { + done = true; + continue; + } + + try { + if (response.mode === "config") { + await runUpdateConfigMode(); + } else if (response.mode === "wallets") { + await runWalletManagementMode(); + } else if (response.mode === "reset") { + await runFreshResetMode(); + } + + const continueManaging = await shouldContinueManaging(); + if (!continueManaging) { + done = true; + } + } catch (error: any) { + console.error(`DKG Publisher plugin management error: ${error.message}`); + } + } + + console.log("DKG Publisher plugin management finished."); +} + +main().catch((error) => { + console.error("Error occurred during DKG Publisher plugin management:", error); + process.exit(1); +}); diff --git a/apps/agent/src/server/scripts/setup.ts b/apps/agent/src/server/scripts/setup.ts index 4715e662..06670bf0 100644 --- a/apps/agent/src/server/scripts/setup.ts +++ b/apps/agent/src/server/scripts/setup.ts @@ -325,11 +325,11 @@ async function setup() { ); if (enginePassword.status === "missing-file") { console.log( - `Publisher MySQL defaults were not found at ${enginePassword.envPath}. Use advanced async setup if you want to provide them manually.`, + `DKG Publisher plugin MySQL defaults were not found at ${enginePassword.envPath}. Use advanced async setup if you want to provide them manually.`, ); } else if (enginePassword.status === "missing-key") { console.log( - `Publisher MySQL defaults are incomplete in ${enginePassword.envPath} because REPOSITORY_PASSWORD is missing. Use advanced async setup if you want to provide the password manually.`, + `DKG Publisher plugin MySQL defaults are incomplete in ${enginePassword.envPath} because REPOSITORY_PASSWORD is missing. Use advanced async setup if you want to provide the password manually.`, ); } @@ -393,7 +393,7 @@ async function setup() { type: "select", name: "asyncPublishingMode", message: - "Enable async publishing on the DKG for smoother publishing, easier status tracking, and queue management?", + "Enable async publishing (DKG Publisher plugin) on the DKG for smoother publishing, easier status tracking, and queue management?", choices: asyncPublishingChoices, initial: 0, }, @@ -403,7 +403,7 @@ async function setup() { publishingResponse.asyncPublishingMode === "advanced" ? (printSection( "Advanced Async Publishing", - "Provide MySQL, Redis, wallet, worker, and storage overrides for the DKG Publisher Plugin.", + "Provide MySQL, Redis, wallet, worker, and storage overrides for the DKG Publisher plugin (async publishing).", ), await collectAdvancedPublisherOptions(enginePassword.status === "found")) : null; @@ -531,7 +531,7 @@ async function setup() { } catch (error: any) { publisherProvisionError = error.message; console.warn( - `Publisher provisioning warning: ${publisherProvisionError}`, + `DKG Publisher plugin provisioning warning: ${publisherProvisionError}`, ); } } @@ -644,12 +644,12 @@ To create new users, run 'npm run script:createUser' inside of the agent directo if (publisherDefaults) { if (publisherProvisionResult) { console.log( - `Publisher DB ready: created=${publisherProvisionResult.databaseCreated}, walletsInserted=${publisherProvisionResult.walletsInserted}`, + `DKG Publisher plugin DB ready: created=${publisherProvisionResult.databaseCreated}, walletsInserted=${publisherProvisionResult.walletsInserted}`, ); } if (publisherProvisionError) { console.log( - `Publisher provisioning warning: ${publisherProvisionError}. Async publishing was left disabled in apps/agent/.env.`, + `DKG Publisher plugin provisioning warning: ${publisherProvisionError}. Async publishing was left disabled in apps/agent/.env.`, ); } } diff --git a/apps/agent/src/server/setupPublisher.ts b/apps/agent/src/server/setupPublisher.ts index 946adc2a..afad1bfc 100644 --- a/apps/agent/src/server/setupPublisher.ts +++ b/apps/agent/src/server/setupPublisher.ts @@ -1,6 +1,7 @@ import path from "path"; import { promises as fs } from "fs"; import dotenv from "dotenv"; +import mysql from "mysql2/promise"; import { provisionPublisherDatabase } from "@dkg/plugin-dkg-publisher/provision"; import { isValidPrivateKey, @@ -31,12 +32,162 @@ export interface PublisherAdvancedOptions { storageBaseUrl?: string; } +export interface AgentEnvState { + envPath: string; + content: string; + values: Record; +} + +export interface PublisherMysqlConfig { + host: string; + port: number; + user: string; + password: string; + database: string; +} + +export interface PublisherRedisConfig { + host: string; + port: number; + password: string; +} + +export interface PublisherResolvedConfig { + databaseUrl: string; + redisUrl: string; + workerCount: number; + pollFrequency: number; + storagePath: string; + storageBaseUrl: string; + mysql: PublisherMysqlConfig; + redis: PublisherRedisConfig; +} + +export interface PublisherWalletRecord { + id: number; + address: string; + blockchain: string; + isActive: boolean; + isLocked: boolean; + lockedAt: Date | null; + lastUsedAt: Date | null; + totalUses: number; + successfulUses: number; + failedUses: number; + createdAt: Date | null; +} + +export interface PublisherWalletActivationResult { + id: number; + address: string; + isActive: boolean; + wasLocked: boolean; + forcedUnlock: boolean; +} + +export interface PublisherResetResult { + droppedTables: string[]; + databaseCreated: boolean; + walletsInserted: number; +} + const MYSQL_IDENTIFIER_PATTERN = /^[A-Za-z0-9_]+$/; +const ENV_KEY_PATTERN = /^\s*([A-Za-z_][A-Za-z0-9_]*)\s*=/; +const DEFAULT_APP_URL = "http://localhost:9200"; +const DEFAULT_STORAGE_PATH = "./data/publisher"; +const DEFAULT_POLL_FREQUENCY = 2000; +const PUBLISHER_TABLES = [ + "__drizzle_migrations", + "wallet_metrics", + "publishing_attempts", + "assets", + "wallets", + "batches", + "metrics_hourly", +] as const; export function resolveNodeRoot(currentWorkingDirectory = process.cwd()) { return path.resolve(currentWorkingDirectory, "../.."); } +export function resolveAgentEnvPath(currentWorkingDirectory = process.cwd()) { + return path.resolve(currentWorkingDirectory, ".env"); +} + +function parsePositiveInt(value: string | undefined, fallback: number) { + const parsed = Number.parseInt(value || "", 10); + return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; +} + +function serializeEnvValue(value: string | number | boolean) { + if (typeof value === "number" || typeof value === "boolean") { + return `${value}`; + } + + return `"${value.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`; +} + +function parseMysqlConnectionString(databaseUrl: string): PublisherMysqlConfig { + const parsed = new URL(databaseUrl); + const database = decodeURIComponent(parsed.pathname.replace(/^\//, "")); + + if (!database) { + throw new Error("DKGP_DATABASE_URL must include a database name"); + } + + if (!MYSQL_IDENTIFIER_PATTERN.test(database)) { + throw new Error( + "DKGP_DATABASE_URL contains an invalid database name. Use letters, numbers, and underscores only.", + ); + } + + return { + host: parsed.hostname || "localhost", + port: parsePositiveInt(parsed.port, 3306), + user: decodeURIComponent(parsed.username || "root"), + password: decodeURIComponent(parsed.password || ""), + database, + }; +} + +function parseRedisConnectionString(redisUrl: string): PublisherRedisConfig { + const parsed = new URL(redisUrl); + return { + host: parsed.hostname || "localhost", + port: parsePositiveInt(parsed.port, 6379), + password: decodeURIComponent(parsed.password || ""), + }; +} + +function normalizeRedisUrlCandidate(value: string) { + const trimmed = value.trim(); + if (!trimmed) { + return ""; + } + + if (trimmed.startsWith("redis://") || trimmed.startsWith("rediss://")) { + return trimmed; + } + + if (!trimmed.includes("://")) { + return `redis://${trimmed}`; + } + + return trimmed; +} + +async function withPublisherConnection( + databaseUrl: string, + operation: (connection: mysql.Connection) => Promise, +) { + const connection = await mysql.createConnection(databaseUrl); + try { + return await operation(connection); + } finally { + await connection.end(); + } +} + export async function resolveEngineMysqlPassword(): Promise { const envPath = process.env.DKG_ENGINE_ENV_PATH || @@ -100,6 +251,90 @@ export function isValidMysqlIdentifier(value: string) { export { isValidPrivateKey, normalizePrivateKey, stripPrivateKeyPrefix }; +export async function readAgentEnv( + envPath = resolveAgentEnvPath(), +): Promise { + const content = await fs.readFile(envPath, "utf8"); + return { + envPath, + content, + values: dotenv.parse(content), + }; +} + +export async function upsertAgentEnvValues( + updates: Record, + envPath = resolveAgentEnvPath(), +) { + let existingContent = ""; + try { + existingContent = await fs.readFile(envPath, "utf8"); + } catch (error: any) { + if (error?.code !== "ENOENT") { + throw error; + } + } + + const lines = existingContent.length + ? existingContent.split(/\r?\n/) + : ([] as string[]); + const pending = new Map(); + + for (const [key, value] of Object.entries(updates)) { + if (value === undefined) { + continue; + } + if (value === null) { + pending.set(key, null); + continue; + } + pending.set(key, `${key}=${serializeEnvValue(value)}`); + } + + const nextLines: string[] = []; + const seenEnvKeys = new Set(); + for (const line of lines) { + const match = line.match(ENV_KEY_PATTERN); + if (!match) { + nextLines.push(line); + continue; + } + + const key = match[1]; + if (!key) { + nextLines.push(line); + continue; + } + + if (seenEnvKeys.has(key)) { + // Keep only the first occurrence for each env key to avoid ambiguous duplicates. + continue; + } + seenEnvKeys.add(key); + + const replacement = pending.get(key); + if (replacement !== undefined) { + if (replacement !== null) { + nextLines.push(replacement); + } + pending.delete(key); + continue; + } + + nextLines.push(line); + } + + for (const replacement of pending.values()) { + if (replacement !== null) { + nextLines.push(replacement); + } + } + + const nextContent = `${nextLines.join("\n").replace(/\n*$/, "")}\n`; + await fs.writeFile(envPath, nextContent, "utf8"); + return readAgentEnv(envPath); +} + export function buildRedisUrl(options: PublisherAdvancedOptions = {}) { if (options.redisUrl) { return options.redisUrl; @@ -130,6 +365,51 @@ export function buildPublisherDefaults( }; } +export function resolvePublisherConfigFromAgentEnv( + envValues: Record, + appUrl = DEFAULT_APP_URL, +): PublisherResolvedConfig | null { + const databaseUrl = envValues.DKGP_DATABASE_URL; + if (!databaseUrl) { + return null; + } + + const mysql = parseMysqlConnectionString(databaseUrl); + const fallbackRedisUrl = `redis://${envValues.REDIS_PASSWORD ? `:${encodeURIComponent(envValues.REDIS_PASSWORD)}@` : ""}${envValues.REDIS_HOST || "localhost"}:${envValues.REDIS_PORT || "6379"}`; + const redisUrlCandidate = normalizeRedisUrlCandidate(envValues.REDIS_URL || ""); + + let redisUrl = fallbackRedisUrl; + let redis: PublisherRedisConfig; + try { + if (redisUrlCandidate) { + redis = parseRedisConnectionString(redisUrlCandidate); + redisUrl = redisUrlCandidate; + } else { + redis = parseRedisConnectionString(fallbackRedisUrl); + } + } catch { + // Recover from malformed REDIS_URL in existing env by falling back to host/port/password. + redis = parseRedisConnectionString(fallbackRedisUrl); + redisUrl = fallbackRedisUrl; + } + + return { + databaseUrl, + redisUrl, + workerCount: parsePositiveInt(envValues.WORKER_COUNT, 1), + pollFrequency: parsePositiveInt( + envValues.POLL_FREQUENCY, + DEFAULT_POLL_FREQUENCY, + ), + storagePath: envValues.STORAGE_PATH || DEFAULT_STORAGE_PATH, + storageBaseUrl: + envValues.STORAGE_BASE_URL || + new URL("/storage", appUrl).toString().replace(/\/$/, ""), + mysql, + redis, + }; +} + export async function provisionAsyncPublishing( databaseUrl: string, walletSeeds: Array<{ @@ -139,3 +419,135 @@ export async function provisionAsyncPublishing( ) { return provisionPublisherDatabase(databaseUrl, walletSeeds); } + +export async function listPublisherWallets( + databaseUrl: string, +): Promise { + return withPublisherConnection(databaseUrl, async (connection) => { + const [rows] = await connection.query( + `SELECT + id, + address, + blockchain, + is_active AS isActive, + is_locked AS isLocked, + locked_at AS lockedAt, + last_used_at AS lastUsedAt, + total_uses AS totalUses, + successful_uses AS successfulUses, + failed_uses AS failedUses, + created_at AS createdAt + FROM wallets + ORDER BY created_at ASC, id ASC`, + ); + + return rows.map((row) => ({ + id: Number(row.id), + address: row.address, + blockchain: row.blockchain, + isActive: Boolean(row.isActive), + isLocked: Boolean(row.isLocked), + lockedAt: row.lockedAt ? new Date(row.lockedAt) : null, + lastUsedAt: row.lastUsedAt ? new Date(row.lastUsedAt) : null, + totalUses: Number(row.totalUses || 0), + successfulUses: Number(row.successfulUses || 0), + failedUses: Number(row.failedUses || 0), + createdAt: row.createdAt ? new Date(row.createdAt) : null, + })); + }); +} + +export async function addPublisherWallets( + databaseUrl: string, + walletSeeds: Array<{ privateKey: string; blockchain: string }>, +) { + if (!walletSeeds.length) { + return { walletsInserted: 0 }; + } + + const result = await provisionPublisherDatabase(databaseUrl, walletSeeds); + return { walletsInserted: result.walletsInserted }; +} + +export async function setPublisherWalletActive( + databaseUrl: string, + walletId: number, + isActive: boolean, + options: { forceUnlock?: boolean } = {}, +): Promise { + return withPublisherConnection(databaseUrl, async (connection) => { + const [rows] = await connection.query( + `SELECT id, address, is_locked AS isLocked FROM wallets WHERE id = ? LIMIT 1`, + [walletId], + ); + + if (!rows.length) { + throw new Error(`Wallet with id ${walletId} was not found`); + } + + const row = rows[0]; + const wasLocked = Boolean(row.isLocked); + + if (!isActive && wasLocked && !options.forceUnlock) { + throw new Error( + `Wallet ${walletId} is currently locked and cannot be deactivated without force unlock`, + ); + } + + let forcedUnlock = false; + if (!isActive && wasLocked && options.forceUnlock) { + await connection.execute( + `UPDATE wallets SET is_locked = FALSE, locked_at = NULL WHERE id = ?`, + [walletId], + ); + forcedUnlock = true; + } + + await connection.execute(`UPDATE wallets SET is_active = ? WHERE id = ?`, [ + isActive, + walletId, + ]); + + return { + id: Number(row.id), + address: row.address, + isActive, + wasLocked, + forcedUnlock, + }; + }); +} + +export async function resetPublisherDatabase( + databaseUrl: string, + walletSeeds: Array<{ + privateKey: string; + blockchain: string; + }> = [], +): Promise { + try { + await withPublisherConnection(databaseUrl, async (connection) => { + await connection.execute("SET FOREIGN_KEY_CHECKS = 0"); + try { + for (const tableName of PUBLISHER_TABLES) { + await connection.execute(`DROP TABLE IF EXISTS \`${tableName}\``); + } + } finally { + await connection.execute("SET FOREIGN_KEY_CHECKS = 1"); + } + }); + } catch (error: any) { + // Database may not exist yet; provisioning below will create it. + if (error?.code !== "ER_BAD_DB_ERROR") { + throw error; + } + } + + const provisionResult = await provisionPublisherDatabase(databaseUrl, walletSeeds); + + return { + droppedTables: [...PUBLISHER_TABLES], + databaseCreated: provisionResult.databaseCreated, + walletsInserted: provisionResult.walletsInserted, + }; +} diff --git a/apps/agent/tests/unit/setupPublisher.spec.ts b/apps/agent/tests/unit/setupPublisher.spec.ts new file mode 100644 index 00000000..28f404c6 --- /dev/null +++ b/apps/agent/tests/unit/setupPublisher.spec.ts @@ -0,0 +1,140 @@ +import { expect } from "chai"; +import os from "os"; +import path from "path"; +import { mkdtemp, readFile, rm, writeFile } from "fs/promises"; + +import { + resolvePublisherConfigFromAgentEnv, + upsertAgentEnvValues, +} from "../../src/server/setupPublisher"; + +describe("setupPublisher helpers", () => { + it("resolves publisher config from consolidated agent env values", () => { + const resolved = resolvePublisherConfigFromAgentEnv( + { + DKGP_DATABASE_URL: + "mysql://root:my%21pass@localhost:3306/dkg_publisher_db", + REDIS_URL: "redis://:redis%23secret@localhost:6379", + WORKER_COUNT: "3", + POLL_FREQUENCY: "1500", + STORAGE_PATH: "./publisher-data", + STORAGE_BASE_URL: "http://localhost:9200/storage", + }, + "http://localhost:9200", + ); + + expect(resolved).to.not.equal(null); + expect(resolved!.mysql.user).to.equal("root"); + expect(resolved!.mysql.password).to.equal("my!pass"); + expect(resolved!.mysql.database).to.equal("dkg_publisher_db"); + expect(resolved!.redis.password).to.equal("redis#secret"); + expect(resolved!.workerCount).to.equal(3); + expect(resolved!.pollFrequency).to.equal(1500); + expect(resolved!.storagePath).to.equal("./publisher-data"); + }); + + it("resolves redis settings from host/port/password when REDIS_URL is missing", () => { + const resolved = resolvePublisherConfigFromAgentEnv( + { + DKGP_DATABASE_URL: + "mysql://root:password@localhost:3306/dkg_publisher_db", + REDIS_HOST: "redis.internal", + REDIS_PORT: "6380", + REDIS_PASSWORD: "redis#secret", + }, + "http://localhost:9200", + ); + + expect(resolved).to.not.equal(null); + expect(resolved!.redisUrl).to.equal( + "redis://:redis%23secret@redis.internal:6380", + ); + expect(resolved!.redis.host).to.equal("redis.internal"); + expect(resolved!.redis.port).to.equal(6380); + expect(resolved!.redis.password).to.equal("redis#secret"); + }); + + it("recovers from legacy REDIS_URL format without protocol", () => { + const resolved = resolvePublisherConfigFromAgentEnv( + { + DKGP_DATABASE_URL: + "mysql://root:password@localhost:3306/dkg_publisher_db", + REDIS_URL: "localhost:6379", + }, + "http://localhost:9200", + ); + + expect(resolved).to.not.equal(null); + expect(resolved!.redisUrl).to.equal("redis://localhost:6379"); + expect(resolved!.redis.host).to.equal("localhost"); + expect(resolved!.redis.port).to.equal(6379); + }); + + it("updates existing keys and appends missing keys when writing agent env", async () => { + const tempDir = await mkdtemp(path.join(os.tmpdir(), "dkg-agent-env-")); + const envPath = path.join(tempDir, ".env"); + + try { + await writeFile( + envPath, + [ + 'ASYNC_PUBLISHING_ENABLED="false"', + 'DKG_BLOCKCHAIN="otp:20430"', + ].join("\n"), + "utf8", + ); + + await upsertAgentEnvValues( + { + ASYNC_PUBLISHING_ENABLED: true, + DKGP_DATABASE_URL: "mysql://root:password@localhost:3306/dkg_publisher_db", + }, + envPath, + ); + + const updated = await readFile(envPath, "utf8"); + expect(updated).to.contain("ASYNC_PUBLISHING_ENABLED=true"); + expect(updated).to.contain('DKG_BLOCKCHAIN="otp:20430"'); + expect(updated).to.contain( + 'DKGP_DATABASE_URL="mysql://root:password@localhost:3306/dkg_publisher_db"', + ); + } finally { + await rm(tempDir, { recursive: true, force: true }); + } + }); + + it("removes keys and collapses duplicate entries when writing agent env", async () => { + const tempDir = await mkdtemp(path.join(os.tmpdir(), "dkg-agent-env-")); + const envPath = path.join(tempDir, ".env"); + + try { + await writeFile( + envPath, + [ + 'POLL_FREQUENCY="5000"', + 'POLL_FREQUENCY="9000"', + 'ASYNC_PUBLISHING_ENABLED="false"', + ].join("\n"), + "utf8", + ); + + await upsertAgentEnvValues( + { + ASYNC_PUBLISHING_ENABLED: true, + POLL_FREQUENCY: null, + }, + envPath, + ); + + const updated = await readFile(envPath, "utf8"); + const asyncMatches = updated.match(/^ASYNC_PUBLISHING_ENABLED=/gm) || []; + const pollMatches = updated.match(/^POLL_FREQUENCY=/gm) || []; + + expect(asyncMatches.length).to.equal(1); + expect(updated).to.contain("ASYNC_PUBLISHING_ENABLED=true"); + expect(pollMatches.length).to.equal(0); + } finally { + await rm(tempDir, { recursive: true, force: true }); + } + }); +}); diff --git a/packages/plugin-dkg-publisher/README.md b/packages/plugin-dkg-publisher/README.md index f9f585c0..40452e67 100644 --- a/packages/plugin-dkg-publisher/README.md +++ b/packages/plugin-dkg-publisher/README.md @@ -32,8 +32,16 @@ runtime env into `apps/agent/.env` and provision the Publisher MySQL database. In advanced mode, setup can also seed additional publishing wallets into the Publisher wallet pool. -`npm run setup` in this package remains as a legacy compatibility path for older -workflows that still rely on `.env.publisher`. +For post-setup operations (update Publisher config, manage wallets, or reset +Publisher data), use: + +```bash +cd apps/agent +npm run script:publisher +``` + +`npm run setup` in this package is legacy compatibility for older workflows that +still rely on `.env.publisher`. ### 2. Start the Application @@ -107,6 +115,14 @@ npm run script:setup # The primary wallet is seeded automatically and setup can add more wallets ``` +**After setup:** + +```bash +cd apps/agent +npm run script:publisher +# Choose "Manage wallets" +``` + **After 5 minutes**, workers automatically restart with new concurrency. No application restart needed! ## Monitoring From 08f1f178f936730698e5b7b295f260ad2dcf823b Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Thu, 5 Mar 2026 13:17:40 +0100 Subject: [PATCH 05/10] [improvement] Clarify Publisher async MCP semantics and canonicalize publishOptions input Align Publisher vs Essentials tool descriptions to reduce sync/async ambiguity. Expand knowledge-asset-publish controls via publishOptions (privacy/priority/epochs/maxAttempts) without retrieval-tool behavior changes. --- .../src/plugins/dkg-tools.ts | 5 +- .../plugin-dkg-publisher/src/mcp/tools.ts | 87 +++++++++++++++---- .../tests/dkg-publisher.spec.ts | 33 ++++++- 3 files changed, 106 insertions(+), 19 deletions(-) diff --git a/packages/plugin-dkg-essentials/src/plugins/dkg-tools.ts b/packages/plugin-dkg-essentials/src/plugins/dkg-tools.ts index 3b91de92..c47e25f5 100644 --- a/packages/plugin-dkg-essentials/src/plugins/dkg-tools.ts +++ b/packages/plugin-dkg-essentials/src/plugins/dkg-tools.ts @@ -179,8 +179,9 @@ export default defineDkgPlugin((ctx, mcp, api) => { { title: "DKG Knowledge Asset create tool", description: - "A tool for creating and publishing Knowledge Assets on OriginTrail Decentralized Knowledge Graph (DKG), " + - "taking either a single JSON-LD string or a single file id as input. " + + "A synchronous direct-publish tool for creating and publishing Knowledge Assets on OriginTrail Decentralized Knowledge Graph (DKG). " + + "Use this when you want immediate publish execution and direct UAL output. " + + "Takes either a single JSON-LD string or a single file id as input. " + "Optionally, you can specify privacy as 'private' or 'public' (default: 'private').", inputSchema: { jsonld: z diff --git a/packages/plugin-dkg-publisher/src/mcp/tools.ts b/packages/plugin-dkg-publisher/src/mcp/tools.ts index cfcf0060..001bc1bc 100644 --- a/packages/plugin-dkg-publisher/src/mcp/tools.ts +++ b/packages/plugin-dkg-publisher/src/mcp/tools.ts @@ -21,9 +21,10 @@ export function registerMcpTools( { title: "Publish Knowledge Asset", description: - "Register a JSON-LD asset for publishing to the DKG. " + - "You can provide the content directly as a JSON object, or provide a blobId to load content from a previously uploaded file. " + - "Use blobId for large files. Use the MCP query tools to check status and view recent published assets.", + "Register a JSON-LD asset for publishing to the DKG through the DKG Publisher plugin async queue. " + + "This tool queues publishing and returns a tracking record, not an immediate final UAL result. " + + "For direct synchronous publishing with immediate UAL output, use the Essentials `dkg-create` tool. " + + "You can provide content directly as a JSON object, or provide a blobId to load content from a previously uploaded file.", inputSchema: { content: z .object({}) @@ -44,7 +45,15 @@ export function registerMcpTools( sourceId: z.string().optional(), }) .optional(), - privacy: z.enum(["private", "public"]).optional(), + publishOptions: z + .object({ + privacy: z.enum(["private", "public"]).optional(), + priority: z.number().min(1).max(100).optional(), + epochs: z.number().optional(), + maxAttempts: z.number().optional(), + }) + .optional() + .describe("Optional async publishing controls."), }, }, async (input: any, req: any) => { @@ -91,12 +100,25 @@ export function registerMcpTools( } const assetService = serviceContainer.get("assetService"); + const resolvedPrivacy = input.publishOptions?.privacy ?? "private"; + const resolvedPriority = input.publishOptions?.priority; + const resolvedEpochs = input.publishOptions?.epochs; + const resolvedMaxAttempts = input.publishOptions?.maxAttempts; const assetInput = { content, metadata: input.metadata, publishOptions: { - privacy: input.privacy || "private", + privacy: resolvedPrivacy, + ...(typeof resolvedPriority === "number" + ? { priority: resolvedPriority } + : {}), + ...(typeof resolvedEpochs === "number" + ? { epochs: resolvedEpochs } + : {}), + ...(typeof resolvedMaxAttempts === "number" + ? { maxAttempts: resolvedMaxAttempts } + : {}), }, }; @@ -141,9 +163,15 @@ export function registerMcpTools( "knowledge-asset-status-by-content-id", { title: "Get Knowledge Asset Information by Content ID", - description: "Check, lookup, show, or query a knowledge asset by its JSON-LD @id (URN). Use this when the user provides a URN like 'urn:test:asset:...' or asks about a specific asset ID. Returns status, UAL, transaction hash, and publishing details.", + description: + "Check async publishing status for a specific JSON-LD @id (URN) that was submitted through the DKG Publisher plugin queue. " + + "Returns tracked Publisher status details (status, UAL if published, transaction hash, attempts, and errors).", inputSchema: { - contentId: z.string().describe("The @id from the JSON-LD content (e.g., 'urn:test:asset:manual-test-1')"), + contentId: z + .string() + .describe( + "The @id from the JSON-LD content (e.g., 'urn:test:asset:manual-test-1')", + ), }, }, async (input: any, req: any) => { @@ -189,11 +217,26 @@ export function registerMcpTools( "knowledge-asset-list-recent", { title: "List Recent Knowledge Assets", - description: "Show, list, or display recent knowledge assets. Use when user asks 'show me recent assets', 'what was published', 'last X assets', 'publishes 200-500', etc. Can filter by status (published, failed, publishing, queued). Supports pagination with offset for large queries.", + description: + "List recent async publishing records tracked by the DKG Publisher plugin. " + + "Use this for queue/history views (optionally filtered by status) with pagination support.", inputSchema: { - limit: z.number().min(1).default(20).optional().describe("Number of assets to return (default: 20)"), - offset: z.number().min(0).default(0).optional().describe("Number of assets to skip (for pagination, default: 0)"), - status: z.enum(["published", "failed", "publishing", "queued"]).optional().describe("Filter by status (optional)"), + limit: z + .number() + .min(1) + .default(20) + .optional() + .describe("Number of assets to return (default: 20)"), + offset: z + .number() + .min(0) + .default(0) + .optional() + .describe("Number of assets to skip (for pagination, default: 0)"), + status: z + .enum(["published", "failed", "publishing", "queued"]) + .optional() + .describe("Filter by status (optional)"), }, }, async (input: any, req: any) => { @@ -282,11 +325,25 @@ export function registerMcpTools( "knowledge-asset-query-by-status", { title: "Find Knowledge Assets by Status", - description: "Find, show, list, or query knowledge assets by publishing status. Use when user asks 'show me all published', 'failed assets', 'what's publishing', 'publishes 100-200', etc. Supports statuses: published (successfully published), failed (publishing failed), publishing (currently being published), queued (waiting to publish). Supports pagination with offset for large queries.", + description: + "Query async publishing records in the DKG Publisher plugin by a required status. " + + "Use this for focused queue/operations views (published, failed, publishing, queued) with pagination.", inputSchema: { - status: z.enum(["published", "failed", "publishing", "queued"]).describe("The status to filter by"), - limit: z.number().min(1).default(20).optional().describe("Maximum number of results (default: 20)"), - offset: z.number().min(0).default(0).optional().describe("Number of assets to skip (for pagination, default: 0)"), + status: z + .enum(["published", "failed", "publishing", "queued"]) + .describe("The status to filter by"), + limit: z + .number() + .min(1) + .default(20) + .optional() + .describe("Maximum number of results (default: 20)"), + offset: z + .number() + .min(0) + .default(0) + .optional() + .describe("Number of assets to skip (for pagination, default: 0)"), }, }, async (input: any, req: any) => { diff --git a/packages/plugin-dkg-publisher/tests/dkg-publisher.spec.ts b/packages/plugin-dkg-publisher/tests/dkg-publisher.spec.ts index ec20da41..b0ca7e4a 100644 --- a/packages/plugin-dkg-publisher/tests/dkg-publisher.spec.ts +++ b/packages/plugin-dkg-publisher/tests/dkg-publisher.spec.ts @@ -83,8 +83,18 @@ describe("@dkg/plugin-dkg-publisher checks", () => { const publishTool = tools.find((tool) => tool.name === "knowledge-asset-publish"); expect(publishTool).to.not.equal(undefined); expect(publishTool!.title).to.equal("Publish Knowledge Asset"); - expect(publishTool!.description).to.equal("Register a JSON-LD asset for publishing to the DKG"); + expect(publishTool!.description).to.include( + "Register a JSON-LD asset for publishing to the DKG", + ); + expect(publishTool!.description).to.include("async queue"); expect(publishTool!.inputSchema).to.not.equal(undefined); + expect((publishTool!.inputSchema as any).publishOptions).to.not.equal( + undefined, + ); + expect((publishTool!.inputSchema as any).privacy).to.equal(undefined); + expect((publishTool!.inputSchema as any).priority).to.equal(undefined); + expect((publishTool!.inputSchema as any).epochs).to.equal(undefined); + expect((publishTool!.inputSchema as any).maxAttempts).to.equal(undefined); }); it("should register API routes correctly", async () => { @@ -159,6 +169,25 @@ describe("@dkg/plugin-dkg-publisher checks", () => { }); describe("Data Structure Processing", () => { + it("should accept optional async publish controls", async () => { + const withPublishControls = createTestAsset({ + publishOptions: { + privacy: "public", + priority: 25, + epochs: 4, + maxAttempts: 5, + }, + }); + + const result = await mockMcpClient.callTool({ + name: "knowledge-asset-publish", + arguments: withPublishControls, + }); + + expect(result.content).to.be.an("array"); + expect(result.content).to.have.length.greaterThan(0); + }); + it("should process different content types correctly", async () => { const stringContent = createTestAsset({ content: { @@ -362,4 +391,4 @@ describe("@dkg/plugin-dkg-publisher checks", () => { } }); }); -}); \ No newline at end of file +}); From 7660a51b5fbebcfd527394b7f390d99515e831fe Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Thu, 5 Mar 2026 13:33:32 +0100 Subject: [PATCH 06/10] [improvement] Remove legacy publisher setup script and references Delete packages/plugin-dkg-publisher/setup.js and remove setup entrypoints from plugin package scripts. Keep Agent-owned setup/management as the only supported flow and update legacy bootstrap wording. --- packages/plugin-dkg-publisher/README.md | 3 - .../package-addition.json | 1 - packages/plugin-dkg-publisher/package.json | 1 - packages/plugin-dkg-publisher/setup.js | 1125 ----------------- .../src/database/bootstrap.ts | 14 +- 5 files changed, 7 insertions(+), 1137 deletions(-) delete mode 100644 packages/plugin-dkg-publisher/setup.js diff --git a/packages/plugin-dkg-publisher/README.md b/packages/plugin-dkg-publisher/README.md index 40452e67..2d47c57c 100644 --- a/packages/plugin-dkg-publisher/README.md +++ b/packages/plugin-dkg-publisher/README.md @@ -40,9 +40,6 @@ cd apps/agent npm run script:publisher ``` -`npm run setup` in this package is legacy compatibility for older workflows that -still rely on `.env.publisher`. - ### 2. Start the Application The plugin runs automatically when you start your DKG Agent: diff --git a/packages/plugin-dkg-publisher/package-addition.json b/packages/plugin-dkg-publisher/package-addition.json index f1da997f..5481597d 100644 --- a/packages/plugin-dkg-publisher/package-addition.json +++ b/packages/plugin-dkg-publisher/package-addition.json @@ -1,6 +1,5 @@ { "scripts": { - "km:setup": "node setup.js", "km:migrate": "npm run db:migrate", "km:worker": "npm run worker", "km:dashboard": "npm run dashboard", diff --git a/packages/plugin-dkg-publisher/package.json b/packages/plugin-dkg-publisher/package.json index 79037921..3efbb7b9 100644 --- a/packages/plugin-dkg-publisher/package.json +++ b/packages/plugin-dkg-publisher/package.json @@ -28,7 +28,6 @@ "check-types": "tsc --noEmit", "lint": "eslint . --max-warnings 0", "test": "mocha --loader ../../node_modules/tsx/dist/loader.mjs 'tests/**/*.spec.ts'", - "setup": "node setup.js", "db:generate": "drizzle-kit generate:mysql", "db:migrate": "node migrate.js", "db:push": "drizzle-kit push:mysql", diff --git a/packages/plugin-dkg-publisher/setup.js b/packages/plugin-dkg-publisher/setup.js deleted file mode 100644 index f7847fad..00000000 --- a/packages/plugin-dkg-publisher/setup.js +++ /dev/null @@ -1,1125 +0,0 @@ -#!/usr/bin/env node - -/** - * DKG Publisher Plugin Setup Script - * - * This script configures the DKG Publisher plugin for any DKG-Node project. - * It handles database creation, migrations, wallet setup, and environment configuration. - */ - -const fs = require("fs").promises; -const path = require("path"); -const readline = require("readline"); -const mysql = require("mysql2/promise"); -const crypto = require("crypto"); -const CryptoJS = require("crypto-js"); -const { Wallet } = require("ethers"); - -// Colors for console output -const colors = { - green: "\x1b[32m", - red: "\x1b[31m", - yellow: "\x1b[33m", - blue: "\x1b[34m", - cyan: "\x1b[36m", - reset: "\x1b[0m", - bold: "\x1b[1m", -}; - -function log(message, color = "reset") { - const colorCode = colors[color] || colors.reset; - console.log(`${colorCode}${message}${colors.reset}`); -} - -function logStep(step, message) { - log( - `\n${colors.bold}[${step}]${colors.reset} ${colors.cyan}${message}${colors.reset}`, - ); -} - -// Interactive input helper -function ask(question, options = {}) { - const rl = readline.createInterface({ - input: process.stdin, - output: process.stdout, - }); - - return new Promise((resolve) => { - const prompt = options.password - ? question.replace(/:/g, " (input hidden):") - : question; - - // Mute output for password fields so typed characters are not echoed - if (options.password) { - rl._writeToOutput = (str) => { - // Only suppress characters after the prompt has been written - if (str === prompt || str.includes(prompt)) { - process.stdout.write(str); - } - }; - } - - rl.question(`${colors.yellow}${prompt}${colors.reset} `, (answer) => { - if (options.password) { - // Print a newline since the user's Enter was not echoed - process.stdout.write("\n"); - } - rl.close(); - - // Handle empty input - use default if available - const finalAnswer = answer.trim() || options.default || ""; - - // Handle required fields - if (options.required && !finalAnswer) { - log("This field is required!", "red"); - return resolve(ask(question, options)); - } - - // Handle validation - if (options.validate && !options.validate(finalAnswer)) { - log(options.error || "Invalid input!", "red"); - return resolve(ask(question, options)); - } - - resolve(finalAnswer); - }); - }); -} - -// Create file with content (won't overwrite existing) -async function createFile(filePath, content, overwrite = false) { - try { - if (!overwrite) { - try { - await fs.access(filePath); - log( - `File ${path.basename(filePath)} already exists, skipping...`, - "yellow", - ); - return false; - } catch { - // File doesn't exist, continue - } - } - - await fs.writeFile(filePath, content, "utf8"); - log(`Created ${path.basename(filePath)}`, "green"); - return true; - } catch (error) { - log(`Error creating ${path.basename(filePath)}: ${error.message}`, "red"); - return false; - } -} - -// Generate secure random key -function generateSecretKey(length = 32) { - return crypto.randomBytes(length).toString("hex"); -} - -// Encrypt wallet private key using AES (same method as runtime) -function encryptPrivateKey(privateKey, encryptionKey) { - return CryptoJS.AES.encrypt(privateKey, encryptionKey).toString(); -} - -// Validate Ethereum address -function isValidAddress(address) { - return /^0x[a-fA-F0-9]{40}$/.test(address); -} - -// Validate private key -function isValidPrivateKey(key) { - return /^0x[a-fA-F0-9]{64}$/.test(key) || /^[a-fA-F0-9]{64}$/.test(key); -} - -// Derive wallet address from private key -function getAddressFromPrivateKey(privateKey) { - try { - // Ensure private key has 0x prefix - const formattedKey = privateKey.startsWith("0x") - ? privateKey - : `0x${privateKey}`; - const wallet = new Wallet(formattedKey); - return wallet.address; - } catch (error) { - throw new Error(`Invalid private key: ${error.message}`); - } -} - -// Bootstrap migration journal for databases created by a previous version of -// setup.js (raw DDL, no __drizzle_migrations table). Without this, migrate() -// would try to re-run CREATE TABLE statements on existing tables and fail. -// This mirrors the logic in src/database/bootstrap.ts but uses raw mysql2 queries. -async function bootstrapJournalForSetup(pool) { - const fsSync = require("fs"); - - const [journals] = await pool.execute( - `SELECT COUNT(*) as cnt FROM information_schema.tables WHERE table_schema = DATABASE() AND table_name = '__drizzle_migrations'`, - ); - if (Number(journals[0].cnt) > 0) return; - - const [tables] = await pool.execute( - `SELECT COUNT(*) as cnt FROM information_schema.tables WHERE table_schema = DATABASE() AND table_name IN ('assets', 'wallets', 'publishing_attempts', 'batches')`, - ); - const tableCount = Number(tables[0].cnt); - if (tableCount === 0) return; // Fresh DB - - if (tableCount < 4) { - throw new Error( - "Database is in a partial state (some tables missing). Please choose 'Start fresh' (option 1).", - ); - } - - log(" Bootstrapping migration journal for existing database...", "cyan"); - - await pool.execute(` - CREATE TABLE IF NOT EXISTS __drizzle_migrations ( - id SERIAL PRIMARY KEY, - hash text NOT NULL, - created_at bigint - ) - `); - - const journalPath = path.join( - __dirname, - "src/database/migrations/meta/_journal.json", - ); - const journal = JSON.parse(fsSync.readFileSync(journalPath, "utf-8")); - - // Seed 0000 and 0001 (setup.js schema = post-0001 state) - for (const entry of journal.entries) { - if (entry.idx > 1) break; - const sqlFile = path.join( - __dirname, - `src/database/migrations/${entry.tag}.sql`, - ); - const content = fsSync.readFileSync(sqlFile, "utf-8"); - const hash = crypto.createHash("sha256").update(content).digest("hex"); - await pool.execute( - `INSERT INTO __drizzle_migrations (hash, created_at) VALUES (?, ?)`, - [hash, entry.when], - ); - } - - // Check if 0002 changes are already present - const entry0002 = journal.entries.find((e) => e.idx === 2); - if (entry0002) { - const [hasErrorDetails] = await pool.execute( - `SELECT COUNT(*) as cnt FROM information_schema.columns WHERE table_schema = DATABASE() AND table_name = 'publishing_attempts' AND column_name = 'error_details'`, - ); - const [hasPrivateKey] = await pool.execute( - `SELECT COUNT(*) as cnt FROM information_schema.columns WHERE table_schema = DATABASE() AND table_name = 'wallets' AND column_name = 'private_key'`, - ); - if ( - Number(hasErrorDetails[0].cnt) > 0 && - Number(hasPrivateKey[0].cnt) > 0 - ) { - const sqlFile = path.join( - __dirname, - `src/database/migrations/${entry0002.tag}.sql`, - ); - const content = fsSync.readFileSync(sqlFile, "utf-8"); - const hash = crypto.createHash("sha256").update(content).digest("hex"); - await pool.execute( - `INSERT INTO __drizzle_migrations (hash, created_at) VALUES (?, ?)`, - [hash, entry0002.when], - ); - } - } - - log(" โœ“ Migration journal bootstrapped", "green"); -} - -// Check if configuration already exists -async function checkExistingConfig() { - const existingEnv = ".env.publisher"; - const existingCompose = "docker-compose.knowledge-manager.yml"; - - let hasConfig = false; - let configDetails = { - env: false, - compose: false, - database: null, - redis: null, - }; - - try { - await fs.access(existingEnv); - configDetails.env = true; - hasConfig = true; - - // Parse existing env - const envContent = await fs.readFile(existingEnv, "utf8"); - const dbMatch = envContent.match( - /DKGP_DATABASE_URL=mysql:\/\/([^:]+):?([^@]*)@([^:]+):(\d+)\/(\w+)/, - ); - const redisMatch = envContent.match( - /REDIS_URL=redis:\/\/(?:([^@]+)@)?([^:]+):(\d+)/, - ); - - if (dbMatch) { - configDetails.database = { - user: dbMatch[1], - password: dbMatch[2] || "", - host: dbMatch[3], - port: dbMatch[4], - name: dbMatch[5], - }; - } - - if (redisMatch) { - configDetails.redis = { - password: redisMatch[1] || "", - host: redisMatch[2], - port: redisMatch[3], - }; - } - } catch {} - - try { - await fs.access(existingCompose); - configDetails.compose = true; - hasConfig = true; - } catch {} - - return { hasConfig, configDetails }; -} - -// Add wallets only mode -async function addWalletsOnly(configDetails) { - if (!configDetails.database) { - log( - "โŒ No database configuration found. Please run full setup first.", - "red", - ); - return; - } - - // Read encryption key from existing .env file - let encryptionKey; - try { - const envContent = await fs.readFile(".env.publisher", "utf8"); - const keyMatch = envContent.match(/ENCRYPTION_KEY=(.+)/); - if (!keyMatch) { - log("โŒ ENCRYPTION_KEY not found in .env.publisher", "red"); - return; - } - encryptionKey = keyMatch[1]; - log("โœ“ Found encryption key", "green"); - } catch (error) { - log("โŒ Could not read .env.publisher file", "red"); - return; - } - - logStep("1/2", "Connect to Database"); - - const { database } = configDetails; - let connection; - - try { - connection = await mysql.createConnection({ - host: database.host, - port: parseInt(database.port), - user: database.user, - password: database.password, - database: database.name, - }); - - log("โœ“ Connected to existing database", "green"); - - // Check existing wallets - const [existingWallets] = await connection.execute( - "SELECT address, blockchain FROM wallets WHERE is_active = TRUE", - ); - - if (existingWallets.length > 0) { - log(`\n๐Ÿ“‹ Found ${existingWallets.length} existing wallet(s):`, "cyan"); - existingWallets.forEach((wallet, index) => { - log( - ` ${index + 1}. ${wallet.address} (${wallet.blockchain})`, - "white", - ); - }); - } - - logStep("2/2", "Add New Wallets"); - - const wallets = []; - let addMoreWallets = true; - let walletCount = existingWallets.length + 1; - - while (addMoreWallets) { - log(`${colors.bold}Wallet ${walletCount}:${colors.reset}`); - - const privateKey = await ask(`Private Key (0x... or without 0x):`, { - required: true, - validate: isValidPrivateKey, - error: "Invalid private key format", - }); - - // Ensure private key has 0x prefix - const formattedPrivateKey = privateKey.startsWith("0x") - ? privateKey - : `0x${privateKey}`; - - // Derive address from private key - let address; - try { - address = getAddressFromPrivateKey(formattedPrivateKey); - log(` โ†ณ Derived address: ${address}`, "cyan"); - } catch (error) { - log(`โŒ ${error.message}`, "red"); - continue; - } - - // Check if wallet already exists - const [duplicate] = await connection.execute( - "SELECT id FROM wallets WHERE address = ?", - [address], - ); - if (duplicate.length > 0) { - log(`โš ๏ธ Wallet ${address} already exists, skipping...`, "yellow"); - const addMore = await ask(`Add another wallet? (y/N):`, { - default: "n", - }); - addMoreWallets = - addMore.toLowerCase() === "y" || addMore.toLowerCase() === "yes"; - continue; - } - - const blockchain = await ask( - `Blockchain (default: ${existingWallets[0]?.blockchain || "otp:20430"}):`, - { - default: existingWallets[0]?.blockchain || "otp:20430", - }, - ); - - wallets.push({ - address, - privateKey: formattedPrivateKey, - blockchain, - }); - - log(`โœ“ Wallet ${walletCount} added`, "green"); - - const addMore = await ask(`Add another wallet? (y/N):`, { default: "n" }); - addMoreWallets = - addMore.toLowerCase() === "y" || addMore.toLowerCase() === "yes"; - walletCount++; - } - - // Insert new wallets - for (const wallet of wallets) { - await connection.execute( - `INSERT INTO wallets (address, private_key, blockchain) VALUES (?, ?, ?)`, - [wallet.address, wallet.privateKey, wallet.blockchain], - ); - } - - log(`\nโœ… Added ${wallets.length} new wallet(s) to database`, "green"); - log(`๐Ÿ“‹ Total wallets: ${existingWallets.length + wallets.length}`, "cyan"); - } catch (error) { - log(`Database error: ${error.message}`, "red"); - } finally { - if (connection) await connection.end(); - } -} - -// Main setup function -async function setup() { - log( - `${colors.bold}${colors.blue}โ•”โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•—${colors.reset}`, - ); - log( - `${colors.bold}${colors.blue}โ•‘ DKG Publisher Plugin Setup โ•‘${colors.reset}`, - ); - log( - `${colors.bold}${colors.blue}โ•‘ โ•‘${colors.reset}`, - ); - log( - `${colors.bold}${colors.blue}โ•‘ This script will configure the DKG Publisher plugin โ•‘${colors.reset}`, - ); - log( - `${colors.bold}${colors.blue}โ•‘ for publishing JSON-LD assets to the DKG blockchain. โ•‘${colors.reset}`, - ); - log( - `${colors.bold}${colors.blue}โ•šโ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•${colors.reset}\n`, - ); - - try { - // Check for existing configuration - const { hasConfig, configDetails } = await checkExistingConfig(); - - let setupMode; - - if (hasConfig) { - log("๐Ÿ” Existing configuration detected:", "yellow"); - if (configDetails.env) log(" โ€ข .env.publisher found", "cyan"); - if (configDetails.compose) - log(" โ€ข docker-compose.knowledge-manager.yml found", "cyan"); - - setupMode = await ask( - "\nChoose setup mode:\n1. Start fresh (โš ๏ธ will delete existing tables and backup config files)\n2. Update existing configuration\n3. Add wallets only\nChoice (1-3):", - { - validate: (input) => ["1", "2", "3"].includes(input), - error: "Please enter 1, 2, or 3", - }, - ); - - if (setupMode === "1") { - // Backup existing files - const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); - try { - await fs.rename( - ".env.publisher", - `.env.publisher.backup-${timestamp}`, - ); - log("โœ“ Backed up existing .env file", "green"); - } catch {} - try { - await fs.rename( - "docker-compose.knowledge-manager.yml", - `docker-compose.knowledge-manager.yml.backup-${timestamp}`, - ); - log("โœ“ Backed up existing docker-compose file", "green"); - } catch {} - } else if (setupMode === "3") { - // Wallets-only mode - return await addWalletsOnly(configDetails); - } - } - - // Use existing configuration as defaults if available - const existingDb = configDetails.database; - const existingRedis = configDetails.redis; - - // Step 1: Database Configuration - logStep("1/7", "Database Configuration"); - - const dbHost = await ask( - `MySQL Host (default: ${existingDb?.host || "localhost"}):`, - { default: existingDb?.host || "localhost" }, - ); - const dbPort = await ask( - `MySQL Port (default: ${existingDb?.port || "3306"}):`, - { default: existingDb?.port || "3306" }, - ); - const dbUser = await ask( - `MySQL Username (default: ${existingDb?.user || "root"}):`, - { default: existingDb?.user || "root" }, - ); - const dbPassword = await ask("MySQL Password:", { - password: true, - default: existingDb?.password || "", - }); - const dbName = await ask( - `Database Name (default: ${existingDb?.name || "dkg_publisher_db"}):`, - { default: existingDb?.name || "dkg_publisher_db" }, - ); - - // Step 2: Redis Configuration - logStep("2/7", "Redis Configuration"); - - const redisHost = await ask( - `Redis Host (default: ${existingRedis?.host || "localhost"}):`, - { default: existingRedis?.host || "localhost" }, - ); - const redisPort = await ask( - `Redis Port (default: ${existingRedis?.port || "6379"}):`, - { default: existingRedis?.port || "6379" }, - ); - const redisPassword = await ask("Redis Password (leave empty if none):", { - default: existingRedis?.password || "", - }); - - // Step 3: DKG Configuration - logStep("3/7", "DKG Network Configuration"); - - const dkgEndpoint = await ask( - "DKG Engine URL (default: http://localhost:8900):", - { - default: "http://localhost:8900", - }, - ); - - const blockchainOptions = [ - "hardhat1:31337 (Local Development)", - "gnosis:100 (Gnosis Mainnet)", - "gnosis:10200 (Gnosis Testnet)", - "base:8453 (Base Mainnet)", - "base:84532 (Base Testnet)", - "otp:20430 (NeuroWeb Testnet)", - "otp:2043 (NeuroWeb Mainnet)", - "custom (Enter manually)", - ]; - - log("\nAvailable blockchain networks:"); - blockchainOptions.forEach((option, index) => { - log(`${index + 1}. ${option}`, "cyan"); - }); - - const blockchainChoice = await ask("Choose blockchain (1-8):", { - validate: (input) => - ["1", "2", "3", "4", "5", "6", "7", "8"].includes(input), - error: "Please enter 1, 2, 3, 4, 5, 6, 7, or 8", - }); - - let blockchain; - switch (blockchainChoice) { - case "1": - blockchain = "hardhat1:31337"; - break; - case "2": - blockchain = "gnosis:100"; - break; - case "3": - blockchain = "gnosis:10200"; - break; - case "4": - blockchain = "base:8453"; - break; - case "5": - blockchain = "base:84532"; - break; - case "6": - blockchain = "otp:20430"; - break; - case "7": - blockchain = "otp:2043"; - break; - case "8": - blockchain = await ask("Enter blockchain (format: name:chainId):", { - required: true, - validate: (input) => input.includes(":"), - error: "Format must be name:chainId (e.g., gnosis:100)", - }); - break; - } - - if (!blockchain) { - log("โŒ Blockchain selection failed", "red"); - throw new Error("Invalid blockchain selection"); - } - - // Step 4: Wallet Configuration - logStep("4/7", "Wallet Pool Setup"); - - log( - "The DKG Publisher plugin requires at least one wallet for publishing.", - ); - log( - "For high throughput, configure multiple wallets (10-100+ recommended).\n", - ); - - const wallets = []; - let addMoreWallets = true; - let walletCount = 1; - - while (addMoreWallets) { - log(`${colors.bold}Wallet ${walletCount}:${colors.reset}`); - - const privateKey = await ask(`Private Key (0x... or without 0x):`, { - required: true, - validate: isValidPrivateKey, - error: "Invalid private key format", - }); - - // Ensure private key has 0x prefix - const formattedPrivateKey = privateKey.startsWith("0x") - ? privateKey - : `0x${privateKey}`; - - // Derive address from private key - let address; - try { - address = getAddressFromPrivateKey(formattedPrivateKey); - log(` โ†ณ Derived address: ${address}`, "cyan"); - } catch (error) { - log(`โŒ ${error.message}`, "red"); - continue; - } - - wallets.push({ - address, - privateKey: formattedPrivateKey, - blockchain, - }); - - log(`โœ“ Wallet ${walletCount} added`, "green"); - - const addMore = await ask(`Add another wallet? (y/N):`, { default: "n" }); - addMoreWallets = - addMore.toLowerCase() === "y" || addMore.toLowerCase() === "yes"; - walletCount++; - } - - // Step 5: Storage Configuration - logStep("5/7", "File Storage Configuration"); - - const storageType = await ask( - "Storage type (filesystem/s3) [default: filesystem]:", - { - default: "filesystem", - validate: (input) => ["filesystem", "s3"].includes(input.toLowerCase()), - error: 'Please enter "filesystem" or "s3"', - }, - ); - - let storageConfig = { type: storageType.toLowerCase() }; - - if (storageConfig.type === "filesystem") { - const storagePath = await ask("Storage directory (default: ./storage):", { - default: "./storage", - }); - storageConfig.path = storagePath; - } else { - const s3Bucket = await ask("S3 Bucket name:", { required: true }); - const s3Region = await ask("S3 Region (default: us-east-1):", { - default: "us-east-1", - }); - const s3AccessKey = await ask("AWS Access Key ID:", { required: true }); - const s3SecretKey = await ask("AWS Secret Access Key:", { - password: true, - required: true, - }); - - storageConfig = { - type: "s3", - bucket: s3Bucket, - region: s3Region, - accessKeyId: s3AccessKey, - secretAccessKey: s3SecretKey, - }; - } - - // Step 6: Worker Configuration - logStep("6/7", "Worker Pool Configuration"); - - log("Configure worker processes for optimal throughput:"); - log( - `With ${wallets.length} wallet(s), recommended workers: ${Math.min(Math.ceil(wallets.length / 10), 10)}`, - ); - log( - "Note: Worker concurrency is now auto-calculated based on wallet count.", - ); - - const workerCount = await ask( - `Number of worker processes (default: ${Math.min(Math.ceil(wallets.length / 10), 5)}):`, - { - default: Math.min(Math.ceil(wallets.length / 10), 5).toString(), - validate: (input) => !isNaN(parseInt(input)) && parseInt(input) > 0, - error: "Please enter a positive number", - }, - ); - - // Step 7: Create Configuration Files - logStep("6/7", "Creating Configuration Files"); - - // Generate security keys once for reuse - const encryptionKey = generateSecretKey(); - const jwtSecret = generateSecretKey(); - - // Environment file - const envContent = `# DKG Publisher Plugin Configuration -# Generated by setup script on ${new Date().toISOString()} - -# Database Configuration -DKGP_DATABASE_URL=mysql://${dbUser}${dbPassword ? ":" + dbPassword : ""}@${dbHost}:${dbPort}/${dbName} - -# Redis Configuration -REDIS_URL=redis://${redisPassword ? `${redisPassword}@` : ""}${redisHost}:${redisPort} - -# DKG Network Configuration -DKG_ENDPOINT=${dkgEndpoint} -DKG_BLOCKCHAIN=${blockchain} - -# File Storage Configuration -STORAGE_TYPE=${storageConfig.type} -${ - storageConfig.type === "filesystem" - ? `` - : `AWS_S3_BUCKET=${storageConfig.bucket} -AWS_S3_REGION=${storageConfig.region} -AWS_ACCESS_KEY_ID=${storageConfig.accessKeyId} -AWS_SECRET_ACCESS_KEY=${storageConfig.secretAccessKey}` -} - -# Worker Configuration -WORKER_COUNT=${workerCount} -# Note: WORKER_CONCURRENCY is auto-calculated from wallet count (no need to set manually) - -# Security -ENCRYPTION_KEY=${encryptionKey} -JWT_SECRET=${jwtSecret} - -# Monitoring (optional) -# SENTRY_DSN= -# DATADOG_API_KEY= -`; - - const overwriteConfig = setupMode === "1" || setupMode === "2"; // Fresh or Update - await createFile(".env.publisher", envContent, overwriteConfig); - - // Skip wallet configuration file - wallets will be inserted directly into database - - // Docker compose file - const dockerComposeContent = `version: '3.8' -services: - mysql: - image: mysql:8.0 - environment: - MYSQL_ROOT_PASSWORD: ${dbPassword} - MYSQL_DATABASE: ${dbName} - MYSQL_USER: km_user - MYSQL_PASSWORD: ${generateSecretKey(16)} - ports: - - "${dbPort}:3306" - volumes: - - mysql_data:/var/lib/mysql - - ./migrations:/docker-entrypoint-initdb.d - command: > - --default-authentication-plugin=mysql_native_password - --innodb-lock-wait-timeout=10 - --max-connections=200 - - redis: - image: redis:7-alpine - ports: - - "${redisPort}:6379" - volumes: - - redis_data:/data - ${redisPassword ? `command: redis-server --requirepass ${redisPassword}` : ""} - - knowledge-manager: - image: dkg-knowledge-manager:latest - env_file: - - .env.publisher - depends_on: - - mysql - - redis - scale: ${workerCount} - volumes: - ${storageConfig.type === "filesystem" ? `- ./storage:/app/storage` : ""} - -volumes: - mysql_data: - redis_data: -`; - - await createFile( - "docker-compose.knowledge-manager.yml", - dockerComposeContent, - overwriteConfig, - ); - - // Package.json scripts - const packageJsonAddition = { - scripts: { - "km:setup": "node setup.js", - "km:migrate": "npm run db:migrate", - "km:worker": "npm run worker", - "km:dashboard": "npm run dashboard", - "km:status": - "node -e \"console.log('Use the /api/knowledge/health endpoint for status')\"", - "km:docker:up": - "docker-compose -f docker-compose.knowledge-manager.yml up -d", - "km:docker:down": - "docker-compose -f docker-compose.knowledge-manager.yml down", - }, - dependencies: { - "@dkg/plugin-dkg-publisher": "^1.0.0", - bullmq: "^4.15.0", - ioredis: "^5.3.2", - mysql2: "^3.6.5", - "drizzle-orm": "^0.29.0", - }, - }; - - await createFile( - "package-addition.json", - JSON.stringify(packageJsonAddition, null, 2), - ); - - // Create storage directory if filesystem - if (storageConfig.type === "filesystem") { - try { - await fs.mkdir(path.resolve(storageConfig.path), { recursive: true }); - log(`Created storage directory: ${storageConfig.path}`, "green"); - } catch (error) { - log( - `Warning: Could not create storage directory: ${error.message}`, - "yellow", - ); - } - } - - // Database setup - logStep("7/7", "Database Setup"); - log("Setting up database and creating tables..."); - let connection; - try { - // Connect to MySQL without database first - log(`Connecting to MySQL at ${dbHost}:${dbPort}...`); - connection = await mysql.createConnection({ - host: dbHost, - port: parseInt(dbPort), - user: dbUser, - password: dbPassword, - }); - - log("โœ“ Connected to MySQL server", "green"); - - // Create database if it doesn't exist - await connection.execute( - `CREATE DATABASE IF NOT EXISTS \`${dbName}\` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci`, - ); - log(`โœ“ Database '${dbName}' created/verified`, "green"); - - // Switch to the database - await connection.changeUser({ database: dbName }); - log(`โœ“ Connected to database '${dbName}'`, "green"); - - // Run Drizzle migrations to create/update tables - log("Running database migrations...", "cyan"); - - // Check if this is a fresh start (Mode 1) โ€” drop all tables first - if (setupMode === "1" || !hasConfig) { - log(" Dropping existing tables for fresh setup...", "white"); - await connection.execute("SET FOREIGN_KEY_CHECKS = 0"); - await connection.execute("DROP TABLE IF EXISTS __drizzle_migrations"); - await connection.execute("DROP TABLE IF EXISTS wallet_metrics"); - await connection.execute("DROP TABLE IF EXISTS publishing_attempts"); - await connection.execute("DROP TABLE IF EXISTS assets"); - await connection.execute("DROP TABLE IF EXISTS wallets"); - await connection.execute("DROP TABLE IF EXISTS batches"); - await connection.execute("DROP TABLE IF EXISTS metrics_hourly"); - await connection.execute("SET FOREIGN_KEY_CHECKS = 1"); - } - - // Close the single connection โ€” Drizzle needs a pool - await connection.end(); - connection = null; - - // Run Drizzle migrations via a dedicated connection with FK checks disabled. - // Migration 0001 changes column types that have FK references, and MySQL - // validates FK compatibility on each ALTER TABLE. - const { drizzle } = require("drizzle-orm/mysql2"); - const { migrate } = require("drizzle-orm/mysql2/migrator"); - const migrationConn = await mysql.createConnection({ - host: dbHost, - port: parseInt(dbPort), - user: dbUser, - password: dbPassword, - database: dbName, - }); - try { - // Bootstrap migration journal for existing databases without one - // (e.g. created by a previous version of setup.js with raw DDL) - await bootstrapJournalForSetup(migrationConn); - - await migrate(drizzle(migrationConn), { - migrationsFolder: path.join(__dirname, "src/database/migrations"), - }); - } finally { - await migrationConn.end(); - } - log("โœ“ Database migrations completed", "green"); - - // Reconnect with single connection for wallet insertion - connection = await mysql.createConnection({ - host: dbHost, - port: parseInt(dbPort), - user: dbUser, - password: dbPassword, - database: dbName, - }); - - // Verify tables were created - const [tables] = await connection.execute("SHOW TABLES"); - const tableNames = tables.map((row) => Object.values(row)[0]); - log( - `โœ“ Created ${tableNames.length} tables: ${tableNames.join(", ")}`, - "green", - ); - - // Insert wallet configurations - if (wallets.length > 0) { - log(`Inserting ${wallets.length} wallet(s)...`, "cyan"); - log( - `Debug: Wallet addresses to insert: ${wallets.map((w) => w.address).join(", ")}`, - "yellow", - ); - let insertedCount = 0; - - for (let i = 0; i < wallets.length; i++) { - const wallet = wallets[i]; - try { - log( - ` Processing wallet ${i + 1}/${wallets.length}: ${wallet.address}`, - "cyan", - ); - - // Check if wallet already exists first - const [existing] = await connection.execute( - `SELECT id FROM wallets WHERE address = ?`, - [wallet.address], - ); - - if (existing.length > 0) { - log( - ` - Wallet ${wallet.address} already exists (ID: ${existing[0].id}), skipped`, - "yellow", - ); - continue; - } - - // Store private key as plain text (simplified approach) - const privateKey = wallet.privateKey; - - const [result] = await connection.execute( - `INSERT INTO wallets (address, private_key, blockchain) VALUES (?, ?, ?)`, - [wallet.address, privateKey, wallet.blockchain], - ); - - insertedCount++; - log( - ` โœ“ Wallet ${wallet.address} inserted (ID: ${result.insertId})`, - "green", - ); - } catch (walletError) { - log( - ` โŒ Failed to insert wallet ${wallet.address}: ${walletError.message}`, - "red", - ); - } - } - - log( - `โœ“ ${insertedCount} wallet(s) successfully configured in database`, - "green", - ); - - // Verify wallets were inserted - const [walletCount] = await connection.execute( - "SELECT COUNT(*) as count FROM wallets WHERE is_active = TRUE", - ); - log( - `โœ“ Total active wallets in database: ${walletCount[0].count}`, - "green", - ); - } else { - log( - "โš ๏ธ No wallets configured. You can add them later using setup script option 3.", - "yellow", - ); - } - - await connection.end(); - log("โœ“ Database connection closed", "green"); - } catch (error) { - log(`\nโŒ Database setup failed: ${error.message}`, "red"); - log("Possible causes:", "yellow"); - log("โ€ข MySQL server not running", "white"); - log("โ€ข Incorrect credentials", "white"); - log("โ€ข Network connection issues", "white"); - log("โ€ข Permission problems", "white"); - - if (connection) { - try { - await connection.end(); - } catch (closeError) { - // Ignore connection close errors - } - } - - throw error; // Re-throw to stop setup process - } - - // Final success message - log( - `\n${colors.bold}${colors.green}โ•”โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•—${colors.reset}`, - ); - log( - `${colors.bold}${colors.green}โ•‘ Setup Complete! โ•‘${colors.reset}`, - ); - log( - `${colors.bold}${colors.green}โ•šโ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•${colors.reset}\n`, - ); - - log("Configuration files created:", "green"); - log(" โ€ข .env.publisher (environment variables)", "cyan"); - log(" โ€ข docker-compose.knowledge-manager.yml (Docker setup)", "cyan"); - log(" โ€ข package-addition.json (npm scripts to add)", "cyan"); - - log("\nNext steps:", "yellow"); - log( - "The DKG Publisher plugin is now ready! Here's how to use it:", - "white", - ); - log(""); - log("โœ… Already completed:", "green"); - log(" โ€ข Database created with tables", "white"); - log(" โ€ข Configuration files generated", "white"); - log(" โ€ข Wallets configured in database", "white"); - log(""); - log("๐Ÿš€ To start using:", "blue"); - log("1. Make sure MySQL and Redis are running locally", "white"); - log("2. Configure DKG Agent to load this plugin", "white"); - log("3. The service will auto-start with configured workers", "white"); - log("4. Test API endpoints or use MCP tools", "white"); - log(""); - log("๐Ÿ“‹ Optional:", "cyan"); - log("โ€ข Add more wallets: npm run setup (choose option 3)", "white"); - log(" Workers will auto-restart to match new wallet count", "white"); - log("โ€ข View dashboard at: /admin/queues (when agent is running)", "white"); - log("โ€ข Check health: GET /api/knowledge/health", "white"); - - log("\nExample usage in DKG Agent plugin:", "yellow"); - log( - `// The DKG Publisher plugin runs as a DKG plugin -// and provides these API endpoints: - -// Register asset for publishing -POST /api/knowledge/assets -{ - "content": { "@context": "https://schema.org", "@type": "Certificate" }, - "metadata": { "source": "my-app", "sourceId": "cert-123" }, - "publishOptions": { "privacy": "private", "priority": 80 } -} - -// Get asset status -GET /api/knowledge/assets/{id} - -// Get metrics -GET /api/knowledge/metrics/queue -GET /api/knowledge/metrics/wallets -GET /api/knowledge/health - -// MCP Tool -knowledge-asset-publish`, - "white", - ); - - log("\nโš ๏ธ Security Notes:", "red"); - log("โ€ข Keep your DATABASE_URL and ENCRYPTION_KEY secure", "yellow"); - log("โ€ข Use environment variables for production deployments", "yellow"); - - log(`\n${colors.bold}Happy publishing! ๐Ÿš€${colors.reset}\n`); - } catch (error) { - log(`\nSetup failed: ${error.message}`, "red"); - console.error(error); - process.exit(1); - } -} - -// Handle graceful shutdown -process.on("SIGINT", () => { - log("\nSetup cancelled by user.", "yellow"); - process.exit(0); -}); - -// Run setup if called directly -if (require.main === module) { - setup(); -} - -module.exports = { setup }; diff --git a/packages/plugin-dkg-publisher/src/database/bootstrap.ts b/packages/plugin-dkg-publisher/src/database/bootstrap.ts index 6474b7ee..51a8483c 100644 --- a/packages/plugin-dkg-publisher/src/database/bootstrap.ts +++ b/packages/plugin-dkg-publisher/src/database/bootstrap.ts @@ -5,9 +5,9 @@ import fs from "fs"; import path from "path"; /** - * Bootstrap migration journal for databases created by setup.js (raw DDL). + * Bootstrap migration journal for databases created by the legacy publisher setup script (raw DDL). * - * setup.js creates tables directly without Drizzle migration tracking. + * The legacy setup script creates tables directly without Drizzle migration tracking. * Without this guard, runMigrations() would try to run 0000 (CREATE TABLE) * on existing tables and fail. * @@ -23,7 +23,7 @@ export async function bootstrapMigrationJournal(db: Database): Promise { return; // Already managed by Drizzle } - // Check if core tables exist (setup.js creates these) + // Check if core tables exist (legacy setup script creates these) const coreTables = ["assets", "wallets", "publishing_attempts", "batches"]; const existingTables = await Promise.all( coreTables.map((t) => tableExists(db, t)), @@ -38,14 +38,14 @@ export async function bootstrapMigrationJournal(db: Database): Promise { const missing = coreTables.filter((_, i) => !existingTables[i]); throw new Error( `Database is in a partial state: tables ${missing.join(", ")} are missing. ` + - `This usually means setup.js crashed mid-creation. ` + + `This usually means the legacy setup script crashed mid-creation. ` + `Please drop all tables and run setup again.`, ); } - // Tables exist but no journal โ€” setup.js-created database + // Tables exist but no journal - legacy setup script-created database console.log( - "๐Ÿ“‹ Detected setup.js-created database without migration journal. Bootstrapping...", + "Detected legacy setup script-created database without migration journal. Bootstrapping...", ); // Create the __drizzle_migrations table (same schema Drizzle uses) @@ -65,7 +65,7 @@ export async function bootstrapMigrationJournal(db: Database): Promise { const journalPath = path.join(migrationsDir, "meta/_journal.json"); const journal = JSON.parse(fs.readFileSync(journalPath, "utf-8")); - // Always seed 0000 and 0001 (setup.js schema = post-0001 state) + // Always seed 0000 and 0001 (legacy setup script schema = post-0001 state) for (const entry of journal.entries) { if (entry.idx > 1) break; // Only seed 0000 and 0001 unconditionally From 9e79b4082c1c9bca2d31b7960ce5629c8a53157c Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Thu, 5 Mar 2026 13:52:39 +0100 Subject: [PATCH 07/10] [bug] Fix publisher ESM import and document-to-markdown API path Use @bull-board/api/bullMQAdapter.js so async publisher loads under ESM. Align document-to-markdown REST endpoint to /api/document-to-markdown for route consistency. --- .../src/plugins/document-to-markdown/index.ts | 2 +- packages/plugin-dkg-publisher/src/services/QueueService.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/plugin-dkg-essentials/src/plugins/document-to-markdown/index.ts b/packages/plugin-dkg-essentials/src/plugins/document-to-markdown/index.ts index 9007d612..e26521db 100644 --- a/packages/plugin-dkg-essentials/src/plugins/document-to-markdown/index.ts +++ b/packages/plugin-dkg-essentials/src/plugins/document-to-markdown/index.ts @@ -134,7 +134,7 @@ export function createDocumentToMarkdownPlugin( // REST endpoint for document-to-markdown conversion api.post( - "/document-to-markdown", + "/api/document-to-markdown", openAPIRoute( { summary: "Convert document to Markdown", diff --git a/packages/plugin-dkg-publisher/src/services/QueueService.ts b/packages/plugin-dkg-publisher/src/services/QueueService.ts index 0f366967..bd6c1fe2 100644 --- a/packages/plugin-dkg-publisher/src/services/QueueService.ts +++ b/packages/plugin-dkg-publisher/src/services/QueueService.ts @@ -2,7 +2,7 @@ import { EventEmitter } from "events"; import { Queue, Worker, QueueEvents } from "bullmq"; import IORedis from "ioredis"; import { createBullBoard } from "@bull-board/api"; -import { BullMQAdapter } from "@bull-board/api/bullMQAdapter"; +import { BullMQAdapter } from "@bull-board/api/bullMQAdapter.js"; import { ExpressAdapter } from "@bull-board/express"; import { PublishingService } from "./PublishingService"; import { WalletService } from "./WalletService"; From d60f21c7acd5a788b6e884fa9cd9a06dca229647 Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Thu, 5 Mar 2026 14:04:00 +0100 Subject: [PATCH 08/10] [bug] Enable tsup shims for publisher ESM runtime compatibility Add --shims to publisher dev/build scripts so __dirname is defined in dist/*.mjs. Prevents __dirname runtime crashes when async publisher is loaded via ESM imports. --- packages/plugin-dkg-publisher/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/plugin-dkg-publisher/package.json b/packages/plugin-dkg-publisher/package.json index 3efbb7b9..01cb9016 100644 --- a/packages/plugin-dkg-publisher/package.json +++ b/packages/plugin-dkg-publisher/package.json @@ -23,8 +23,8 @@ } }, "scripts": { - "dev": "tsup src/*.ts --format cjs,esm --dts --watch", - "build": "tsup src/*.ts --format cjs,esm --dts", + "dev": "tsup src/*.ts --format cjs,esm --dts --watch --shims", + "build": "tsup src/*.ts --format cjs,esm --dts --shims", "check-types": "tsc --noEmit", "lint": "eslint . --max-warnings 0", "test": "mocha --loader ../../node_modules/tsx/dist/loader.mjs 'tests/**/*.spec.ts'", From 1f0f1c646574c52ce9d324ae199f7ec0431e9e13 Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Thu, 5 Mar 2026 14:22:03 +0100 Subject: [PATCH 09/10] [improvement] Increase publisher default epochs to 12 Set runtime default epochs to 12 when publishOptions.epochs is not provided. Align assets table schema default for epochs to 12 for consistency. --- packages/plugin-dkg-publisher/src/database/schema.ts | 2 +- packages/plugin-dkg-publisher/src/services/AssetService.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/plugin-dkg-publisher/src/database/schema.ts b/packages/plugin-dkg-publisher/src/database/schema.ts index c04009ca..57661edc 100644 --- a/packages/plugin-dkg-publisher/src/database/schema.ts +++ b/packages/plugin-dkg-publisher/src/database/schema.ts @@ -34,7 +34,7 @@ export const assets = mysqlTable( // Publishing configuration priority: int("priority").default(50), privacy: mysqlEnum("privacy", ["private", "public"]).default("private"), - epochs: int("epochs").default(2), + epochs: int("epochs").default(12), replications: int("replications").default(1), maxAttempts: int("max_attempts").default(3), diff --git a/packages/plugin-dkg-publisher/src/services/AssetService.ts b/packages/plugin-dkg-publisher/src/services/AssetService.ts index cb16b45c..d2d9d1e9 100644 --- a/packages/plugin-dkg-publisher/src/services/AssetService.ts +++ b/packages/plugin-dkg-publisher/src/services/AssetService.ts @@ -31,7 +31,7 @@ export class AssetService extends EventEmitter { status: "queued", queuedAt: sql`NOW()`, privacy: input.publishOptions?.privacy || "private", - epochs: input.publishOptions?.epochs || 2, + epochs: input.publishOptions?.epochs || 12, maxAttempts: input.publishOptions?.maxAttempts || 3, retryCount: 0, }); From 129b8c80aed3dddc9f5d8f0c0078e32d2cd84bf0 Mon Sep 17 00:00:00 2001 From: Jurij Skornik Date: Thu, 5 Mar 2026 15:26:41 +0100 Subject: [PATCH 10/10] [feat] Add Essentials /api/dkg/create endpoint and remove leaked sqlite file Expose sync publish over HTTP with the same create logic as dkg-create MCP (including blob id support). Remove accidentally committed apps/agent/original_dkg_node file and add API coverage tests. --- apps/agent/original_dkg_node | Bin 57344 -> 0 bytes .../src/plugins/dkg-tools.ts | 110 +++++++++++++++--- .../tests/dkg-tools.spec.ts | 29 +++++ 3 files changed, 125 insertions(+), 14 deletions(-) delete mode 100644 apps/agent/original_dkg_node diff --git a/apps/agent/original_dkg_node b/apps/agent/original_dkg_node deleted file mode 100644 index 2ca01577cc26601bdf6f1d30ba40e7b4bc2cc691..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 57344 zcmeI*OKjW70R~{wvgO!`r0oL`+7qE9u+>;G-;cBb;@C>+l^=<1A;~TZX-E!bW+hP( zsjQnKDD0*!u-CopqPO1K%VN>P9(yRZ=%LpFz4ua}=oaXuJ+wfFlw`}X9lJnKtka*6 zX>n#aocU%%S>lYjd4I)^sNeID_NAXpo0A_-XQqC6;rU6sBOC${fB*y_ z@P8w)+qg8D&*%9cl_Fv{X)mPRFj-|5H_8iJWnpXK_DWeOBw+<%c6zGryLBO=JCSg2 zV|jI9&_TAt@aZ|B z?)lv?sI0v^VhENFD@oE9Tuk6r6G;Nmo_gQ zs{MY+2Z`(Z4t_hL4ca|Ku~6MwUtVJwtd`dp$^lG0My&1gjm9{EJFMYoFx~0+tb8>R z)rH}qM&j?RZ||cb^ShIN+od~6emWnp=3sr+zo zz(a}a@i7Jk5oYU2KJ{WTlg{7F^IUR-@JW;XWmqGWPShg<#aCo{QyK!*lP!Y4m?Qy){N%H1AH8thC!e)76d0}PrO^LNKTeeBKeWXjVaHmU2L|yha zBeZ)h+V!0{dGYR8I{&VAQZR&dYrx`}^z<*#;H} zKmY;|fB*y_009U<00Izz09Q#*aJ!iaPnA8@G;CWoEZHN9q^O$cXu59MwnV8dYpSPv z7S%1qq^2f%qU=&h)lDK&hdPA+_-FtAtX`Q&G7~jPv_wUus;v@2sHIR#QXIw6RZDeM zgAh@oqAOdrBzvwQ%L;L1Q6#QtD5`51{Kxj<9}1Q6Bs1blwrf(V+lJ%mu0k}?vNcna zO)7eh%CeF@nH8JrqM}pRwiMNnRnvB4RuwhQjQsqI-&9gbX0Bx#qH4-ic2z~RZB24@ zTk#B{JE|jF#8FJ$&@IQb6vw8zVp@_Y8iplG4poS*vdq>Oeo+}qGBah58Xi>~QIu8H zVb58?Y>^t8?Gk2a5mmM&Vz3hHmP|}Xl4Ms?i9t zT*(wQ*VbfFS9MKOY)PlKVtTUXc$VhKF{`ahrX<=dJB5n2PG$b1?|lEWpH8zLozDG< z%l#|&kKEsLf6e`wZD4@_1Rwwb2tWV=5P$##AOHafK;XOyjHft09j_+h)p)#0#jCMY zikphB|Bq#QT=o;T!U6#ZKmY;|fB*y_009U<00OU9V9yvEufDh@N|r?p%PpCTWtG@f zZK-4#j#ZKzO`^n<*+p^V2Zb(m*@Pn}s`a{lSh(?SVLOUCH|FP?fkT?xK^WaIMN^u0 zwni5pzpFYXQ}JxE;T<4d&&6s!VSsLMNKwTQ+d9(#*Y_& zJQ2Q68f=bNsbAT%7+Gw%!2Dph`D6--TpG$=-;P?%qjsD9g9wYsJb{D4ouD&%gtOy> zl#TR{7|^gvgMj_!>TlN$Cix^DiX^dN3DSP5#CVuRU*({{zI#+tV|!bs`8>YvY`ii?`2$1A}V#%l5A;OiOmqVNDhzE1Lv)I!fYwhN}XKRn2t-l>rwyRr9m4};8 zYcyRxld4XgX^= z|G&WvjH*EZ0uX=z1Rwwb2tWV=5P$##m_YpbpU?e;V-GA4fB*y_009U<00Izz00bZa z0SKIbfv>0Lr-<9~+qdWrX?2?Px)Zc8{{Q?pENTD&2tWV=5P$##AOHafKmY;|I89*W z^M5iG;I#CJhX4d1009U<00Izz00bZa0SG|g{0SuY|I5A2u?H3iKmY;|fB*y_009U< z00Izz00d4I*j3oA73X)?jZCJ0+eE`Gsgkai47QXkDk>#YC7z>+x+~h&k&*aUNVgHR zW#7H>^rmE8X|a{6>2uLdd9HI)x-##I>yMi|kC%ha?VXkGqj_3sJZ)Uj0^Ot5{MLi} z4_0rlg!G}m89ZBfq6Nmv!hEN_6}A@rM?1~-+xHgO{r&`YgFzv_y`T^}L5E7R(x3m2 n`~RKVyhs262tWV=5P$##AOHafKmY;|_!0`l_y0S~*AV{!H-a { } } + async function resolveJsonLdInput(jsonld: string): Promise { + if (!jsonld) { + console.error("No JSON-LD content provided after file read."); + throw new Error("No JSON-LD content provided."); + } + + if (jsonld.startsWith("{") || jsonld.startsWith("[")) { + return jsonld; + } + + const blob = await ctx.blob.get(jsonld); + if (!blob) { + console.error(`File with id "${jsonld}" not found`); + throw new Error(`File with id "${jsonld}" not found`); + } + + return consumers.text(blob.data); + } + function validateSparqlInput(query: string): SparqlValidationResult { const validation = validateSparqlQuery(query); if (!validation.valid) { @@ -191,21 +210,8 @@ export default defineDkgPlugin((ctx, mcp, api) => { }, }, async (input) => { - if (!input.jsonld) { - console.error("No JSON-LD content provided after file read."); - throw new Error("No JSON-LD content provided."); - } const privacy = input.privacy || "private"; - const content = - input.jsonld.startsWith("{") || input.jsonld.startsWith("[") - ? input.jsonld - : await ctx.blob.get(input.jsonld).then((r) => { - if (!r) { - console.error(`File with id "${input.jsonld}" not found`); - throw new Error(`File with id "${input.jsonld}" not found`); - } - return consumers.text(r.data); - }); + const content = await resolveJsonLdInput(input.jsonld); const { ual, error } = await publishJsonLdAsset(content, privacy); if (error) { @@ -306,6 +312,82 @@ export default defineDkgPlugin((ctx, mcp, api) => { }, ); + api.post( + "/api/dkg/create", + openAPIRoute( + { + tag: "DKG Publishing", + summary: "Create and Publish DKG Asset", + description: + "Synchronously create and publish a Knowledge Asset on DKG from JSON-LD content or uploaded blob id.", + body: z.object({ + jsonld: z + .string() + .describe("JSON-LD content or ID of an uploaded file"), + privacy: z.enum(["private", "public"]).optional().default("private"), + }), + response: { + schema: z.object({ + success: z.boolean(), + data: z + .object({ + ual: z.string(), + explorerLink: z.string(), + message: z.string(), + }) + .optional(), + error: z.string().optional(), + }), + }, + finalizeRouteConfig: (config) => ({ + ...config, + security: [], + }), + }, + async (req, res) => { + try { + const privacy = req.body.privacy || "private"; + const content = await resolveJsonLdInput(req.body.jsonld); + + const { ual, error } = await publishJsonLdAsset(content, privacy); + if (error) { + console.error("Error creating asset:", error); + return res.status(500).json({ + success: false, + error: "Failed to create asset: " + error, + }); + } + + if (!ual) { + return res.status(500).json({ + success: false, + error: "Failed to create asset: missing UAL in response", + }); + } + + const explorerLink = getExplorerUrl(ual); + const message = `Knowledge Asset collection successfully created.\n\nUAL: ${ual}\nDKG Explorer link: ${explorerLink}`; + + return res.json({ + success: true, + data: { + ual, + explorerLink, + message, + }, + }); + } catch (error) { + const errorMessage = + error instanceof Error ? error.message : String(error); + return res.status(500).json({ + success: false, + error: errorMessage, + }); + } + }, + ), + ); + api.post( "/api/dkg/query", openAPIRoute( diff --git a/packages/plugin-dkg-essentials/tests/dkg-tools.spec.ts b/packages/plugin-dkg-essentials/tests/dkg-tools.spec.ts index 5adccff0..9c257921 100644 --- a/packages/plugin-dkg-essentials/tests/dkg-tools.spec.ts +++ b/packages/plugin-dkg-essentials/tests/dkg-tools.spec.ts @@ -451,6 +451,35 @@ describe("@dkg/plugin-dkg-essentials checks", () => { }); describe("HTTP API Routes", () => { + it("should expose POST /api/dkg/create from Essentials", async () => { + const response = await request(app) + .post("/api/dkg/create") + .send({ + jsonld: JSON.stringify({ + "@context": "https://schema.org", + "@type": "Organization", + name: "HTTP Create Test Organization", + }), + privacy: "private", + }) + .expect(200); + + expect(response.body.success).to.equal(true); + expect(response.body.data.ual).to.equal("did:dkg:otp:20430/0x123456/12345"); + expect(response.body.data.explorerLink).to.include( + "https://dkg-testnet.origintrail.io/explore?ual=", + ); + }); + + it("should return 400 for invalid POST /api/dkg/create input", async () => { + await request(app) + .post("/api/dkg/create") + .send({ + privacy: "private", + }) + .expect(400); + }); + it("should expose POST /api/dkg/query from Essentials", async () => { const response = await request(app) .post("/api/dkg/query")