fix(mcp): npm launcher falls back to api.github.com and honours proxies

mcp/crw-mcp/bin/crw-mcp.js

@@ -5,7 +5,9 @@ const crypto = require("crypto");
 const fs = require("fs");
 const path = require("path");
 const os = require("os");
+const http = require("http");
 const https = require("https");
+const tls = require("tls");
 
 const VERSION = require("../package.json").version;
 const REPO = "fastcrw/crw";
@@ -56,16 +58,55 @@ function cacheDir() {
   return path.join(base, "crw-mcp", `v${VERSION}`);
 }
 
+// HTTPS_PROXY / HTTP_PROXY support: Node's https ignores them, and restricted
+// networks often only allow egress through one. Plain-http proxies only (the
+// usual corporate setup), tunnelled with CONNECT. ponytail: NO_PROXY is not
+// honoured, add when someone needs a bypass list.
+function proxyAgent() {
+  const raw =
+    process.env.HTTPS_PROXY || process.env.https_proxy ||
+    process.env.HTTP_PROXY || process.env.http_proxy;
+  if (!raw) return undefined;
+  const proxy = new URL(raw);
+  const headers = {};
+  if (proxy.username) {
+    const cred = `${decodeURIComponent(proxy.username)}:${decodeURIComponent(proxy.password)}`;
+    headers["Proxy-Authorization"] = `Basic ${Buffer.from(cred).toString("base64")}`;
+  }
+  const agent = new https.Agent();
+  agent.createConnection = (opts, cb) => {
+    const req = http.request({
+      host: proxy.hostname,
+      port: proxy.port || 80,
+      method: "CONNECT",
+      path: `${opts.host}:${opts.port || 443}`,
+      headers,
+    });
+    req.once("connect", (res, socket) => {
+      if (res.statusCode !== 200) {
+        socket.destroy();
+        return cb(new Error(`proxy CONNECT failed: HTTP ${res.statusCode}`));
+      }
+      cb(null, tls.connect({ socket, servername: opts.servername || opts.host }));
+    });
+    req.once("error", cb);
+    req.end();
+  };
+  return agent;
+}
+
 // Always resolves a Buffer: the archive is verified in memory, so an unverified
 // byte never reaches disk at all.
-function httpsGet(url, redirects = 0) {
+function httpsGet(url, extraHeaders = {}, redirects = 0) {
   return new Promise((resolve, reject) => {
+    const headers = { "User-Agent": `crw-mcp/${VERSION}`, ...extraHeaders };
     https
-      .get(url, { headers: { "User-Agent": `crw-mcp/${VERSION}` } }, (res) => {
+      .get(url, { headers, agent: proxyAgent() }, (res) => {
         if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
           res.resume();
           if (redirects > 5) return reject(new Error("too many redirects"));
-          return resolve(httpsGet(res.headers.location, redirects + 1));
+          // Extra headers are not forwarded: the redirect target is another host.
+          return resolve(httpsGet(res.headers.location, {}, redirects + 1));
         }
         if (res.statusCode !== 200) {
           res.resume();
@@ -80,6 +121,34 @@ function httpsGet(url, redirects = 0) {
   });
 }
 
+// Same release file through api.github.com, for networks where the
+// github.com download host is blocked but the API host is reachable.
+async function viaApi(name) {
+  const tag = `https://api.github.com/repos/${REPO}/releases/tags/v${VERSION}`;
+  const release = JSON.parse((await httpsGet(tag)).toString("utf8"));
+  const asset = (release.assets || []).find((a) => a.name === name);
+  if (!asset) throw new Error(`${name} is not an asset of v${VERSION}`);
+  return httpsGet(
+    `https://api.github.com/repos/${REPO}/releases/assets/${asset.id}`,
+    { Accept: "application/octet-stream" }
+  );
+}
+
+// A 404 means the release lacks the file, so the API would say the same. Any
+// other failure is a network problem worth one retry through the API host.
+async function fetchAsset(name) {
+  try {
+    return await httpsGet(`https://github.com/${REPO}/releases/download/v${VERSION}/${name}`);
+  } catch (e) {
+    if (/^HTTP 404 /.test(e.message)) throw e;
+    try {
+      return await viaApi(name);
+    } catch (e2) {
+      throw new Error(`${e.message}; api.github.com fallback: ${e2.message}`);
+    }
+  }
+}
+
 // Parse one entry out of a coreutils-style SHA256SUMS. Written on Linux, read
 // here on every platform, so tolerate CRLF and the "*" binary-mode marker.
 function digestFor(sumsText, asset) {
@@ -98,14 +167,12 @@ async function fromDownload() {
   if (fs.existsSync(bin)) return bin;
 
   fs.mkdirSync(dir, { recursive: true });
-  const base = `https://github.com/${REPO}/releases/download/v${VERSION}`;
-  const url = `${base}/${plat.asset}`;
 
   // Fail closed: fetch the expected digest first, so a release without one is
   // refused before anything is downloaded rather than after.
   let sums;
   try {
-    sums = (await httpsGet(`${base}/SHA256SUMS`)).toString("utf8");
+    sums = (await fetchAsset("SHA256SUMS")).toString("utf8");
   } catch (e) {
     // Only a 404 means the release genuinely lacks checksums. Reporting a
     // proxy or DNS failure as "our release is broken" sends users to file
@@ -123,7 +190,7 @@ async function fromDownload() {
 
   // stderr only: stdout is the MCP (JSON-RPC) channel.
   console.error(`crw-mcp: downloading ${plat.asset} (v${VERSION})...`);
-  const data = await httpsGet(url);
+  const data = await fetchAsset(plat.asset);
 
   const actual = crypto.createHash("sha256").update(data).digest("hex");
   if (actual !== expected) {
@@ -168,7 +235,7 @@ async function resolveBinary() {
   return fromEnv() || fromPackage() || (await fromDownload());
 }
 
-module.exports = { digestFor, fromDownload, cacheDir, plat, binName };
+module.exports = { digestFor, fromDownload, cacheDir, plat, binName, proxyAgent };
 
 // Only run when invoked as the CLI, so tests can require this file.
 if (require.main === module) {
@@ -198,8 +265,10 @@ if (require.main === module) {
   .catch((err) => {
     console.error(
       `crw-mcp: could not locate or download the ${key} binary.\n  ${err.message}\n` +
-        `  Set CRW_MCP_BINARY=/path/to/crw-mcp to use a local build, or install\n` +
-        `  from https://github.com/${REPO}/releases.`
+        `  Manual install: download ${plat.asset} from\n` +
+        `  https://github.com/${REPO}/releases/tag/v${VERSION}, check it against SHA256SUMS,\n` +
+        `  and put ${binName} in ${cacheDir()}\n` +
+        `  (or set CRW_MCP_BINARY=/path/to/${binName}). HTTPS_PROXY is honoured.`
     );
     process.exit(1);
   });

mcp/crw-mcp/test/download.test.js

@@ -9,18 +9,20 @@ const { test } = require("node:test");
 const assert = require("node:assert");
 const crypto = require("crypto");
 const fs = require("fs");
+const http = require("http");
 const https = require("https");
 const os = require("os");
 const path = require("path");
 const { Readable } = require("stream");
 
-const { fromDownload, cacheDir, plat, binName } = require("../bin/crw-mcp.js");
+const { fromDownload, cacheDir, plat, binName, proxyAgent } = require("../bin/crw-mcp.js");
 
 // Replace https.get with a queue of canned responses. The launcher holds the
 // same module object we mutate here, so this reaches the real code path.
-function stubHttps(queue) {
+function stubHttps(queue, seen = []) {
   const original = https.get;
-  https.get = (_url, _opts, cb) => {
+  https.get = (url, _opts, cb) => {
+    seen.push(url);
     const item = queue.shift();
     const body = item.body === undefined ? [] : [Buffer.from(item.body)];
     const res = Readable.from(body);
@@ -131,6 +133,111 @@ test("a verified archive is extracted and returned as an executable path", async
   });
 });
 
+test("a blocked download host falls back to api.github.com and is still verified", async () => {
+  await withTempCache(async () => {
+    const archive = buildTarGz(binName, "#!/bin/sh\nexit 0\n");
+    const digest = crypto.createHash("sha256").update(archive).digest("hex");
+    const release = JSON.stringify({
+      assets: [
+        { name: "SHA256SUMS", id: 11 },
+        { name: plat.asset, id: 22 },
+      ],
+    });
+    const seen = [];
+    const restore = stubHttps(
+      [
+        { status: 500 }, // github.com SHA256SUMS blocked
+        { body: release }, // api release lookup
+        { body: `${digest}  ${plat.asset}\n` }, // api asset 11
+        { status: 500 }, // github.com archive blocked
+        { body: release },
+        { body: archive }, // api asset 22
+      ],
+      seen
+    );
+    try {
+      const resolved = await fromDownload();
+      assert.ok(fs.existsSync(resolved));
+      assert.ok(seen.some((u) => u.endsWith("/releases/assets/22")));
+    } finally {
+      restore();
+    }
+  });
+});
+
+test("the api fallback still refuses a mismatched archive", async () => {
+  await withTempCache(async () => {
+    const release = JSON.stringify({
+      assets: [
+        { name: "SHA256SUMS", id: 11 },
+        { name: plat.asset, id: 22 },
+      ],
+    });
+    const restore = stubHttps([
+      { status: 500 },
+      { body: release },
+      { body: `${"0".repeat(64)}  ${plat.asset}\n` },
+      { status: 500 },
+      { body: release },
+      { body: "tampered" },
+    ]);
+    try {
+      await assert.rejects(fromDownload(), /checksum mismatch/);
+      assert.equal(fs.existsSync(path.join(cacheDir(), binName)), false);
+    } finally {
+      restore();
+    }
+  });
+});
+
+test("a 404 is final: the api host is not asked", async () => {
+  await withTempCache(async () => {
+    const seen = [];
+    const restore = stubHttps([{ status: 404 }], seen);
+    try {
+      await assert.rejects(fromDownload(), /publishes no SHA256SUMS/);
+      assert.equal(seen.length, 1);
+    } finally {
+      restore();
+    }
+  });
+});
+
+test("HTTPS_PROXY routes the download through a CONNECT tunnel", async () => {
+  const targets = [];
+  const proxy = http.createServer();
+  proxy.on("connect", (req, socket) => {
+    targets.push(req.url);
+    socket.end("HTTP/1.1 502 Bad Gateway\r\n\r\n");
+  });
+  await new Promise((r) => proxy.listen(0, "127.0.0.1", r));
+  const prev = process.env.HTTPS_PROXY;
+  process.env.HTTPS_PROXY = `http://127.0.0.1:${proxy.address().port}`;
+  try {
+    await withTempCache(async () => {
+      await assert.rejects(fromDownload(), /proxy CONNECT failed: HTTP 502/);
+    });
+    assert.equal(targets[0], "github.com:443");
+  } finally {
+    if (prev === undefined) delete process.env.HTTPS_PROXY;
+    else process.env.HTTPS_PROXY = prev;
+    proxy.close();
+  }
+});
+
+test("no proxy variable means no proxy agent", () => {
+  const saved = {};
+  for (const k of ["HTTPS_PROXY", "https_proxy", "HTTP_PROXY", "http_proxy"]) {
+    saved[k] = process.env[k];
+    delete process.env[k];
+  }
+  try {
+    assert.equal(proxyAgent(), undefined);
+  } finally {
+    for (const [k, v] of Object.entries(saved)) if (v !== undefined) process.env[k] = v;
+  }
+});
+
 // Build a real .tar.gz in-process so the extraction path runs for real.
 function buildTarGz(name, content, mode = "0000755") {
   const body = Buffer.from(content);