Download paywalled PDFs with the browser sign-in

SECURITY.md

@@ -117,6 +117,29 @@ metadata, or transport fragments. Authorization-context names separate
 declared access realms, but they cannot detect that the account behind a reused
 name has changed. Use a new context name when the intended account changes.
 
+`ghostget pdf <url>` with a browser sign-in option (`--cookie-source`,
+`--browser-profile`, `--cookie-profile`, `--auth`, or `--cookies-file`)
+downloads the PDF itself instead of passing the link on anonymously. It uses
+only HTTPS and DNS-pinned public addresses, follows at most five redirects,
+and refuses credential-bearing links, plain HTTP, private IP literals, and
+local names such as `.local` or single-label hosts before any cookie is read
+for that host. A public name that resolves to a private address is refused
+before any request is sent. It reads cookies separately for each host it visits
+through the same cookie filter as `read`, so a site receives only the cookies
+that belong to it, never another host's. When a redirect leaves the link's own
+site (any host other than the link's host or a parent or subdomain of it), the
+new site does not receive its `SameSite=Strict` cookies, as a browser would
+hold them back after a redirect started elsewhere. It does not launch the
+browser profile, so no profile egress consent applies, and
+`--trust-profile-egress` or `--mode` copied from `read` are ignored. The whole
+download, including every cookie read, is bounded by `--timeout-ms`; the
+response is bounded by `--max-pdf-bytes`, must start with a PDF signature, and
+is streamed into an owner-only file in an owner-only temporary folder that is
+removed after the import. If the site refuses and none of the selected
+browser's cookies went to it, the error says no sign-in was found rather than
+that the sign-in lacks access. Cookie values are not printed, logged, or
+stored.
+
 The unauthenticated direct-media adapter intentionally permits loopback and
 private-network HTTP(S) targets because its URL is supplied by the local user.
 It is not an SSRF boundary for remotely supplied URLs. It reads a bounded

package.json

@@ -387,6 +387,7 @@
     "src/oauth-google.ts",
     "src/oauth-x.ts",
     "src/path-helper.ts",
+    "src/pdf-auth.ts",
     "src/persistent-helper-bridge.ts",
     "src/pinned-https.ts",
     "src/plan-assets.ts",

scripts/npm-release-workflow.test.ts

@@ -1220,7 +1220,7 @@ describe("npm publication contract", () => {
         (MAX_UNPACKED_BYTES + MAX_PACKED_ENTRIES * 1_023 + 1_024) / 512,
       ) * 512,
     );
-    expect(MAX_PACKAGE_TAR_BYTES).toBe(24_615_424);
+    expect(MAX_PACKAGE_TAR_BYTES).toBe(24_658_944);
     expect(MAX_PACKAGE_TAR_BYTES % 512).toBe(0);
     expect(artifact).toContain("maxOutputLength: MAX_PACKAGE_TAR_BYTES");
     expect(artifact).not.toContain("const maximumTarBytes");
@@ -1430,8 +1430,8 @@ describe("npm publication contract", () => {
     expect(budget).toContain("785b8fa60c329d7ac46bc8fcf4d959b5fa9d96455bba9e7cdea63f6a3827c4f6");
     expect(Object.isFrozen(repairPackageMeasurement)).toBeTrue();
     expect(repairPackageMeasurement).toMatchObject({
-      archiveSha256: "9641f93ab7dd2c52174cf2ddf3e41f407ebca562c46e0dd0af2e6cd01447c2de",
-      packedBytes: 12_132_235, unpackedBytes: 23_980_585, entryCount: 619,
+      archiveSha256: "e34c1d14165fdc4985280f7d56626bcd17055de9e2906bc9db1569807ed568ab",
+      packedBytes: 12_133_122, unpackedBytes: 24_023_022, entryCount: 620,
       packedPlatformProjection: 12_387, packedPortabilityAllowance: 4_096,
       payloadPlatformProjection: 353, payloadAllowance: 65,
     });
@@ -1441,8 +1441,10 @@ describe("npm publication contract", () => {
     expect(budget).toContain("12,141,169 packed; 23,937,025 + 353 + 65 = 23,937,443 unpacked");
     expect(budget).toContain("23,930,250 + 353 + 65 = 23,930,668 unpacked");
     expect(budget).toContain("12,141,373 packed; 23,937,545 + 353 + 65 = 23,937,963 unpacked");
-    expect(MAX_PACKED_BYTES).toBe(12_148_718);
-    expect(MAX_PACKED_BYTES).toBe(12_132_235 + 12_387 + 4_096);
+    expect(budget).toContain("12,133,122 + 12,387 + 4,096 = 12,149,605 packed");
+    expect(budget).toContain("24,023,022 + 353 + 65 = 24,023,440 unpacked");
+    expect(MAX_PACKED_BYTES).toBe(12_149_605);
+    expect(MAX_PACKED_BYTES).toBe(12_133_122 + 12_387 + 4_096);
     expect(budget).toContain("aa127b3193c9bb3b0cb5deece5927be60ccb7111a50169320d322ffdeaa13f39");
     expect(budget).toContain("0c331bab3ab3df69a108e18f5f29845b0db90c281cbd6455c0d90fa0b24081e2");
     expect(budget).toContain("873cad8139fda303e2d19c6afd61cf549cf9b4d1d76b2a1d6d632a6afe6bd0d1");
@@ -1537,8 +1539,8 @@ describe("npm publication contract", () => {
     expect(budget).toContain("11,696,091 + 4,096 = 11,700,187");
     expect(budget).toContain("35449445752 attempt 1, package job 105913938839");
     expect(budget).toContain("exactly 596 files");
-    expect(MAX_PACKED_ENTRIES).toBe(619);
-    expect(MAX_PACKED_FILES).toBe(619);
+    expect(MAX_PACKED_ENTRIES).toBe(620);
+    expect(MAX_PACKED_FILES).toBe(620);
     expect(budget).toContain("Ghostget 0.18.6 same-boot setup-cleanup candidate over main edbe567");
     expect(budget).toContain("11,656,173");
     expect(budget).toContain("22,513,450 payload bytes across exactly 557 files");
@@ -1563,7 +1565,7 @@ describe("npm publication contract", () => {
     expect(budget).toContain("47684b3e2eb5cf3ed07fbb520aade8c7251d993f75262fbf1af627d9081a1a5f");
     expect(budget).toContain("23,688,277 + 353 + 65 = 23,688,695");
     expect(budget).toContain("23,759,283 + 353 + 65 = 23,759,701");
-    expect(MAX_UNPACKED_BYTES).toBe(23_981_003);
+    expect(MAX_UNPACKED_BYTES).toBe(24_023_440);
     expect(budget).toContain("23,037,873 + 65 = 23,037,938");
     expect(budget).toContain("f9f3ab38a682690ceaa2699a7309997512030f0fa500a9dc29dcd108123dc41f");
     expect(budget).toContain("23,038,557 + 65 = 23,038,622");
@@ -1596,7 +1598,7 @@ describe("npm publication contract", () => {
     expect(budget).toContain("01875f12ab73a49d6c7d6bf520dc3d318db816addee2fa7981889f35c958cf7c");
     expect(budget).toContain("b12909f08f7c19460ced56e30619f4860a1183f4b0106170c07837dae577a937");
     expect(budget).toContain("0b212ac291218528dcf979370110a36f10850e046ca90a536057d9a44e807d1d");
-    expect(MAX_UNPACKED_BYTES).toBe(23_980_585 + 353 + 65);
+    expect(MAX_UNPACKED_BYTES).toBe(24_023_022 + 353 + 65);
     expect(budget).toContain("22,794,052 + 65 = 22,794,117");
     expect(budget).toContain("c482efe748f880e3717727d6d39fd92a68953e6eea766642b329ba47ae772d80");
     expect(budget).toContain("22,759,423 + 65 = 22,759,488");
@@ -1630,10 +1632,10 @@ describe("npm publication contract", () => {
       expect(Object.isFrozen(range)).toBe(true);
     }
     expect(packageArtifactBudget).toEqual({
-      entryCount: { min: 619, max: 619 },
-      fileCount: { min: 619, max: 619 },
-      packedBytes: { min: 1_600_000, max: 12_148_718 },
-      unpackedBytes: { min: 9_000_000, max: 23_981_003 },
+      entryCount: { min: 620, max: 620 },
+      fileCount: { min: 620, max: 620 },
+      packedBytes: { min: 1_600_000, max: 12_149_605 },
+      unpackedBytes: { min: 9_000_000, max: 24_023_440 },
     });
   });
 

scripts/package-budget.ts

@@ -2130,15 +2130,27 @@
 // Retain the same platform projections and portability allowances:
 // 12,132,235 + 12,387 + 4,096 = 12,148,718 packed;
 // 23,980,585 + 353 + 65 = 23,981,003 unpacked.
+//
+// Signed-in `ghostget pdf <url>` downloads add src/pdf-auth.ts to the packed
+// source and a lazy import from cli.ts; no dist chunk changes. Over merged
+// main 65944e7 (which carries the X contacts.list snapshot above), the new
+// source file grows the inventory to 620 entries. After `bun run build`, a
+// clean npm 11.19.0 pack --ignore-scripts with Node 24.20.0 on darwin arm64
+// measured 620 entries, 12,133,122 packed bytes, and 24,023,022 unpacked
+// bytes; archive SHA-256
+// e34c1d14165fdc4985280f7d56626bcd17055de9e2906bc9db1569807ed568ab.
+// Retain the same platform projections and portability allowances:
+// 12,133,122 + 12,387 + 4,096 = 12,149,605 packed;
+// 24,023,022 + 353 + 65 = 24,023,440 unpacked.
 export const repairPackageMeasurement = Object.freeze({
-  scope: "X contacts.list follow-collection qualification over merged main 46e31838 (Ghostget 0.18.46)",
+  scope: "Ghostget signed-in PDF downloads over merged main 65944e7",
   command: "npm pack --ignore-scripts",
-  npmVersion: "11.16.0",
+  npmVersion: "11.19.0",
   platform: "darwin-arm64",
-  archiveSha256: "9641f93ab7dd2c52174cf2ddf3e41f407ebca562c46e0dd0af2e6cd01447c2de",
-  packedBytes: 12_132_235,
-  unpackedBytes: 23_980_585,
-  entryCount: 619,
+  archiveSha256: "e34c1d14165fdc4985280f7d56626bcd17055de9e2906bc9db1569807ed568ab",
+  packedBytes: 12_133_122,
+  unpackedBytes: 24_023_022,
+  entryCount: 620,
   packedPlatformProjection: 12_387,
   packedPortabilityAllowance: 4_096,
   payloadPlatformProjection: 353,

skills/ghostget/SKILL.md

@@ -36,6 +36,7 @@ automation.
 
 - Capture a URL: `ghostget <url>` or `ghostget clip <url>`.
 - Read without persistence: `ghostget read <url>`.
+- Save a PDF as a note: `ghostget pdf <file-or-url>`. For a paywalled paper the user can open through a library or university sign-in, add that browser: `ghostget pdf <url> --cookie-source chrome` (or `--browser-profile <name>`, `--auth <id>`, `--cookies-file <path>`). Ghostget then downloads over HTTPS itself, sends each site only its own cookies, and fails with "does not seem to have access" when the site returns a sign-in page, or with "no ... sign-in was found" when that browser or profile has none for the site (`ghostget browsers` lists profiles); ask the user to open the link in that browser first rather than retrying. Without these options the download stays anonymous.
 - Archive media: `ghostget archive <url>` or `ghostget audio|video|transcript <url>`.
 - Discover supported article embeds through the provider's bounded semantic media read, then archive each exact returned finite item separately. Do not treat a collection page as one media item or scrape its DOM to manufacture asset routes.
 - Inspect support: `ghostget plugin list`, `ghostget plugin show <id>`, and `ghostget capabilities [adapter]`. For a typed, schema-backed projection use `ghostget contracts catalog --json`; check a read-only collection plan with `ghostget contracts check --plan <file> --json`.

src/cli.test.ts

@@ -230,6 +230,44 @@ describe("lazy ghostget CLI entrypoint", () => {
     }
   });
 
+  test("pdf without sign-in options still delegates unchanged; with them Ghostget handles it first", async () => {
+    const previousExitCode = process.exitCode;
+    try {
+      const received: (readonly string[])[] = [];
+      const stdout: string[] = [];
+      const loadKnowledge = () => Promise.resolve({
+        main: (raw?: readonly string[]) => {
+          received.push(raw ?? []);
+          return Promise.resolve(0);
+        },
+      });
+      const noRuntime = () => {
+        throw new Error("pdf must not load the provider runtime");
+      };
+      await runGhostgetCliProcess(
+        ["pdf", "https://example.org/a.pdf", "--json"],
+        { stdout: () => undefined, stderr: () => undefined },
+        noRuntime,
+        noRuntime,
+        loadKnowledge,
+      );
+      expect(received).toEqual([["pdf", "https://example.org/a.pdf", "--json"]]);
+
+      await runGhostgetCliProcess(
+        ["pdf", "https://example.org/a.pdf", "--cookie-source", "netscape", "--json"],
+        { stdout: (text) => stdout.push(text), stderr: () => undefined },
+        noRuntime,
+        noRuntime,
+        loadKnowledge,
+      );
+      expect(received).toHaveLength(1);
+      expect(process.exitCode).toBe(2);
+      expect(JSON.parse(stdout.join(""))).toMatchObject({ ok: false, error: { code: "usage" } });
+    } finally {
+      process.exitCode = previousExitCode ?? 0;
+    }
+  });
+
   test("routes only complete capabilities and plugin inspection shapes", async () => {
     expect(routedGhostgetCatalogCommand(["capabilities", "--json"])).toEqual({
       command: "capabilities",
@@ -490,7 +528,7 @@ throw new Error("private fallback did not load a forbidden module");
 
   test("has only static help and release identity as eager dependencies and bounds startup CPU work", async () => {
     const source = readFileSync(cliPath, "utf8");
-    expect(source).toContain('import { ghostgetBareUsage, ghostgetHelpRequest } from "./usage"');
+    expect(source).toContain('import { ghostgetBareUsage, ghostgetHelpRequest, hasPdfSignInOptions } from "./usage"');
     expect(source).toContain('import { cliStyle, renderCliError } from "./cli-style"');
     expect(source).toContain('import { GHOSTGET_VERSION } from "./version"');
     expect(source).toContain('import("./ghostget")');

src/cli.ts

@@ -1,6 +1,6 @@
 #!/usr/bin/env bun
 
-import { ghostgetBareUsage, ghostgetHelpRequest } from "./usage";
+import { ghostgetBareUsage, ghostgetHelpRequest, hasPdfSignInOptions } from "./usage";
 import { cliStyle, renderCliError } from "./cli-style";
 import { terminalIntro } from "./cli-intro";
 import { GHOSTGET_VERSION } from "./version";
@@ -290,6 +290,16 @@ export async function runGhostgetCliProcess(
     process.exitCode = await support.runGhostgetSupportCommand(rawArguments.slice(1), resolvedOutput);
     return;
   }
+  if (rawArguments[0] === "pdf" && help === null && hasPdfSignInOptions(rawArguments.slice(1))) {
+    const { runSignedInPdfCommand, runWordcellPdfWithDownload } = await import("./pdf-auth");
+    process.exitCode = await runSignedInPdfCommand(rawArguments.slice(1), resolvedOutput, {
+      environment: process.env,
+      runWordcellPdf: async (pdfArguments, download) => download === undefined
+        ? (await loadKnowledgeCli()).main(["pdf", ...pdfArguments], resolvedOutput)
+        : runWordcellPdfWithDownload(pdfArguments, download, process.env, resolvedOutput),
+    });
+    return;
+  }
   if (isPublicGhostgetCommand(rawArguments)) {
     const knowledge = await loadKnowledgeCli();
     process.exitCode = await knowledge.main(rawArguments, resolvedOutput);

src/fixtures/cli-help/topic-notes.txt

@@ -16,3 +16,12 @@ Commands
   adapters [--json]                  List page-capture adapters
 
 Most commands take --root <directory> to pick the notes folder.
+
+Paywalled PDFs
+  Add the browser you use to open the paper, and Ghostget downloads it
+  with that sign-in:
+    ghostget pdf <url> --cookie-source chrome
+  Also: --browser-profile <name|path>, --cookie-profile <name>,
+  --auth <id> or --cookies-file <path>. Each site gets only its own cookies.
+  If the site sends a sign-in page instead, open the link in your browser
+  first, then run it again.