diff --git a/SESSION.md b/SESSION.md index b1a45a9..070beef 100644 --- a/SESSION.md +++ b/SESSION.md @@ -1,47 +1,9 @@ -# Session: Gmail Schema Availability Fix - -## Linked work - -- GitHub issue: https://github.com/brycejohnson1417/Picc-web-app/issues/181 -- Branch: `codex/181-gmail-schema-error` -- Production evidence: the signed-in Contacts page exposes a Prisma missing-table error for `public.GmailConnection` after clicking **Find contacts**. - -## Scope - -- Trace the Gmail lookup from Contacts through its server boundary and migration history. -- Add a focused regression test that proves database internals cannot leak through the Gmail suggestions API. -- Return a stable, actionable integration-unavailable state when the schema is not ready. -- Identify the exact pending production migration action required to restore Gmail lookup. - -## Out of scope - -- Gmail message sending or wider mailbox permissions. -- OAuth consent-screen, auth/RLS, or tenant access-control changes. -- Unrelated Prisma migrations, data backfills, or schema cleanup. -- Changes to `/Users/brycejohnson/Code/map-app`. - -## Constraints and architecture check - -- Keep Prisma access behind the existing Gmail server boundary. -- Preserve the current read-only Gmail integration and the Settings-based connection flow. -- Production schema changes are approval-lane work and cannot be run or merged without Bryce's explicit approval. -- Owned paths: `lib/server/gmail-connection.ts`, focused Gmail route tests, migration/deployment configuration only if evidence requires it, and `SESSION.md`. -- Open PRs checked: #166, #144, #135, and #82. None owns the Gmail integration or Prisma migration paths. - -## Validation plan - -- RED: reproduce the raw Prisma missing-table error leaking from the Gmail suggestions route. -- GREEN: map unavailable-schema failures to a stable user-safe response while retaining actionable Settings guidance. -- Run focused unit tests, then `npm run verify`. -- Run the targeted signed-in Contacts and Settings browser flow. -- After separately approved production migration/deployment work, verify the live route no longer reports a missing table. - -## Current state - -- Root-cause evidence: the Gmail migration exists at `prisma/migrations/202608141300_add_gmail_connections/migration.sql`, while the Vercel build command is only `npm run build`; the production browser reports the table absent. -- RED confirmed: route coverage reproduced raw missing-table leakage; browser coverage reproduced the Settings card's endless loading state. -- GREEN confirmed: Gmail status/suggestions return a stable 503 without Prisma details, and Settings renders a retryable setup state that recovers to the real connection control. -- `npm run verify`: passed lint, typecheck, 48 Vitest files / 207 tests, Prisma validation, and production build. -- Targeted Gmail browser flow: passed at 390x844 with error and recovered screenshots under `.agents/issue-181/`. -- Full `npm run test:e2e`: 34 passed / 3 unrelated failures (`/home` timeout, Google Map unavailable in the Safari-shell test, subway reload timeout). Serial rerun passed `/home` and subway; the Google Map availability assertion remained failed and is outside this PR's owned paths. -- No production schema changes have been run. +# Issue 107: Spreadsheet parser hardening + +Issue: https://github.com/brycejohnson1417/Picc-web-app/issues/107 +Branch: codex/107-spreadsheet-hardening, from main 4fde001. +Scope: replace the vulnerable registry SheetJS version with the official fixed distribution; bound spreadsheet inputs and retain required workbook/CSV behavior. +Owned: package.json, package-lock.json, lib/integrations/sheets.ts, lib/integrations/spreadsheet-input.ts and tests, lib/server/preferred-partner-proposal.ts, SESSION.md. +Out of scope: Prisma tooling, provider changes, schema, production data, UI redesign. +Overlap checked: #195/#194/#189/#182/#166/#135/#82. No source overlap with current changes except SESSION; inspect #135 proposal overlap before implementation and sequence/rebase as needed. Dependency updates exclusive. +Validation: RED malformed/oversized input tests first; GREEN representative workbook/CSV tests; clean npm ci, npm audit, npm run verify. Existing SheetJS API adapter retained; no new parser abstraction beyond input validation. diff --git a/lib/integrations/sheets.test.ts b/lib/integrations/sheets.test.ts new file mode 100644 index 0000000..47fe285 --- /dev/null +++ b/lib/integrations/sheets.test.ts @@ -0,0 +1,32 @@ +import { afterEach, expect, it } from 'vitest'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import * as XLSX from 'xlsx'; +import { inspectNabisWorkbook } from './sheets'; + +const folders: string[] = []; +function file(name: string, content: string | Buffer) { + const folder = mkdtempSync(join(tmpdir(), 'picc-sheet-test-')); + folders.push(folder); + const path = join(folder, name); + writeFileSync(path, content); + return path; +} +afterEach(() => folders.splice(0).forEach((folder) => rmSync(folder, { recursive: true }))); +it('rejects unsupported file types before parsing', () => { + expect(() => inspectNabisWorkbook(file('report.html', '
Accounts
'))).toThrow('file type'); +}); +it('rejects an oversized workbook before parsing', () => { + expect(() => inspectNabisWorkbook(file('report.xlsx', Buffer.alloc(10 * 1024 * 1024 + 1)))).toThrow('10 MB'); +}); +it('rejects text disguised as a workbook', () => { + expect(() => inspectNabisWorkbook(file('report.xlsx', 'Account,Value\nExample,12'))).toThrow('valid Excel'); +}); +it('preserves workbook tab coverage and row samples', () => { + const workbook = XLSX.utils.book_new(); + XLSX.utils.book_append_sheet(workbook, XLSX.utils.aoa_to_sheet([['Name','Value'],['Example',12]]), 'orders'); + const result = inspectNabisWorkbook(file('report.xlsx', XLSX.write(workbook, { type: 'buffer', bookType: 'xlsx' }))); + expect(result.tabs).toEqual(['orders']); + expect(result.sample).toContainEqual({ tab:'orders', header:['Name','Value'], firstDataRow:['Example',12], rowCount:2 }); +}); diff --git a/lib/integrations/sheets.ts b/lib/integrations/sheets.ts index ac09f3e..fab7932 100644 --- a/lib/integrations/sheets.ts +++ b/lib/integrations/sheets.ts @@ -1,8 +1,34 @@ +import { readFileSync, statSync } from 'node:fs'; +import { extname } from 'node:path'; +import { MAX_WORKBOOK_BYTES, MAX_SHEET_ROWS } from './spreadsheet-input'; import * as XLSX from 'xlsx'; import { NABIS_SCHEMA_MAPPING, REQUIRED_NABIS_TABS } from '@/lib/data/sheets-schema'; export function inspectNabisWorkbook(path: string) { - const workbook = XLSX.readFile(path, { cellDates: true }); + const extension = extname(path).toLowerCase(); + if (!['.xlsx', '.xls', '.csv'].includes(extension)) { + throw new Error('Unsupported spreadsheet file type. Use XLSX, XLS, or CSV.'); + } + const file = statSync(path); + if (!file.isFile() || file.size > MAX_WORKBOOK_BYTES) { + throw new Error('Workbook must be a regular file of at most 10 MB.'); + } + const bytes = readFileSync(path); + const zip = bytes.subarray(0, 4).equals(Buffer.from([0x50, 0x4b, 0x03, 0x04])); + const ole = bytes.subarray(0, 8).equals(Buffer.from([0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1])); + if ((extension === '.xlsx' && !zip) || (extension === '.xls' && !ole)) { + throw new Error('File is not a valid Excel workbook.'); + } + const workbook = XLSX.read(bytes, { type: 'buffer', cellDates: true, sheetRows: MAX_SHEET_ROWS + 1 }); + for (const sheet of Object.values(workbook.Sheets)) { + const range = sheet['!fullref'] || sheet['!ref']; + if (range && XLSX.utils.decode_range(range).e.c >= 256) { + throw new Error('Workbook exceeds the 256 column limit per sheet.'); + } + if (range && XLSX.utils.decode_range(range).e.r >= MAX_SHEET_ROWS) { + throw new Error('Workbook exceeds the 50,000 row limit per sheet.'); + } + } const tabs = workbook.SheetNames; const requiredCoverage = REQUIRED_NABIS_TABS.map((tab) => ({ diff --git a/lib/integrations/spreadsheet-input.test.ts b/lib/integrations/spreadsheet-input.test.ts new file mode 100644 index 0000000..f001e48 --- /dev/null +++ b/lib/integrations/spreadsheet-input.test.ts @@ -0,0 +1,22 @@ +import { expect, it } from 'vitest'; +import { requireReportSize, parseReportRows } from './spreadsheet-input'; + +it('accepts a normal CSV with quoted commas and Unicode', () => { + expect(() => requireReportSize('Name,Price\n"Café, large",12')).not.toThrow(); +}); +it('rejects oversized text by bytes rather than character count', () => { + expect(() => requireReportSize('é'.repeat(1024 * 1024 + 1))).toThrow('2 MB'); +}); + +it('preserves quoted commas, Unicode, leading zeros, and prices in CSV reports', () => { + expect(parseReportRows('Name,SKU,Price\n"Café, large",0012,12.50')).toEqual({ + format: 'csv', rows: [{ Name: 'Café, large', SKU: '0012', Price: '12.50' }], + }); +}); +it('retains JSON rows and rejects malformed JSON', () => { + expect(parseReportRows('[{"Name":"Example","Price":12}]').rows).toEqual([{Name:'Example',Price:12}]); + expect(() => parseReportRows('[broken')).toThrow(); +}); +it('rejects excessive columns without returning a truncated report', () => { + expect(() => parseReportRows(Array.from({length:257}, (_,i) => `column${i}`).join(','))).toThrow('256 column'); +}); diff --git a/lib/integrations/spreadsheet-input.ts b/lib/integrations/spreadsheet-input.ts new file mode 100644 index 0000000..31ff2ee --- /dev/null +++ b/lib/integrations/spreadsheet-input.ts @@ -0,0 +1,51 @@ +import * as XLSX from 'xlsx'; + +export const MAX_WORKBOOK_BYTES = 10 * 1024 * 1024; +export const MAX_REPORT_BYTES = 2 * 1024 * 1024; +export const MAX_SHEET_ROWS = 50_000; + +export function requireReportSize(input: string) { + if (Buffer.byteLength(input, 'utf8') > MAX_REPORT_BYTES) { + const error = new Error('Report exceeds the 2 MB limit. Split the export into smaller reports.'); + Object.assign(error, { statusCode: 400 }); + throw error; + } +} + +export function parseReportRows(rawInput: string) { + requireReportSize(rawInput); + const trimmed = rawInput.trim(); + if (!trimmed) { + return { format: 'json' as const, rows: [] as Record[] }; + } + + if (trimmed.startsWith('[') || trimmed.startsWith('{')) { + const parsed = JSON.parse(trimmed) as unknown; + if (Array.isArray(parsed)) { + return { + format: 'json' as const, + rows: parsed.filter((row): row is Record => Boolean(row) && typeof row === 'object'), + }; + } + throw new Error('Expected a JSON array of Headset rows.'); + } + + const workbook = XLSX.read(trimmed, { type: 'string', raw: false, sheetRows: MAX_SHEET_ROWS + 1 }); + const firstSheetName = workbook.SheetNames[0]; + if (!firstSheetName) { + return { format: 'csv' as const, rows: [] as Record[] }; + } + const sheet = workbook.Sheets[firstSheetName]; + const range = sheet['!fullref'] || sheet['!ref']; + if (range && XLSX.utils.decode_range(range).e.c >= 256) { + throw Object.assign(new Error('Report exceeds the 256 column limit.'), { statusCode: 400 }); + } + if (range && XLSX.utils.decode_range(range).e.r >= MAX_SHEET_ROWS) { + throw Object.assign(new Error('Report exceeds the 50,000 row limit.'), { statusCode: 400 }); + } + return { + format: 'csv' as const, + rows: XLSX.utils.sheet_to_json>(sheet, { defval: null, raw: false }), + }; +} + diff --git a/lib/server/preferred-partner-proposal.ts b/lib/server/preferred-partner-proposal.ts index 0ab33f5..37034b5 100644 --- a/lib/server/preferred-partner-proposal.ts +++ b/lib/server/preferred-partner-proposal.ts @@ -1,6 +1,6 @@ +import { parseReportRows } from '@/lib/integrations/spreadsheet-input'; import 'server-only'; -import * as XLSX from 'xlsx'; import { resolveAccountIdentity } from '@/lib/server/account-identity'; import { loadNabisDaysOffRows, loadNyInventoryRows, loadNyWarehouseRows } from '@/lib/server/nabis-api'; import { loadTerritoryStoreDetail } from '@/lib/server/notion-territory'; @@ -375,34 +375,6 @@ function calculateEarliestDeliveryDate(generatedAt: Date, daysOffRows: Array Boolean(row) && typeof row === 'object'), - }; - } - throw new Error('Expected a JSON array of Headset rows.'); - } - - const workbook = XLSX.read(trimmed, { type: 'string', raw: false }); - const firstSheetName = workbook.SheetNames[0]; - if (!firstSheetName) { - return { format: 'csv' as const, rows: [] as InputRow[] }; - } - const sheet = workbook.Sheets[firstSheetName]; - return { - format: 'csv' as const, - rows: XLSX.utils.sheet_to_json(sheet, { defval: null, raw: false }), - }; -} function normalizeHeadsetRows(rawRows: InputRow[]) { return rawRows @@ -1143,7 +1115,7 @@ export async function getPreferredPartnerProposal(input: { accountIdOrPageId: string; rawReport: string; }) { - const { format, rows: rawRows } = parseInputRows(input.rawReport); + const { format, rows: rawRows } = parseReportRows(input.rawReport); const parsedRows = normalizeHeadsetRows(rawRows); if (parsedRows.length === 0) { const error = new Error('Paste a Headset JSON array or CSV export before generating a PPP proposal.'); diff --git a/package-lock.json b/package-lock.json index 8200cc4..a5a8d80 100644 --- a/package-lock.json +++ b/package-lock.json @@ -46,7 +46,7 @@ "sharp": "^0.35.5", "sonner": "^1.7.2", "tailwind-merge": "^2.6.0", - "xlsx": "^0.18.5", + "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz", "zod": "^3.24.1" }, "devDependencies": { @@ -4861,15 +4861,6 @@ "node": ">=0.4.0" } }, - "node_modules/adler-32": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/adler-32/-/adler-32-1.3.1.tgz", - "integrity": "sha512-ynZ4w/nUUv5rrsR8UUGoe1VC9hZj6V5hU9Qw1HlMDJGEJw5S7TfTErWTjMys6M7vr0YWcPqs3qAr4ss0nDfP+A==", - "license": "Apache-2.0", - "engines": { - "node": ">=0.8" - } - }, "node_modules/ajv": { "version": "6.15.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.15.0.tgz", @@ -5324,19 +5315,6 @@ "node": ">=10.0.0" } }, - "node_modules/cfb": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/cfb/-/cfb-1.2.2.tgz", - "integrity": "sha512-KfdUZsSOw19/ObEWasvBP/Ac4reZvAGauZhs6S/gqNhXhI7cKwvlH7ulj+dOEYnca4bm4SGo8C1bTAQvnTjgQA==", - "license": "Apache-2.0", - "dependencies": { - "adler-32": "~1.3.0", - "crc-32": "~1.2.0" - }, - "engines": { - "node": ">=0.8" - } - }, "node_modules/chai": { "version": "6.2.2", "resolved": "https://registry.npmjs.org/chai/-/chai-6.2.2.tgz", @@ -5433,15 +5411,6 @@ "react-dom": "^18 || ^19 || ^19.0.0-rc" } }, - "node_modules/codepage": { - "version": "1.15.0", - "resolved": "https://registry.npmjs.org/codepage/-/codepage-1.15.0.tgz", - "integrity": "sha512-3g6NUTPd/YtuuGrhMnOMRjFc+LJw/bnMp3+0r/Wcz3IXUuCosKRJvMphm5+Q+bvTVGcJJuRvVLuYba+WojaFaA==", - "license": "Apache-2.0", - "engines": { - "node": ">=0.8" - } - }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -5515,18 +5484,6 @@ "url": "https://opencollective.com/core-js" } }, - "node_modules/crc-32": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.2.tgz", - "integrity": "sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==", - "license": "Apache-2.0", - "bin": { - "crc32": "bin/crc32.njs" - }, - "engines": { - "node": ">=0.8" - } - }, "node_modules/cross-spawn": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", @@ -6817,15 +6774,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/frac": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/frac/-/frac-1.1.2.tgz", - "integrity": "sha512-w/XBfkibaTl3YDqASwfDUqkna4Z2p9cFSr1aHDt0WoMTECnRfBOv2WArlZILlqgWlmdIlALXGpM2AOhEk5W3IA==", - "license": "Apache-2.0", - "engines": { - "node": ">=0.8" - } - }, "node_modules/fsevents": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", @@ -8778,6 +8726,7 @@ "version": "2.3.2", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, "hasInstallScript": true, "license": "MIT", "optional": true, @@ -9625,18 +9574,6 @@ "node": ">=0.10.0" } }, - "node_modules/ssf": { - "version": "0.11.2", - "resolved": "https://registry.npmjs.org/ssf/-/ssf-0.11.2.tgz", - "integrity": "sha512-+idbmIXoYET47hH+d7dfm2epdOMUDjqcB4648sTZ+t2JwoyBFL/insLfB/racrDmsKB3diwsDA696pZMieAC5g==", - "license": "Apache-2.0", - "dependencies": { - "frac": "~1.1.2" - }, - "engines": { - "node": ">=0.8" - } - }, "node_modules/stable-hash": { "version": "0.0.5", "resolved": "https://registry.npmjs.org/stable-hash/-/stable-hash-0.0.5.tgz", @@ -11463,24 +11400,6 @@ "node": ">=8" } }, - "node_modules/wmf": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/wmf/-/wmf-1.0.2.tgz", - "integrity": "sha512-/p9K7bEh0Dj6WbXg4JG0xvLQmIadrner1bi45VMJTfnbVHsc7yIajZyoSoK60/dtVBs12Fm6WkUI5/3WAVsNMw==", - "license": "Apache-2.0", - "engines": { - "node": ">=0.8" - } - }, - "node_modules/word": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/word/-/word-0.3.0.tgz", - "integrity": "sha512-OELeY0Q61OXpdUfTp+oweA/vtLVg5VDOXh+3he3PNzLGG/y0oylSOC1xRVj0+l4vQ3tj/bB1HVHv1ocXkQceFA==", - "license": "Apache-2.0", - "engines": { - "node": ">=0.8" - } - }, "node_modules/word-wrap": { "version": "1.2.5", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", @@ -11514,19 +11433,10 @@ } }, "node_modules/xlsx": { - "version": "0.18.5", - "resolved": "https://registry.npmjs.org/xlsx/-/xlsx-0.18.5.tgz", - "integrity": "sha512-dmg3LCjBPHZnQp5/F/+nnTa+miPJxUXB6vtk42YjBBKayDNagxGEeIdWApkYPOf3Z3pm3k62Knjzp7lMeTEtFQ==", + "version": "0.20.3", + "resolved": "https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz", + "integrity": "sha512-oLDq3jw7AcLqKWH2AhCpVTZl8mf6X2YReP+Neh0SJUzV/BdZYjth94tG5toiMB1PPrYtxOCfaoUCkvtuH+3AJA==", "license": "Apache-2.0", - "dependencies": { - "adler-32": "~1.3.0", - "cfb": "~1.2.1", - "codepage": "~1.15.0", - "crc-32": "~1.2.1", - "ssf": "~0.11.2", - "wmf": "~1.0.1", - "word": "~0.3.0" - }, "bin": { "xlsx": "bin/xlsx.njs" }, diff --git a/package.json b/package.json index b8eb0fb..25fd0ae 100644 --- a/package.json +++ b/package.json @@ -73,7 +73,7 @@ "sharp": "^0.35.5", "sonner": "^1.7.2", "tailwind-merge": "^2.6.0", - "xlsx": "^0.18.5", + "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz", "zod": "^3.24.1" }, "devDependencies": {