From 077acb0bed2af8b5da5340c981cb0382cd12764b Mon Sep 17 00:00:00 2001 From: "fortigi-ci-bot[bot]" <280718603+fortigi-ci-bot[bot]@users.noreply.github.com> Date: Mon, 10 Aug 2026 13:13:22 +0000 Subject: [PATCH 1/2] test: reproduce #819 (red) --- app/api/src/export/excelWorkbook.test.js | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/app/api/src/export/excelWorkbook.test.js b/app/api/src/export/excelWorkbook.test.js index 9c16d13b0..6c4086c9c 100644 --- a/app/api/src/export/excelWorkbook.test.js +++ b/app/api/src/export/excelWorkbook.test.js @@ -112,6 +112,22 @@ describe('generateWorkbook', () => { expect(resources).toContain('includeBusinessRoles = "true"'); }); + it('documents the Power Query privacy-level prompt in the workbook README (issue #819)', () => { + // Every generated query combines two Formula-Firewall data sources: + // Excel.CurrentWorkbook() (the BaseUrl/AuthToken named ranges) feeding + // Web.Contents() on a dynamic URL. First evaluation therefore raises + // Excel's "Information is required about data privacy" prompt, and the + // refresh only proceeds once privacy-level checking is ignored for this + // workbook. The README must walk the user through that dialog — + // otherwise the feature reads as broken. + const readme = wb.getWorksheet('README'); + const lines = []; + readme.eachRow((row) => row.eachCell((cell) => lines.push(String(cell.value ?? '')))); + const text = lines.join('\n'); + expect(text).toMatch(/privacy/i); + expect(text).toMatch(/ignore.*privacy level/i); + }); + it('auto-expands the extendedAttributes JSONB column', () => { // Users of the workbook expect sub-keys (userType, onPremisesSyncEnabled, // etc.) to appear as first-class columns, not "Record" cells they have From e1636df1605e17af2df8a8ef50aaa61040b45e43 Mon Sep 17 00:00:00 2001 From: "fortigi-ci-bot[bot]" <280718603+fortigi-ci-bot[bot]@users.noreply.github.com> Date: Mon, 10 Aug 2026 13:20:44 +0000 Subject: [PATCH 2/2] [BUG]Excel powerqueries only works while ignoring privacy labels (#819) --- app/api/src/export/excelWorkbook.js | 21 +++- app/api/src/export/excelWorkbook.test.js | 11 ++ app/ui/e2e/powerquery-workbook-export.spec.js | 107 ++++++++++++++++++ changes/dor-issue-819.md | 3 + docs/admin/excel-powerquery-export.md | 40 ++++++- docs/admin/excel-template-authoring.md | 29 ++++- 6 files changed, 203 insertions(+), 8 deletions(-) create mode 100644 app/ui/e2e/powerquery-workbook-export.spec.js create mode 100644 changes/dor-issue-819.md diff --git a/app/api/src/export/excelWorkbook.js b/app/api/src/export/excelWorkbook.js index bc5a3396b..a76368b28 100644 --- a/app/api/src/export/excelWorkbook.js +++ b/app/api/src/export/excelWorkbook.js @@ -60,7 +60,20 @@ function buildReadMeSheet(wb) { { text: ' - Paste the M code from the sheet, then click Done → Close & Load.' }, { text: ' - The query name in Power Query becomes the table name on this sheet.' }, { text: '' }, - { text: '3. Refresh: Data → Refresh All. Or right-click a query → Refresh.' }, + { text: '3. The first time a query runs, Excel asks about data privacy. This is expected —', style: { font: { bold: true } } }, + { text: ' you must allow it once per workbook or no query will return data:' }, + { text: ' - Every query reads BaseUrl / AuthToken from this workbook and sends them to' }, + { text: ' the Identity Atlas API. Power Query treats that as combining two data' }, + { text: ' sources and shows "Information is required about data privacy".' }, + { text: ' - Click Continue, then tick "Ignore the Privacy Levels and potentially' }, + { text: ' improve performance" and click Save.' }, + { text: ' - You can also set it up front: File → Options and settings → Query Options' }, + { text: ' → CURRENT WORKBOOK → Privacy → "Ignore the Privacy Levels...".' }, + { text: ' - Use the Current Workbook setting, not the Global one. It is safe here: the' }, + { text: ' only two sources involved are this file and the Identity Atlas API whose' }, + { text: ' read token this file already carries, so no data can leak anywhere new.' }, + { text: '' }, + { text: '4. Refresh: Data → Refresh All. Or right-click a query → Refresh.' }, { text: '' }, { text: 'Security notes', style: { font: { bold: true, size: 12 } } }, { text: '' }, @@ -133,9 +146,11 @@ function buildQuerySheet(wb, query) { sheet.getCell('A2').font = { italic: true, color: { argb: 'FF6B7280' } }; // Instructions - sheet.getCell('A4').value = 'Paste the M code below into Power Query: Data → Get Data → Other Sources → Blank Query → Advanced Editor.'; + sheet.getCell('A4').value = 'Paste the M code below into Power Query: Data → Get Data → Other Sources → Blank Query → Advanced Editor. ' + + 'On first run Excel asks "Information is required about data privacy" — click Continue and tick "Ignore the Privacy Levels..." ' + + 'for this workbook (see the README sheet). The query returns no data until you do.'; sheet.getCell('A4').alignment = { wrapText: true }; - sheet.getRow(4).height = 36; + sheet.getRow(4).height = 48; // The M code itself, in a single cell with monospace font and wrap const mCell = sheet.getCell('A6'); diff --git a/app/api/src/export/excelWorkbook.test.js b/app/api/src/export/excelWorkbook.test.js index 6c4086c9c..3e989bd96 100644 --- a/app/api/src/export/excelWorkbook.test.js +++ b/app/api/src/export/excelWorkbook.test.js @@ -128,6 +128,17 @@ describe('generateWorkbook', () => { expect(text).toMatch(/ignore.*privacy level/i); }); + it('repeats the privacy-prompt warning on every query sheet, next to the M code', () => { + // The prompt fires right after the user pastes the M code and clicks Done + // — i.e. while they are looking at a query sheet, not the README. The + // paste instruction on each sheet therefore has to mention it too. + for (const q of QUERIES) { + const instructions = String(wb.getWorksheet(q.sheet).getCell('A4').value ?? ''); + expect(instructions).toMatch(/privacy/i); + expect(instructions).toMatch(/ignore.*privacy level/i); + } + }); + it('auto-expands the extendedAttributes JSONB column', () => { // Users of the workbook expect sub-keys (userType, onPremisesSyncEnabled, // etc.) to appear as first-class columns, not "Record" cells they have diff --git a/app/ui/e2e/powerquery-workbook-export.spec.js b/app/ui/e2e/powerquery-workbook-export.spec.js new file mode 100644 index 000000000..4ce5baa21 --- /dev/null +++ b/app/ui/e2e/powerquery-workbook-export.spec.js @@ -0,0 +1,107 @@ +// @ts-check +// +// Excel Power Query workbook export — the reporter path for #819. +// +// Every query in the generated workbook reads BaseUrl / AuthToken out of the +// workbook's named ranges and feeds them to Web.Contents, so Power Query's +// Formula Firewall raises "Information is required about data privacy" the +// first time a pasted query evaluates, and the query returns no data until +// privacy levels are ignored for that workbook. That step is mandatory, and +// the workbook's own instructions used to omit it — so a user following them +// hit an undocumented modal and concluded the export was broken. +// +// Playwright can't drive Excel, so this walks the part of the path that is +// ours: Admin → Data → Excel Power Query Workbook → "Generate token & +// download workbook", then opens the downloaded .xlsx and checks that the +// instructions a user reads — the README sheet, and the paste instruction on +// each query sheet, which is what's on screen when the prompt fires — walk +// them through the privacy dialog. It also replays what the M code does +// (bearer token → read endpoint) to prove the workbook works once the dialog +// is cleared. +import { test, expect } from '@playwright/test'; +import ExcelJS from 'exceljs'; + +const BASE = process.env.E2E_BASE_URL || 'http://localhost:3001'; + +// "Ignore the Privacy Levels…" is the checkbox the user has to tick; the +// prompt itself is titled "Information is required about data privacy". +const MENTIONS_PRIVACY = /privacy/i; +const MENTIONS_IGNORE_STEP = /ignore.*privacy level/i; + +async function openDataAdminTab(page) { + await page.goto(`${BASE}/#admin?sub=data`); + await page.waitForLoadState('networkidle'); + const button = page.getByRole('button', { name: /Generate token & download workbook/i }); + if (!(await button.isVisible({ timeout: 15000 }).catch(() => false))) { + test.skip(true, 'Power Query export section not available to this user (needs data.export.ui)'); + } + return button; +} + +async function downloadWorkbook(page) { + const button = await openDataAdminTab(page); + const [download] = await Promise.all([ + page.waitForEvent('download', { timeout: 60000 }), + button.click(), + ]); + const filePath = await download.path(); + expect(filePath, 'no workbook file was downloaded').toBeTruthy(); + expect(download.suggestedFilename()).toMatch(/\.xlsx$/); + + const wb = new ExcelJS.Workbook(); + await wb.xlsx.readFile(filePath); + return wb; +} + +function sheetText(sheet) { + const parts = []; + sheet.eachRow((row) => row.eachCell((cell) => parts.push(String(cell.value ?? '')))); + return parts.join('\n'); +} + +test.describe('Excel Power Query workbook export', () => { + test.setTimeout(120000); + + test('the downloaded workbook documents the mandatory privacy-level step', async ({ page }) => { + const wb = await downloadWorkbook(page); + + const readme = wb.getWorksheet('README'); + expect(readme, 'the workbook has no README sheet').toBeTruthy(); + const readmeText = sheetText(readme); + expect(readmeText, 'the README never mentions the data-privacy prompt').toMatch(MENTIONS_PRIVACY); + expect(readmeText, 'the README never tells the user to ignore privacy levels').toMatch(MENTIONS_IGNORE_STEP); + // The safe setting is the per-workbook one — the global switch would turn + // privacy checking off for every file the user opens. + expect(readmeText).toMatch(/current workbook/i); + + // The prompt fires while the user is looking at a query sheet (they've just + // pasted its M code), so each of those sheets has to say so too. + const querySheets = wb.worksheets.filter(ws => !['README', 'Settings'].includes(ws.name)); + expect(querySheets.length, 'the workbook has no query sheets').toBeGreaterThan(0); + for (const sheet of querySheets) { + const instructions = String(sheet.getCell('A4').value ?? ''); + expect(instructions, `${sheet.name}: paste instructions omit the privacy prompt`).toMatch(MENTIONS_PRIVACY); + expect(instructions, `${sheet.name}: paste instructions omit the ignore step`).toMatch(MENTIONS_IGNORE_STEP); + } + }); + + test('the embedded token reads the API the way the M code does', async ({ page, request }) => { + // Proves the documented dialog is the only thing standing between the user + // and their data: same bearer token, same read endpoint the Systems tab's + // M code hits (the tab in the bug report). + const wb = await downloadWorkbook(page); + + const settings = wb.getWorksheet('Settings'); + expect(settings, 'the workbook has no Settings sheet').toBeTruthy(); + const baseUrl = String(settings.getCell('B2').value ?? ''); + const token = String(settings.getCell('B3').value ?? ''); + expect(baseUrl, 'BaseUrl cell is not an /api base').toMatch(/\/api$/); + expect(token, 'AuthToken cell holds no read token').toMatch(/^fgr_/); + + const res = await request.get(`${BASE}/api/systems`, { + headers: { Authorization: `Bearer ${token}` }, + }); + expect(res.status(), 'the workbook token cannot read /api/systems').toBe(200); + expect(Array.isArray(await res.json())).toBe(true); + }); +}); diff --git a/changes/dor-issue-819.md b/changes/dor-issue-819.md new file mode 100644 index 000000000..af54ac79e --- /dev/null +++ b/changes/dor-issue-819.md @@ -0,0 +1,3 @@ +- The Excel Power Query workbook now explains the "Information is required about data privacy" prompt Excel raises the first time a query runs, and walks you through the mandatory "Ignore the Privacy Levels" step — on the README sheet and on every query sheet, next to the M code you paste. Previously the prompt appeared with no explanation and the queries returned no data, making the export look broken. +- Documented the same step in the Excel Power Query export guide, including why the per-workbook setting (never the global one) is the safe choice. +- The Excel template authoring guide now tells maintainers to save the per-workbook privacy setting into the template, and to verify it survived the save. diff --git a/docs/admin/excel-powerquery-export.md b/docs/admin/excel-powerquery-export.md index 81d123121..819608cad 100644 --- a/docs/admin/excel-powerquery-export.md +++ b/docs/admin/excel-powerquery-export.md @@ -70,12 +70,46 @@ up. editor. Click **Done**. - Rename the query to match the sheet name (e.g. `Principals`). - **Home → Close & Load To… → Existing worksheet → that sheet, cell A1**. -5. Back in Excel, **Data → Refresh All**. Every sheet fills with live data. +5. **The first query you load raises a data-privacy prompt — allow it once for + this workbook.** See below; queries return no data until you do. +6. Back in Excel, **Data → Refresh All**. Every sheet fills with live data. + +### The "Information is required about data privacy" prompt + +!!! warning "Expected, and mandatory — this is not an error" + The first time a pasted query evaluates, Excel shows + **"Information is required about data privacy"**. The queries only run once + privacy-level checking is switched off **for this workbook**. + +Every query reads `BaseUrl` and `AuthToken` from the workbook's named ranges +and sends them to the Identity Atlas API. Power Query's Formula Firewall sees +that as two data sources being combined — the workbook and the web API — and +because the `Web.Contents` URL comes from a cell rather than a literal, setting +individual privacy levels does not reliably satisfy it. Ignoring privacy levels +does. + +To clear it: + +1. On the prompt, click **Continue**. +2. Tick **"Ignore the Privacy Levels and potentially improve performance"**. +3. Click **Save**. Refresh again — the data loads. + +You can also set it before pasting any query: **File → Options and settings → +Query Options → CURRENT WORKBOOK → Privacy → "Ignore the Privacy Levels and +potentially improve performance"**. + +!!! danger "Use the Current Workbook setting, never the Global one" + The **Global** option turns privacy checking off for every workbook you open + — don't touch it. The per-workbook setting is safe here: the only two + sources involved are this file and the Identity Atlas API whose read token + the file already carries, so it creates no path for data to leak anywhere + new. > **One-click refresh coming soon.** The current workbook requires the > paste-into-Advanced-Editor step per sheet. A follow-up PR ships a -> hand-built template where the queries auto-load on first open — at -> that point the flow becomes "download → open → Refresh All". +> hand-built template where the queries auto-load on first open and the +> per-workbook privacy setting is already saved in the file — at that +> point the flow becomes "download → open → Refresh All". ## Rotating or retiring tokens diff --git a/docs/admin/excel-template-authoring.md b/docs/admin/excel-template-authoring.md index 0c340a624..a35c0a3e3 100644 --- a/docs/admin/excel-template-authoring.md +++ b/docs/admin/excel-template-authoring.md @@ -79,7 +79,28 @@ Define two named ranges (Formulas → Name Manager → New): > `Excel.CurrentWorkbook(){[Name="BaseUrl"]}[Content]{0}[Column1]`. If the > names go missing, every query stops working at refresh time. -### 4. Add one Power Query per data sheet +### 4. Turn off privacy-level checking for this workbook (do this first) + +Because every query feeds workbook-sourced values (`BaseUrl`, `AuthToken`) into +`Web.Contents`, Power Query's Formula Firewall raises **"Information is required +about data privacy"** the first time a query evaluates, and the queries do not +run until privacy levels are ignored for the workbook. + +**File → Options and settings → Query Options → CURRENT WORKBOOK → Privacy → +"Ignore the Privacy Levels and potentially improve performance" → OK.** + +Set this **before** step 5 so you aren't interrupted mid-way, and never touch +the Global option — it would disable privacy checking for every workbook on +your machine. + +This setting is saved *inside* the workbook, so it travels with the committed +template. That is the whole point: it is what lets end users skip the dialog +that the M-as-text MVP forces them through (see +[the privacy-prompt section](./excel-powerquery-export.md#the-information-is-required-about-data-privacy-prompt)). +**Verify it survived the save** in the sanity check below — if it didn't, the +template gives users no advantage over the MVP on this point. + +### 5. Add one Power Query per data sheet For each of the 7 data sheets: @@ -96,7 +117,7 @@ For each of the 7 data sheets: Repeat for all 7 sheets. After each one, click **Refresh All** to confirm data loads correctly. If a query fails, fix the M code and re-load. -### 5. Replace cell values with placeholders, save, do NOT refresh again +### 6. Replace cell values with placeholders, save, do NOT refresh again Once everything refreshes cleanly: @@ -131,6 +152,10 @@ Before committing: 3. **Don't click Refresh** — there's nothing to refresh against. 4. Right-click each query in the Queries pane → **Properties** → confirm the query name matches the sheet name. +5. **File → Options and settings → Query Options → CURRENT WORKBOOK → + Privacy** — confirm "Ignore the Privacy Levels…" is still selected. If it + reverted, redo step 4 and save again; otherwise every user of the template + hits the privacy dialog on their first refresh. ## What the backend does at download time