[BUG]Excel powerqueries only works while ignoring privacy labels

app/api/src/export/excelWorkbook.js

@@ -60,7 +60,20 @@ function buildReadMeSheet(wb) {
     { text: '     - Paste the M code from the sheet, then click Done → Close & Load.' },
     { text: '     - The query name in Power Query becomes the table name on this sheet.' },
     { text: '' },
-    { text: '3. Refresh: Data → Refresh All. Or right-click a query → Refresh.' },
+    { text: '3. The first time a query runs, Excel asks about data privacy. This is expected —', style: { font: { bold: true } } },
+    { text: '   you must allow it once per workbook or no query will return data:' },
+    { text: '     - Every query reads BaseUrl / AuthToken from this workbook and sends them to' },
+    { text: '       the Identity Atlas API. Power Query treats that as combining two data' },
+    { text: '       sources and shows "Information is required about data privacy".' },
+    { text: '     - Click Continue, then tick "Ignore the Privacy Levels and potentially' },
+    { text: '       improve performance" and click Save.' },
+    { text: '     - You can also set it up front: File → Options and settings → Query Options' },
+    { text: '       → CURRENT WORKBOOK → Privacy → "Ignore the Privacy Levels...".' },
+    { text: '     - Use the Current Workbook setting, not the Global one. It is safe here: the' },
+    { text: '       only two sources involved are this file and the Identity Atlas API whose' },
+    { text: '       read token this file already carries, so no data can leak anywhere new.' },
+    { text: '' },
+    { text: '4. Refresh: Data → Refresh All. Or right-click a query → Refresh.' },
     { text: '' },
     { text: 'Security notes', style: { font: { bold: true, size: 12 } } },
     { text: '' },
@@ -133,9 +146,11 @@ function buildQuerySheet(wb, query) {
   sheet.getCell('A2').font = { italic: true, color: { argb: 'FF6B7280' } };
 
   // Instructions
-  sheet.getCell('A4').value = 'Paste the M code below into Power Query: Data → Get Data → Other Sources → Blank Query → Advanced Editor.';
+  sheet.getCell('A4').value = 'Paste the M code below into Power Query: Data → Get Data → Other Sources → Blank Query → Advanced Editor. '
+    + 'On first run Excel asks "Information is required about data privacy" — click Continue and tick "Ignore the Privacy Levels..." '
+    + 'for this workbook (see the README sheet). The query returns no data until you do.';
   sheet.getCell('A4').alignment = { wrapText: true };
-  sheet.getRow(4).height = 36;
+  sheet.getRow(4).height = 48;
 
   // The M code itself, in a single cell with monospace font and wrap
   const mCell = sheet.getCell('A6');

app/api/src/export/excelWorkbook.test.js

@@ -112,6 +112,33 @@ describe('generateWorkbook', () => {
     expect(resources).toContain('includeBusinessRoles = "true"');
   });
 
+  it('documents the Power Query privacy-level prompt in the workbook README (issue #819)', () => {
+    // Every generated query combines two Formula-Firewall data sources:
+    // Excel.CurrentWorkbook() (the BaseUrl/AuthToken named ranges) feeding
+    // Web.Contents() on a dynamic URL. First evaluation therefore raises
+    // Excel's "Information is required about data privacy" prompt, and the
+    // refresh only proceeds once privacy-level checking is ignored for this
+    // workbook. The README must walk the user through that dialog —
+    // otherwise the feature reads as broken.
+    const readme = wb.getWorksheet('README');
+    const lines = [];
+    readme.eachRow((row) => row.eachCell((cell) => lines.push(String(cell.value ?? ''))));
+    const text = lines.join('\n');
+    expect(text).toMatch(/privacy/i);
+    expect(text).toMatch(/ignore.*privacy level/i);
+  });
+
+  it('repeats the privacy-prompt warning on every query sheet, next to the M code', () => {
+    // The prompt fires right after the user pastes the M code and clicks Done
+    // — i.e. while they are looking at a query sheet, not the README. The
+    // paste instruction on each sheet therefore has to mention it too.
+    for (const q of QUERIES) {
+      const instructions = String(wb.getWorksheet(q.sheet).getCell('A4').value ?? '');
+      expect(instructions).toMatch(/privacy/i);
+      expect(instructions).toMatch(/ignore.*privacy level/i);
+    }
+  });
+
   it('auto-expands the extendedAttributes JSONB column', () => {
     // Users of the workbook expect sub-keys (userType, onPremisesSyncEnabled,
     // etc.) to appear as first-class columns, not "Record" cells they have

app/ui/e2e/powerquery-workbook-export.spec.js

@@ -0,0 +1,107 @@
+// @ts-check
+//
+// Excel Power Query workbook export — the reporter path for #819.
+//
+// Every query in the generated workbook reads BaseUrl / AuthToken out of the
+// workbook's named ranges and feeds them to Web.Contents, so Power Query's
+// Formula Firewall raises "Information is required about data privacy" the
+// first time a pasted query evaluates, and the query returns no data until
+// privacy levels are ignored for that workbook. That step is mandatory, and
+// the workbook's own instructions used to omit it — so a user following them
+// hit an undocumented modal and concluded the export was broken.
+//
+// Playwright can't drive Excel, so this walks the part of the path that is
+// ours: Admin → Data → Excel Power Query Workbook → "Generate token &
+// download workbook", then opens the downloaded .xlsx and checks that the
+// instructions a user reads — the README sheet, and the paste instruction on
+// each query sheet, which is what's on screen when the prompt fires — walk
+// them through the privacy dialog. It also replays what the M code does
+// (bearer token → read endpoint) to prove the workbook works once the dialog
+// is cleared.
+import { test, expect } from '@playwright/test';
+import ExcelJS from 'exceljs';
+
+const BASE = process.env.E2E_BASE_URL || 'http://localhost:3001';
+
+// "Ignore the Privacy Levels…" is the checkbox the user has to tick; the
+// prompt itself is titled "Information is required about data privacy".
+const MENTIONS_PRIVACY = /privacy/i;
+const MENTIONS_IGNORE_STEP = /ignore.*privacy level/i;
+
+async function openDataAdminTab(page) {
+  await page.goto(`${BASE}/#admin?sub=data`);
+  await page.waitForLoadState('networkidle');
+  const button = page.getByRole('button', { name: /Generate token & download workbook/i });
+  if (!(await button.isVisible({ timeout: 15000 }).catch(() => false))) {
+    test.skip(true, 'Power Query export section not available to this user (needs data.export.ui)');
+  }
+  return button;
+}
+
+async function downloadWorkbook(page) {
+  const button = await openDataAdminTab(page);
+  const [download] = await Promise.all([
+    page.waitForEvent('download', { timeout: 60000 }),
+    button.click(),
+  ]);
+  const filePath = await download.path();
+  expect(filePath, 'no workbook file was downloaded').toBeTruthy();
+  expect(download.suggestedFilename()).toMatch(/\.xlsx$/);
+
+  const wb = new ExcelJS.Workbook();
+  await wb.xlsx.readFile(filePath);
+  return wb;
+}
+
+function sheetText(sheet) {
+  const parts = [];
+  sheet.eachRow((row) => row.eachCell((cell) => parts.push(String(cell.value ?? ''))));
+  return parts.join('\n');
+}
+
+test.describe('Excel Power Query workbook export', () => {
+  test.setTimeout(120000);
+
+  test('the downloaded workbook documents the mandatory privacy-level step', async ({ page }) => {
+    const wb = await downloadWorkbook(page);
+
+    const readme = wb.getWorksheet('README');
+    expect(readme, 'the workbook has no README sheet').toBeTruthy();
+    const readmeText = sheetText(readme);
+    expect(readmeText, 'the README never mentions the data-privacy prompt').toMatch(MENTIONS_PRIVACY);
+    expect(readmeText, 'the README never tells the user to ignore privacy levels').toMatch(MENTIONS_IGNORE_STEP);
+    // The safe setting is the per-workbook one — the global switch would turn
+    // privacy checking off for every file the user opens.
+    expect(readmeText).toMatch(/current workbook/i);
+
+    // The prompt fires while the user is looking at a query sheet (they've just
+    // pasted its M code), so each of those sheets has to say so too.
+    const querySheets = wb.worksheets.filter(ws => !['README', 'Settings'].includes(ws.name));
+    expect(querySheets.length, 'the workbook has no query sheets').toBeGreaterThan(0);
+    for (const sheet of querySheets) {
+      const instructions = String(sheet.getCell('A4').value ?? '');
+      expect(instructions, `${sheet.name}: paste instructions omit the privacy prompt`).toMatch(MENTIONS_PRIVACY);
+      expect(instructions, `${sheet.name}: paste instructions omit the ignore step`).toMatch(MENTIONS_IGNORE_STEP);
+    }
+  });
+
+  test('the embedded token reads the API the way the M code does', async ({ page, request }) => {
+    // Proves the documented dialog is the only thing standing between the user
+    // and their data: same bearer token, same read endpoint the Systems tab's
+    // M code hits (the tab in the bug report).
+    const wb = await downloadWorkbook(page);
+
+    const settings = wb.getWorksheet('Settings');
+    expect(settings, 'the workbook has no Settings sheet').toBeTruthy();
+    const baseUrl = String(settings.getCell('B2').value ?? '');
+    const token = String(settings.getCell('B3').value ?? '');
+    expect(baseUrl, 'BaseUrl cell is not an /api base').toMatch(/\/api$/);
+    expect(token, 'AuthToken cell holds no read token').toMatch(/^fgr_/);
+
+    const res = await request.get(`${BASE}/api/systems`, {
+      headers: { Authorization: `Bearer ${token}` },
+    });
+    expect(res.status(), 'the workbook token cannot read /api/systems').toBe(200);
+    expect(Array.isArray(await res.json())).toBe(true);
+  });
+});

changes/dor-issue-819.md

@@ -0,0 +1,3 @@
+- The Excel Power Query workbook now explains the "Information is required about data privacy" prompt Excel raises the first time a query runs, and walks you through the mandatory "Ignore the Privacy Levels" step — on the README sheet and on every query sheet, next to the M code you paste. Previously the prompt appeared with no explanation and the queries returned no data, making the export look broken.
+- Documented the same step in the Excel Power Query export guide, including why the per-workbook setting (never the global one) is the safe choice.
+- The Excel template authoring guide now tells maintainers to save the per-workbook privacy setting into the template, and to verify it survived the save.

docs/admin/excel-powerquery-export.md

@@ -70,12 +70,46 @@ up.
      editor. Click **Done**.
    - Rename the query to match the sheet name (e.g. `Principals`).
    - **Home → Close & Load To… → Existing worksheet → that sheet, cell A1**.
-5. Back in Excel, **Data → Refresh All**. Every sheet fills with live data.
+5. **The first query you load raises a data-privacy prompt — allow it once for
+   this workbook.** See below; queries return no data until you do.
+6. Back in Excel, **Data → Refresh All**. Every sheet fills with live data.
+
+### The "Information is required about data privacy" prompt
+
+!!! warning "Expected, and mandatory — this is not an error"
+    The first time a pasted query evaluates, Excel shows
+    **"Information is required about data privacy"**. The queries only run once
+    privacy-level checking is switched off **for this workbook**.
+
+Every query reads `BaseUrl` and `AuthToken` from the workbook's named ranges
+and sends them to the Identity Atlas API. Power Query's Formula Firewall sees
+that as two data sources being combined — the workbook and the web API — and
+because the `Web.Contents` URL comes from a cell rather than a literal, setting
+individual privacy levels does not reliably satisfy it. Ignoring privacy levels
+does.
+
+To clear it:
+
+1. On the prompt, click **Continue**.
+2. Tick **"Ignore the Privacy Levels and potentially improve performance"**.
+3. Click **Save**. Refresh again — the data loads.
+
+You can also set it before pasting any query: **File → Options and settings →
+Query Options → CURRENT WORKBOOK → Privacy → "Ignore the Privacy Levels and
+potentially improve performance"**.
+
+!!! danger "Use the Current Workbook setting, never the Global one"
+    The **Global** option turns privacy checking off for every workbook you open
+    — don't touch it. The per-workbook setting is safe here: the only two
+    sources involved are this file and the Identity Atlas API whose read token
+    the file already carries, so it creates no path for data to leak anywhere
+    new.
 
 > **One-click refresh coming soon.** The current workbook requires the
 > paste-into-Advanced-Editor step per sheet. A follow-up PR ships a
-> hand-built template where the queries auto-load on first open — at
-> that point the flow becomes "download → open → Refresh All".
+> hand-built template where the queries auto-load on first open and the
+> per-workbook privacy setting is already saved in the file — at that
+> point the flow becomes "download → open → Refresh All".
 
 ## Rotating or retiring tokens
 

docs/admin/excel-template-authoring.md

@@ -79,7 +79,28 @@ Define two named ranges (Formulas → Name Manager → New):
 > `Excel.CurrentWorkbook(){[Name="BaseUrl"]}[Content]{0}[Column1]`. If the
 > names go missing, every query stops working at refresh time.
 
-### 4. Add one Power Query per data sheet
+### 4. Turn off privacy-level checking for this workbook (do this first)
+
+Because every query feeds workbook-sourced values (`BaseUrl`, `AuthToken`) into
+`Web.Contents`, Power Query's Formula Firewall raises **"Information is required
+about data privacy"** the first time a query evaluates, and the queries do not
+run until privacy levels are ignored for the workbook.
+
+**File → Options and settings → Query Options → CURRENT WORKBOOK → Privacy →
+"Ignore the Privacy Levels and potentially improve performance" → OK.**
+
+Set this **before** step 5 so you aren't interrupted mid-way, and never touch
+the Global option — it would disable privacy checking for every workbook on
+your machine.
+
+This setting is saved *inside* the workbook, so it travels with the committed
+template. That is the whole point: it is what lets end users skip the dialog
+that the M-as-text MVP forces them through (see
+[the privacy-prompt section](./excel-powerquery-export.md#the-information-is-required-about-data-privacy-prompt)).
+**Verify it survived the save** in the sanity check below — if it didn't, the
+template gives users no advantage over the MVP on this point.
+
+### 5. Add one Power Query per data sheet
 
 For each of the 7 data sheets:
 
@@ -96,7 +117,7 @@ For each of the 7 data sheets:
 Repeat for all 7 sheets. After each one, click **Refresh All** to confirm
 data loads correctly. If a query fails, fix the M code and re-load.
 
-### 5. Replace cell values with placeholders, save, do NOT refresh again
+### 6. Replace cell values with placeholders, save, do NOT refresh again
 
 Once everything refreshes cleanly:
 
@@ -131,6 +152,10 @@ Before committing:
 3. **Don't click Refresh** — there's nothing to refresh against.
 4. Right-click each query in the Queries pane → **Properties** → confirm
    the query name matches the sheet name.
+5. **File → Options and settings → Query Options → CURRENT WORKBOOK →
+   Privacy** — confirm "Ignore the Privacy Levels…" is still selected. If it
+   reverted, redo step 4 and save again; otherwise every user of the template
+   hits the privacy dialog on their first refresh.
 
 ## What the backend does at download time