From bc51605e76ed6c260047650b701143f7455bd96d Mon Sep 17 00:00:00 2001 From: "claude[bot]" <41898282+claude[bot]@users.noreply.github.com> Date: Fri, 17 Apr 2026 12:10:29 +0000 Subject: [PATCH 001/160] Fix #92: Detail tabs now update correctly when switching between entities When switching between user, resource, or other detail tabs, React was reusing the same component instance with stale state. Added unique key props to force component remount, ensuring data updates immediately. Co-Authored-By: Claude Sonnet 4.5 --- app/ui/src/App.jsx | 14 +++++++------- changes/fix-issue-92.md | 1 + 2 files changed, 8 insertions(+), 7 deletions(-) create mode 100644 changes/fix-issue-92.md diff --git a/app/ui/src/App.jsx b/app/ui/src/App.jsx index e77036ee8..d4b95d59f 100644 --- a/app/ui/src/App.jsx +++ b/app/ui/src/App.jsx @@ -313,38 +313,38 @@ export default function App() { if (page.startsWith('user:')) { const id = page.substring(5); const cacheKey = `user:${id}`; - return closeDetailTab('user', id)} onOpenDetail={openDetailTab} />; + return closeDetailTab('user', id)} onOpenDetail={openDetailTab} />; } if (page.startsWith('resource:')) { const id = page.substring(9); const cacheKey = `resource:${id}`; - return closeDetailTab('resource', id)} onOpenDetail={openDetailTab} />; + return closeDetailTab('resource', id)} onOpenDetail={openDetailTab} />; } if (page.startsWith('group:')) { // Backward compat: #group:id opens ResourceDetailPage const id = page.substring(6); const cacheKey = `group:${id}`; - return closeDetailTab('group', id)} onOpenDetail={openDetailTab} />; + return closeDetailTab('group', id)} onOpenDetail={openDetailTab} />; } if (page.startsWith('access-package:')) { const id = page.substring(15); const cacheKey = `access-package:${id}`; - return closeDetailTab('access-package', id)} />; + return closeDetailTab('access-package', id)} />; } if (page.startsWith('department:')) { const name = page.substring(11); const cacheKey = `department:${name}`; - return closeDetailTab('department', name)} onOpenDetail={openDetailTab} />; + return closeDetailTab('department', name)} onOpenDetail={openDetailTab} />; } if (page.startsWith('context:')) { const id = page.substring(8); const cacheKey = `context:${id}`; - return closeDetailTab('context', id)} onOpenDetail={openDetailTab} />; + return closeDetailTab('context', id)} onOpenDetail={openDetailTab} />; } if (page.startsWith('identity:')) { const id = page.substring(9); const cacheKey = `identity:${id}`; - return closeDetailTab('identity', id)} onOpenDetail={openDetailTab} />; + return closeDetailTab('identity', id)} onOpenDetail={openDetailTab} />; } return null; }; diff --git a/changes/fix-issue-92.md b/changes/fix-issue-92.md new file mode 100644 index 000000000..a437b1fcb --- /dev/null +++ b/changes/fix-issue-92.md @@ -0,0 +1 @@ +- Fixed detail page tabs not updating when switching between different users, resources, or other entities — tabs now show the correct entity data immediately From 5e976f7f0dd6a3853b0235f22b424cbdc194d35e Mon Sep 17 00:00:00 2001 From: "claude[bot]" <41898282+claude[bot]@users.noreply.github.com> Date: Fri, 17 Apr 2026 12:13:19 +0000 Subject: [PATCH 002/160] Fix #94: Org Chart now shows all departments when scrolled Changed the org chart container from `flex justify-center` to `inline-flex min-w-full justify-center` to ensure all department nodes remain accessible during horizontal scrolling. The previous centering behavior caused edge departments to fall outside the scrollable area. Co-Authored-By: Claude Sonnet 4.5 --- app/ui/src/components/OrgChartPage.jsx | 2 +- changes/fix-issue-94.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changes/fix-issue-94.md diff --git a/app/ui/src/components/OrgChartPage.jsx b/app/ui/src/components/OrgChartPage.jsx index 833feebe1..a0d1df975 100644 --- a/app/ui/src/components/OrgChartPage.jsx +++ b/app/ui/src/components/OrgChartPage.jsx @@ -676,7 +676,7 @@ export default function OrgChartPage({ onOpenDetail, onCacheData }) { {/* Org chart */}
-
+
Date: Fri, 17 Apr 2026 12:18:25 +0000 Subject: [PATCH 003/160] Fix #96: Display Extended Attributes objects as formatted JSON instead of [object Object] --- app/ui/src/utils/formatters.js | 1 + changes/fix-issue-96.md | 1 + 2 files changed, 2 insertions(+) create mode 100644 changes/fix-issue-96.md diff --git a/app/ui/src/utils/formatters.js b/app/ui/src/utils/formatters.js index ffd88cc59..4da4465cf 100644 --- a/app/ui/src/utils/formatters.js +++ b/app/ui/src/utils/formatters.js @@ -9,6 +9,7 @@ export function formatValue(val) { if (val === null || val === undefined) return '\u2014'; if (val === true) return 'Yes'; if (val === false) return 'No'; + if (typeof val === 'object') return JSON.stringify(val, null, 2); if (typeof val === 'string' && val.match(/^\d{4}-\d{2}-\d{2}T/)) return formatDate(val); return String(val); } diff --git a/changes/fix-issue-96.md b/changes/fix-issue-96.md new file mode 100644 index 000000000..04f7b1042 --- /dev/null +++ b/changes/fix-issue-96.md @@ -0,0 +1 @@ +- Fixed Extended Attributes displaying "[object Object]" for complex values like sign in activity — now shows properly formatted JSON From 305af6731f409d6c2d97609fa33ed1b924ac306d Mon Sep 17 00:00:00 2001 From: Taeke Date: Fri, 17 Apr 2026 15:04:13 +0200 Subject: [PATCH 004/160] Fix bump-version workflow to use VERSION_BUMP_PAT to bypass branch protection --- .github/workflows/bump-version.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/bump-version.yml b/.github/workflows/bump-version.yml index ad2ec614f..2677a4918 100644 --- a/.github/workflows/bump-version.yml +++ b/.github/workflows/bump-version.yml @@ -31,7 +31,7 @@ jobs: - uses: actions/checkout@v4 with: ref: main - token: ${{ secrets.GITHUB_TOKEN }} + token: ${{ secrets.VERSION_BUMP_PAT }} - name: Merge changelog fragments and bump version shell: pwsh From 2cc2c213fb8772d6a3e7c2483a5ada2c0771f382 Mon Sep 17 00:00:00 2001 From: Taeke Date: Fri, 17 Apr 2026 15:04:36 +0200 Subject: [PATCH 005/160] Add changelog fragment for bump-version PAT fix --- changes/fix-bump-version-pat.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/fix-bump-version-pat.md diff --git a/changes/fix-bump-version-pat.md b/changes/fix-bump-version-pat.md new file mode 100644 index 000000000..58d0b856b --- /dev/null +++ b/changes/fix-bump-version-pat.md @@ -0,0 +1 @@ +- Fixed automated version bumps failing due to branch protection requiring pull requests From c9cb6b7c308f402f9502f476dbc6b8158d9880a6 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 17 Apr 2026 13:06:07 +0000 Subject: [PATCH 006/160] chore: bump version to 5.1.20260417.1306 --- CHANGES.md | 33 +++++++++++++++++++ .../bugfixes-fix-risk-scoring-card-link.md | 1 - changes/docs-sbom.md | 2 -- changes/feature-risk-scoring-scheduler.md | 4 --- changes/fix-65-attribute-select-all.md | 1 - changes/fix-bump-version-pat.md | 1 - changes/fix-dashboard-risk-scoring-link.md | 1 - changes/fix-demo-direct-reports.md | 1 - .../fix-issue-67-account-correlation-ui.md | 3 -- changes/fix-issue-72.md | 1 - changes/fix-issue-81.md | 1 - changes/fix-issue-92.md | 1 - changes/fix-issue-94.md | 1 - changes/fix-issue-96.md | 1 - ...x-mssql-shim-boolean-and-sysutcdatetime.md | 2 -- changes/fix-sync-log-empty-message.md | 1 - ...ue-69-container-stats-historical-graphs.md | 1 - changes/issue-70-json-syntax-highlighting.md | 1 - changes/quickstart-upgrade-pull-always.md | 1 - changes/unified-issue-workflow.md | 5 --- setup/IdentityAtlas.psd1 | 2 +- 21 files changed, 34 insertions(+), 31 deletions(-) delete mode 100644 changes/bugfixes-fix-risk-scoring-card-link.md delete mode 100644 changes/docs-sbom.md delete mode 100644 changes/feature-risk-scoring-scheduler.md delete mode 100644 changes/fix-65-attribute-select-all.md delete mode 100644 changes/fix-bump-version-pat.md delete mode 100644 changes/fix-dashboard-risk-scoring-link.md delete mode 100644 changes/fix-demo-direct-reports.md delete mode 100644 changes/fix-issue-67-account-correlation-ui.md delete mode 100644 changes/fix-issue-72.md delete mode 100644 changes/fix-issue-81.md delete mode 100644 changes/fix-issue-92.md delete mode 100644 changes/fix-issue-94.md delete mode 100644 changes/fix-issue-96.md delete mode 100644 changes/fix-mssql-shim-boolean-and-sysutcdatetime.md delete mode 100644 changes/fix-sync-log-empty-message.md delete mode 100644 changes/issue-69-container-stats-historical-graphs.md delete mode 100644 changes/issue-70-json-syntax-highlighting.md delete mode 100644 changes/quickstart-upgrade-pull-always.md delete mode 100644 changes/unified-issue-workflow.md diff --git a/CHANGES.md b/CHANGES.md index 0f1f05d33..87a424722 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,36 @@ +## Changes in this PR + +- Fixed Risk Scoring card on Dashboard to link directly to Admin → Risk Scoring subtab +- Added Software Bill of Materials (SBOM) documentation listing all components, dependencies, and infrastructure elements +- Fixed SBOM navigation entry in mkdocs.yml to ensure proper documentation site build +- Added automatic scheduling for risk scoring runs (similar to crawler scheduling) +- Risk classifiers can now have multiple schedules configured via Admin → Risk Classifiers +- Scheduled scoring runs execute in the background and re-score all entities with the active classifier +- Schedules support hourly, daily, and weekly frequencies +- Added "Select All" and "Deselect All" buttons to the attribute picker in the EntraID crawler wizard, making it easier to manage large attribute lists +- Fixed automated version bumps failing due to branch protection requiring pull requests +- Fixed Dashboard Risk Scoring card link to properly navigate to Admin → Risk Scoring sub-tab +- Fixed direct reports not showing in demo dataset (org chart queries now filter for current records in temporal Principals table) +- Added in-browser wizard for generating account correlation rulesets via LLM (Admin → Account Correlation) +- Users can now create correlation signals and account type rules through a conversational UI instead of PowerShell commands +- Wizard follows the same pattern as Risk Scoring: Sources → Generate & Refine → Save +- Fixed Risk Scoring page not refreshing automatically after completing the risk profile wizard +- Fixed crawler schedules not firing when created via legacy wizard (scheduler now supports both `schedule` and `schedules` config formats) +- Fixed detail page tabs not updating when switching between different users, resources, or other entities — tabs now show the correct entity data immediately +- Fixed Org Chart UI so all departments are visible when scrolled horizontally; departments no longer fall off the edge of the viewable area +- Fixed Extended Attributes displaying "[object Object]" for complex values like sign in activity — now shows properly formatted JSON +- Automated version bumping on PR merge — `bump-version.yml` Action increments the Minor version in `setup/IdentityAtlas.psd1` and updates the timestamp on every PR merge to `main`. Branches no longer touch the version file, eliminating recurring merge conflicts on `setup/IdentityAtlas.psd1`. +- Automated changelog merging on PR merge — branches now create a fragment file in `changes/` instead of editing `CHANGES.md` directly. The same `bump-version.yml` Action collects all fragments and prepends them to `CHANGES.md` on merge, eliminating recurring merge conflicts on `CHANGES.md`. +- Fixed Sync Log empty state message to reference adding a crawler instead of Start-FGSync +- Added historical performance graphs to Containers tab showing last 10 minutes of CPU, memory, and network usage for each container +- Added syntax highlighting and collapsible sections to JSON display in risk profile wizard for improved readability +- Fixed upgrade instructions to use `--pull always` instead of a separate `pull` + `up -d`, so a single command always fetches the latest image from the registry +- Merged issue triage and nightly auto-fix into a single unified workflow that classifies and fixes bugs immediately when an issue is opened +- Added automatic issue classification: bugs vs feature requests, with priority labels (critical, high, medium, low) +- Feature requests are now labeled as `enhancement` with a priority but require manual triage before auto-fix +- Added nightly re-evaluation job (23:00 Amsterdam time) that checks issues labeled `needs-clarification` or `cant-autofix` for new comments — if enough detail has been added, the issue is promoted to `ready-to-fix` and auto-fixed +- Fixed auto-fix prompt to use changelog fragments instead of editing CHANGES.md and setup/IdentityAtlas.psd1 directly + ## Changes in this branch - **Auto-fix workflow: CI-validated fixes with retry** — Reworked the nightly auto-fix into a two-attempt pipeline: Claude investigates and fixes the issue, submits a draft PR, waits for the full CI pipeline (integration tests, Playwright E2E, load tests) to validate it. If CI fails, Claude gets the failure logs and tries to fix the broken tests in a second attempt. Three possible outcomes: `auto-fixed` (PR + CI green), `cant-autofix` with reason (couldn't produce a fix), or `cant-autofix` with PR link (fix exists but CI still failing — needs human review). The prompt now instructs Claude to think step-by-step before implementing. Issues labeled `cant-autofix` or `auto-fixed` are excluded from future runs (remove the label to retry). diff --git a/changes/bugfixes-fix-risk-scoring-card-link.md b/changes/bugfixes-fix-risk-scoring-card-link.md deleted file mode 100644 index 4f292a860..000000000 --- a/changes/bugfixes-fix-risk-scoring-card-link.md +++ /dev/null @@ -1 +0,0 @@ -- Fixed Risk Scoring card on Dashboard to link directly to Admin → Risk Scoring subtab diff --git a/changes/docs-sbom.md b/changes/docs-sbom.md deleted file mode 100644 index 245d52a74..000000000 --- a/changes/docs-sbom.md +++ /dev/null @@ -1,2 +0,0 @@ -- Added Software Bill of Materials (SBOM) documentation listing all components, dependencies, and infrastructure elements -- Fixed SBOM navigation entry in mkdocs.yml to ensure proper documentation site build diff --git a/changes/feature-risk-scoring-scheduler.md b/changes/feature-risk-scoring-scheduler.md deleted file mode 100644 index 6bbeb33b8..000000000 --- a/changes/feature-risk-scoring-scheduler.md +++ /dev/null @@ -1,4 +0,0 @@ -- Added automatic scheduling for risk scoring runs (similar to crawler scheduling) -- Risk classifiers can now have multiple schedules configured via Admin → Risk Classifiers -- Scheduled scoring runs execute in the background and re-score all entities with the active classifier -- Schedules support hourly, daily, and weekly frequencies diff --git a/changes/fix-65-attribute-select-all.md b/changes/fix-65-attribute-select-all.md deleted file mode 100644 index 625332824..000000000 --- a/changes/fix-65-attribute-select-all.md +++ /dev/null @@ -1 +0,0 @@ -- Added "Select All" and "Deselect All" buttons to the attribute picker in the EntraID crawler wizard, making it easier to manage large attribute lists diff --git a/changes/fix-bump-version-pat.md b/changes/fix-bump-version-pat.md deleted file mode 100644 index 58d0b856b..000000000 --- a/changes/fix-bump-version-pat.md +++ /dev/null @@ -1 +0,0 @@ -- Fixed automated version bumps failing due to branch protection requiring pull requests diff --git a/changes/fix-dashboard-risk-scoring-link.md b/changes/fix-dashboard-risk-scoring-link.md deleted file mode 100644 index 6764a3f89..000000000 --- a/changes/fix-dashboard-risk-scoring-link.md +++ /dev/null @@ -1 +0,0 @@ -- Fixed Dashboard Risk Scoring card link to properly navigate to Admin → Risk Scoring sub-tab \ No newline at end of file diff --git a/changes/fix-demo-direct-reports.md b/changes/fix-demo-direct-reports.md deleted file mode 100644 index c874bca38..000000000 --- a/changes/fix-demo-direct-reports.md +++ /dev/null @@ -1 +0,0 @@ -- Fixed direct reports not showing in demo dataset (org chart queries now filter for current records in temporal Principals table) diff --git a/changes/fix-issue-67-account-correlation-ui.md b/changes/fix-issue-67-account-correlation-ui.md deleted file mode 100644 index 478bba651..000000000 --- a/changes/fix-issue-67-account-correlation-ui.md +++ /dev/null @@ -1,3 +0,0 @@ -- Added in-browser wizard for generating account correlation rulesets via LLM (Admin → Account Correlation) -- Users can now create correlation signals and account type rules through a conversational UI instead of PowerShell commands -- Wizard follows the same pattern as Risk Scoring: Sources → Generate & Refine → Save diff --git a/changes/fix-issue-72.md b/changes/fix-issue-72.md deleted file mode 100644 index 5d6098318..000000000 --- a/changes/fix-issue-72.md +++ /dev/null @@ -1 +0,0 @@ -- Fixed Risk Scoring page not refreshing automatically after completing the risk profile wizard \ No newline at end of file diff --git a/changes/fix-issue-81.md b/changes/fix-issue-81.md deleted file mode 100644 index f11e15578..000000000 --- a/changes/fix-issue-81.md +++ /dev/null @@ -1 +0,0 @@ -- Fixed crawler schedules not firing when created via legacy wizard (scheduler now supports both `schedule` and `schedules` config formats) diff --git a/changes/fix-issue-92.md b/changes/fix-issue-92.md deleted file mode 100644 index a437b1fcb..000000000 --- a/changes/fix-issue-92.md +++ /dev/null @@ -1 +0,0 @@ -- Fixed detail page tabs not updating when switching between different users, resources, or other entities — tabs now show the correct entity data immediately diff --git a/changes/fix-issue-94.md b/changes/fix-issue-94.md deleted file mode 100644 index 1aeeb66e8..000000000 --- a/changes/fix-issue-94.md +++ /dev/null @@ -1 +0,0 @@ -- Fixed Org Chart UI so all departments are visible when scrolled horizontally; departments no longer fall off the edge of the viewable area diff --git a/changes/fix-issue-96.md b/changes/fix-issue-96.md deleted file mode 100644 index 04f7b1042..000000000 --- a/changes/fix-issue-96.md +++ /dev/null @@ -1 +0,0 @@ -- Fixed Extended Attributes displaying "[object Object]" for complex values like sign in activity — now shows properly formatted JSON diff --git a/changes/fix-mssql-shim-boolean-and-sysutcdatetime.md b/changes/fix-mssql-shim-boolean-and-sysutcdatetime.md deleted file mode 100644 index 5876aaf2f..000000000 --- a/changes/fix-mssql-shim-boolean-and-sysutcdatetime.md +++ /dev/null @@ -1,2 +0,0 @@ -- Automated version bumping on PR merge — `bump-version.yml` Action increments the Minor version in `setup/IdentityAtlas.psd1` and updates the timestamp on every PR merge to `main`. Branches no longer touch the version file, eliminating recurring merge conflicts on `setup/IdentityAtlas.psd1`. -- Automated changelog merging on PR merge — branches now create a fragment file in `changes/` instead of editing `CHANGES.md` directly. The same `bump-version.yml` Action collects all fragments and prepends them to `CHANGES.md` on merge, eliminating recurring merge conflicts on `CHANGES.md`. diff --git a/changes/fix-sync-log-empty-message.md b/changes/fix-sync-log-empty-message.md deleted file mode 100644 index c7d02b083..000000000 --- a/changes/fix-sync-log-empty-message.md +++ /dev/null @@ -1 +0,0 @@ -- Fixed Sync Log empty state message to reference adding a crawler instead of Start-FGSync diff --git a/changes/issue-69-container-stats-historical-graphs.md b/changes/issue-69-container-stats-historical-graphs.md deleted file mode 100644 index 108434527..000000000 --- a/changes/issue-69-container-stats-historical-graphs.md +++ /dev/null @@ -1 +0,0 @@ -- Added historical performance graphs to Containers tab showing last 10 minutes of CPU, memory, and network usage for each container diff --git a/changes/issue-70-json-syntax-highlighting.md b/changes/issue-70-json-syntax-highlighting.md deleted file mode 100644 index 519fdb688..000000000 --- a/changes/issue-70-json-syntax-highlighting.md +++ /dev/null @@ -1 +0,0 @@ -- Added syntax highlighting and collapsible sections to JSON display in risk profile wizard for improved readability diff --git a/changes/quickstart-upgrade-pull-always.md b/changes/quickstart-upgrade-pull-always.md deleted file mode 100644 index 816bb3885..000000000 --- a/changes/quickstart-upgrade-pull-always.md +++ /dev/null @@ -1 +0,0 @@ -- Fixed upgrade instructions to use `--pull always` instead of a separate `pull` + `up -d`, so a single command always fetches the latest image from the registry diff --git a/changes/unified-issue-workflow.md b/changes/unified-issue-workflow.md deleted file mode 100644 index fd55d4783..000000000 --- a/changes/unified-issue-workflow.md +++ /dev/null @@ -1,5 +0,0 @@ -- Merged issue triage and nightly auto-fix into a single unified workflow that classifies and fixes bugs immediately when an issue is opened -- Added automatic issue classification: bugs vs feature requests, with priority labels (critical, high, medium, low) -- Feature requests are now labeled as `enhancement` with a priority but require manual triage before auto-fix -- Added nightly re-evaluation job (23:00 Amsterdam time) that checks issues labeled `needs-clarification` or `cant-autofix` for new comments — if enough detail has been added, the issue is promoted to `ready-to-fix` and auto-fixed -- Fixed auto-fix prompt to use changelog fragments instead of editing CHANGES.md and setup/IdentityAtlas.psd1 directly diff --git a/setup/IdentityAtlas.psd1 b/setup/IdentityAtlas.psd1 index 81ffb32ef..317dc3330 100644 --- a/setup/IdentityAtlas.psd1 +++ b/setup/IdentityAtlas.psd1 @@ -12,7 +12,7 @@ RootModule = '.\IdentityAtlas.psm1' # Version number of this module. -ModuleVersion = '5.0.20260417.1430' +ModuleVersion = '5.1.20260417.1306' # Supported PSEditions # CompatiblePSEditions = @() From 91b6e545f3f2c426c65b52128fb81ab12f3ad124 Mon Sep 17 00:00:00 2001 From: Taeke Date: Fri, 17 Apr 2026 19:22:40 +0200 Subject: [PATCH 007/160] feat: add stable release branch strategy with edge/latest channels - Introduce release/vX.Y branch model: main builds push :edge, release branches push :latest so customers only receive intentional releases - Add cut-release.yml workflow to create release branches from main and publish the initial X.Y.0.0 image automatically - Update bump-version.yml to increment patch (X.Y.P.0) on release branches and minor+timestamp on main - Update docker-publish.yml to push :edge or :latest based on source branch - Add IMAGE_TAG env var to docker-compose.prod.yml for channel selection - Show amber edge badge in UI footer for dev builds - Add PR checks to release/** branch PRs (pr.yml, pr-integration.yml) - Add branch protection ruleset for release/** via setup-branch-protection.sh - Update README and docker-setup.md with .env setup and channel docs Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/bump-version.yml | 69 +++++++++--- .github/workflows/cut-release.yml | 81 ++++++++++++++ .github/workflows/docker-publish.yml | 66 ++++++++++- .github/workflows/pr-integration.yml | 4 +- .github/workflows/pr.yml | 5 +- CLAUDE.md | 121 +++++++++++++++++---- README.md | 16 ++- app/ui/src/App.jsx | 7 +- changes/feature-release-branch-strategy.md | 10 ++ docker-compose.prod.yml | 9 +- docs/architecture/docker-setup.md | 108 ++++++++++++++++-- setup/config/.env.example | 6 + tools/setup-branch-protection.sh | 115 ++++++++++++++++++++ 13 files changed, 554 insertions(+), 63 deletions(-) create mode 100644 .github/workflows/cut-release.yml create mode 100644 changes/feature-release-branch-strategy.md create mode 100644 tools/setup-branch-protection.sh diff --git a/.github/workflows/bump-version.yml b/.github/workflows/bump-version.yml index 2677a4918..2a1e76bbf 100644 --- a/.github/workflows/bump-version.yml +++ b/.github/workflows/bump-version.yml @@ -1,9 +1,10 @@ # ─── Version Bump + Changelog Merge on PR Merge ────────────────────────────── -# On every PR merge to main this workflow: +# On every PR merge this workflow: # 1. Collects all fragment files from changes/*.md and prepends them to # CHANGES.md, then deletes the fragments. -# 2. Increments the Minor version in setup/IdentityAtlas.psd1 and updates -# the timestamp. +# 2. Bumps the version in setup/IdentityAtlas.psd1: +# - main: increments Minor, updates timestamp → Major.Minor.yyyyMMdd.HHmm +# - release/v*: increments Patch → Major.Minor.Patch.0 # # Branches never touch CHANGES.md or the version file directly — each branch # creates a uniquely named fragment file in changes/ instead, so merge @@ -18,7 +19,9 @@ name: Bump version on PR merge on: pull_request: types: [closed] - branches: [main] + branches: + - main + - 'release/**' jobs: bump-version: @@ -28,14 +31,27 @@ jobs: contents: write steps: + - name: Determine target branch + id: branch + run: | + TARGET="${{ github.event.pull_request.base.ref }}" + echo "name=$TARGET" >> "$GITHUB_OUTPUT" + if [[ "$TARGET" == release/* ]]; then + echo "type=release" >> "$GITHUB_OUTPUT" + else + echo "type=main" >> "$GITHUB_OUTPUT" + fi + - uses: actions/checkout@v4 with: - ref: main + ref: ${{ steps.branch.outputs.name }} token: ${{ secrets.VERSION_BUMP_PAT }} - name: Merge changelog fragments and bump version shell: pwsh run: | + $branchType = "${{ steps.branch.outputs.type }}" + # ── 1. Collect changelog fragments from changes/*.md ────────────── $fragments = Get-ChildItem changes/*.md -ErrorAction SilentlyContinue | Sort-Object Name if ($fragments) { @@ -49,20 +65,39 @@ jobs: Write-Host "No changelog fragments found in changes/ -- skipping CHANGES.md update" } - # ── 2. Bump Minor version in setup/IdentityAtlas.psd1 ───────────── + # ── 2. Bump version in setup/IdentityAtlas.psd1 ─────────────────── $content = Get-Content setup/IdentityAtlas.psd1 -Raw - if ($content -match "ModuleVersion\s*=\s*'(\d+)\.(\d+)\.\d+\.\d+'") { - $major = $Matches[1] - $minor = [int]$Matches[2] + 1 - $stamp = (Get-Date -Format 'yyyyMMdd.HHmm') - $newVer = "$major.$minor.$stamp" - $content = $content -replace "ModuleVersion\s*=\s*'\d+\.\d+\.\d+\.\d+'", "ModuleVersion = '$newVer'" - Set-Content setup/IdentityAtlas.psd1 $content -NoNewline - Write-Host "Bumped to $newVer" - "NEW_VERSION=$newVer" | Out-File -Append $env:GITHUB_ENV + + if ($branchType -eq 'release') { + # Release branch: increment Patch → Major.Minor.Patch.0 + if ($content -match "ModuleVersion\s*=\s*'(\d+)\.(\d+)\.(\d+)\.(\d+)'") { + $major = $Matches[1] + $minor = $Matches[2] + $patch = [int]$Matches[3] + 1 + $newVer = "$major.$minor.$patch.0" + $content = $content -replace "ModuleVersion\s*=\s*'\d+\.\d+\.\d+\.\d+'", "ModuleVersion = '$newVer'" + Set-Content setup/IdentityAtlas.psd1 $content -NoNewline + Write-Host "Bumped to $newVer (release patch)" + "NEW_VERSION=$newVer" | Out-File -Append $env:GITHUB_ENV + } else { + Write-Error "Could not find ModuleVersion in setup/IdentityAtlas.psd1" + exit 1 + } } else { - Write-Error "Could not find ModuleVersion in setup/IdentityAtlas.psd1" - exit 1 + # Main branch: increment Minor, update timestamp → Major.Minor.yyyyMMdd.HHmm + if ($content -match "ModuleVersion\s*=\s*'(\d+)\.(\d+)\.\d+\.\d+'") { + $major = $Matches[1] + $minor = [int]$Matches[2] + 1 + $stamp = (Get-Date -Format 'yyyyMMdd.HHmm') + $newVer = "$major.$minor.$stamp" + $content = $content -replace "ModuleVersion\s*=\s*'\d+\.\d+\.\d+\.\d+'", "ModuleVersion = '$newVer'" + Set-Content setup/IdentityAtlas.psd1 $content -NoNewline + Write-Host "Bumped to $newVer (main dev build)" + "NEW_VERSION=$newVer" | Out-File -Append $env:GITHUB_ENV + } else { + Write-Error "Could not find ModuleVersion in setup/IdentityAtlas.psd1" + exit 1 + } } - name: Commit and push diff --git a/.github/workflows/cut-release.yml b/.github/workflows/cut-release.yml new file mode 100644 index 000000000..a519de22c --- /dev/null +++ b/.github/workflows/cut-release.yml @@ -0,0 +1,81 @@ +# ─── Cut a Release Branch ───────────────────────────────────────────────────── +# Manually triggered. Creates release/vX.Y from main, sets the version to +# X.Y.0.0, then triggers docker-publish to build and push the initial +# :latest image for that version. +# +# Usage: +# 1. Go to Actions → Cut Release Branch → Run workflow +# 2. Enter the version (e.g. "5.2" — major.minor only, no patch) +# 3. The workflow creates release/v5.2, sets ModuleVersion = 5.2.0.0, +# and publishes ghcr.io/fortigi/identity-atlas:latest + :5.2.0.0 +# 4. Bugfixes targeting that release are PRed into release/v5.2 +# 5. Each merge bumps patch and republishes: 5.2.0.0 → 5.2.1.0 → 5.2.2.0 ... +# ───────────────────────────────────────────────────────────────────────────── + +name: Cut Release Branch + +on: + workflow_dispatch: + inputs: + version: + description: 'Release version (major.minor only, e.g. "5.2")' + required: true + +jobs: + cut-release: + runs-on: ubuntu-latest + permissions: + contents: write + actions: write + + steps: + - name: Validate version input + run: | + if ! echo "${{ github.event.inputs.version }}" | grep -qE '^\d+\.\d+$'; then + echo "::error::Version must be in Major.Minor format (e.g. 5.2)" + exit 1 + fi + + - uses: actions/checkout@v4 + with: + ref: main + token: ${{ secrets.VERSION_BUMP_PAT }} + fetch-depth: 0 + + - name: Create release branch and set version + shell: pwsh + run: | + $version = "${{ github.event.inputs.version }}" + $branch = "release/v$version" + $newVer = "$version.0.0" + + # Create and push the branch + git checkout -b $branch + Write-Host "Created branch $branch" + + # Set version to Major.Minor.0.0 + $content = Get-Content setup/IdentityAtlas.psd1 -Raw + $content = $content -replace "ModuleVersion\s*=\s*'\d+\.\d+\.\d+\.\d+'", "ModuleVersion = '$newVer'" + Set-Content setup/IdentityAtlas.psd1 $content -NoNewline + Write-Host "Set ModuleVersion to $newVer" + + "NEW_VERSION=$newVer" | Out-File -Append $env:GITHUB_ENV + "BRANCH=$branch" | Out-File -Append $env:GITHUB_ENV + + - name: Commit and push + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add setup/IdentityAtlas.psd1 + git commit -m "chore: cut release branch, set version to ${NEW_VERSION}" + git push origin "${BRANCH}" + echo "✅ Release branch ${BRANCH} created at version ${NEW_VERSION}" + + - name: Trigger docker-publish for initial release image + run: | + gh workflow run docker-publish.yml \ + --ref main \ + --field branch="${BRANCH}" + echo "✅ docker-publish triggered for ${BRANCH} (will publish :latest + :${NEW_VERSION})" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 9e6730aee..25aecadd2 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -1,6 +1,6 @@ # ─── Docker Image Publishing ───────────────────────────────────────────────── # Builds container images, runs a basic smoke test, then pushes to GitHub -# Container Registry. A broken image never reaches :latest. +# Container Registry. A broken image never reaches :latest or :edge. # # Flow: Build (local) → Start stack → Smoke test → Push # @@ -8,7 +8,9 @@ # ghcr.io/fortigi/identity-atlas — Node.js API + React frontend (web service) # ghcr.io/fortigi/identity-atlas-worker — PowerShell worker (crawlers, risk scoring) # -# Tags: latest + version from IdentityAtlas.psd1 (e.g., 5.0.20260413.1530) +# Tags pushed depend on the source branch: +# main: edge + Major.Minor.yyyyMMdd.HHmm (dev builds, not :latest) +# release/v*: latest + Major.Minor.Patch.0 (stable customer releases) # ───────────────────────────────────────────────────────────────────────────── name: Publish Docker Images @@ -17,8 +19,15 @@ on: workflow_run: workflows: ["Bump version on PR merge"] types: [completed] - branches: [main] + branches: + - main + - 'release/**' workflow_dispatch: + inputs: + branch: + description: 'Branch to build from (main or release/vX.Y)' + required: true + default: 'main' env: REGISTRY: ghcr.io @@ -35,9 +44,25 @@ jobs: packages: write steps: + - name: Determine source branch and image tags + id: config + run: | + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + BRANCH="${{ github.event.inputs.branch }}" + else + BRANCH="${{ github.event.workflow_run.head_branch }}" + fi + echo "branch=$BRANCH" >> "$GITHUB_OUTPUT" + + if [[ "$BRANCH" == release/* ]]; then + echo "channel=release" >> "$GITHUB_OUTPUT" + else + echo "channel=main" >> "$GITHUB_OUTPUT" + fi + - uses: actions/checkout@v4 with: - ref: main + ref: ${{ steps.config.outputs.branch }} - name: Extract version from manifest id: version @@ -138,7 +163,8 @@ jobs: run: docker compose -f docker-compose.qa.yml down -v 2>/dev/null || true # ── Push images (only reached if smoke test passed) ───────────── - - name: Push web image + - name: Push web image (release → latest) + if: steps.config.outputs.channel == 'release' uses: docker/build-push-action@v6 with: context: ./app @@ -151,7 +177,8 @@ jobs: cache-from: type=gha,scope=web cache-to: type=gha,mode=max,scope=web - - name: Push worker image + - name: Push worker image (release → latest) + if: steps.config.outputs.channel == 'release' uses: docker/build-push-action@v6 with: context: . @@ -162,3 +189,30 @@ jobs: ${{ env.REGISTRY }}/fortigi/identity-atlas-worker:${{ steps.version.outputs.version }} cache-from: type=gha,scope=worker cache-to: type=gha,mode=max,scope=worker + + - name: Push web image (main → edge) + if: steps.config.outputs.channel == 'main' + uses: docker/build-push-action@v6 + with: + context: ./app + file: ./app/api/Dockerfile + push: true + tags: | + ${{ env.REGISTRY }}/fortigi/identity-atlas:edge + ${{ env.REGISTRY }}/fortigi/identity-atlas:${{ steps.version.outputs.version }} + build-args: MODULE_VERSION=${{ steps.version.outputs.version }} + cache-from: type=gha,scope=web + cache-to: type=gha,mode=max,scope=web + + - name: Push worker image (main → edge) + if: steps.config.outputs.channel == 'main' + uses: docker/build-push-action@v6 + with: + context: . + file: ./setup/docker/Dockerfile.powershell + push: true + tags: | + ${{ env.REGISTRY }}/fortigi/identity-atlas-worker:edge + ${{ env.REGISTRY }}/fortigi/identity-atlas-worker:${{ steps.version.outputs.version }} + cache-from: type=gha,scope=worker + cache-to: type=gha,mode=max,scope=worker diff --git a/.github/workflows/pr-integration.yml b/.github/workflows/pr-integration.yml index 69d13c10f..f4e8aaa67 100644 --- a/.github/workflows/pr-integration.yml +++ b/.github/workflows/pr-integration.yml @@ -30,7 +30,9 @@ name: PR Integration Tests on: pull_request: - branches: [main] + branches: + - main + - 'release/**' env: DOTNET_NOLOGO: true diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index ab3804271..beb469374 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -16,7 +16,10 @@ name: PR Checks on: pull_request: - branches: [main, dev] + branches: + - main + - dev + - 'release/**' env: DOTNET_NOLOGO: true diff --git a/CLAUDE.md b/CLAUDE.md index 71d014599..1ea512cb2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -31,32 +31,49 @@ Identity Atlas is a Docker-deployed application that pulls authorization data fr | Branch | Purpose | PR required? | Approval required? | |--------|---------|-------------|-------------------| -| `main` | Stable trunk. Never commit directly. | Yes | Yes (at least 1) | +| `main` | Integration trunk. Never commit directly. Merges push `:edge` Docker tag. | Yes | Yes (at least 1) | +| `release/vX.Y` | Stable customer release line. Cut from `main` via `cut-release.yml`. Merges push `:latest` Docker tag. | Yes | No | | `feature/` | All feature work. Created from `main`. Merged back to `main` via PR. | Yes (to `main`) | No | -| `bugfixes/` | Bug fixes. Created from `main`. Merged back to `main` via PR. | Yes (to `main`) | No | +| `bugfixes/` | Bug fixes. Branch from `release/vX.Y` for production hotfixes, or from `main` for pre-release fixes. | Yes | No | **Rules:** -- `feature/` and `bugfixes/` branches must be branched off `main`. -- All merges to `main` go through a Pull Request — no direct pushes ever. +- `feature/` branches must be branched off `main`. +- `bugfixes/` branches branch from **`release/vX.Y`** when fixing a production issue (customers are affected), or from `main` when fixing something not yet released. +- Production bugfixes merged to `release/vX.Y` must also be cherry-picked to `main` so the fix is included in future feature releases. +- All merges go through a Pull Request — no direct pushes to `main` or `release/*` ever. - Branch names: `feature/` or `bugfixes/` (lowercase, hyphens). Example: `feature/risk-score-export`, `bugfixes/fix-login-redirect`. -- When starting work, always create a new `feature/` or `bugfixes/` branch. Never work directly on `main`. +- When starting work, always create a new branch. Never work directly on `main` or `release/*`. - **One issue per branch.** Each branch must fix exactly one issue or implement exactly one feature. Never combine fixes for separate, unrelated issues into a single branch or PR. Exception: if a single code change genuinely resolves more than one issue (e.g. the same root cause), both issue numbers may be referenced in the commit and PR — but this should be rare and the connection must be explicit. ### Version Number Scheme -Version format (4 parts, PowerShell-compatible): `Major.Minor.yyyyMMdd.HHmm` +Two formats, both 4-part (PowerShell-compatible): -| Branch | Version format | Who updates it | When | -|--------|---------------|----------------|------| -| `main` | `Major.Minor.yyyyMMdd.HHmm` | `bump-version.yml` GitHub Action (automated) | On every PR merge. Increments `Minor`, updates timestamp. | -| `feature/*` / `bugfixes/*` | — | **Nobody** | Never touch `setup/IdentityAtlas.psd1` on a branch. | +| Branch | Version format | Example | Docker tag pushed | +|--------|---------------|---------|-------------------| +| `main` | `Major.Minor.yyyyMMdd.HHmm` | `5.3.20260419.1430` | `:edge` | +| `release/vX.Y` | `Major.Minor.Patch.0` | `5.2.1.0` | `:latest` | +| `feature/*` / `bugfixes/*` | — | — | Nobody | + +The timestamp format on `main` makes dev builds instantly recognisable. The semantic `Patch.0` format on release branches gives customers a clear upgrade path. + +**Who updates versions:** + +| Branch | Who updates it | When | +|--------|---------------|------| +| `main` | `bump-version.yml` (automated) | Every PR merge — increments `Minor`, updates timestamp | +| `release/vX.Y` | `bump-version.yml` (automated) | Every PR merge — increments `Patch` (e.g. `5.2.0.0` → `5.2.1.0`) | +| `feature/*` / `bugfixes/*` | **Nobody** | Never touch `setup/IdentityAtlas.psd1` on a branch | **How to apply:** -1. **Starting a branch**: Branch from `main`. Leave `setup/IdentityAtlas.psd1` untouched. -2. **After any code change on a branch**: Add bullets to `changes/.md`. Do not edit `CHANGES.md` or `ModuleVersion`. -3. **When merging feature/bugfixes → main via PR**: The `bump-version.yml` Action runs automatically after merge and commits the Minor increment + new timestamp to `main`. The `docker-publish.yml` Action then triggers on that commit to build and push Docker images with the updated version. -4. **Major version bump**: Edit `setup/IdentityAtlas.psd1` manually on `main` (via a PR) for a breaking change. Increment `Major`, reset `Minor` to `0`. +1. **Starting a feature or pre-release bugfix branch**: Branch from `main`. Leave `setup/IdentityAtlas.psd1` untouched. +2. **Starting a production hotfix branch**: Branch from `release/vX.Y`. Leave `setup/IdentityAtlas.psd1` untouched. +3. **After any code change on a branch**: Add bullets to `changes/.md`. Do not edit `CHANGES.md` or `ModuleVersion`. +4. **When merging → main via PR**: `bump-version.yml` increments Minor + timestamp. `docker-publish.yml` builds and pushes `:edge` + versioned tag. +5. **When merging → release/vX.Y via PR**: `bump-version.yml` increments Patch. `docker-publish.yml` builds and pushes `:latest` + versioned tag. +6. **Cutting a new release**: Run the `cut-release.yml` workflow (Actions → Cut Release Branch → enter `Major.Minor`). It creates `release/vX.Y` from `main` and sets the version to `X.Y.0.0`. +7. **Major version bump**: Edit `setup/IdentityAtlas.psd1` manually on `main` (via a PR) for a breaking change. Increment `Major`, reset `Minor` to `0`. ### Changelog fragments (replaces direct CHANGES.md edits) @@ -723,17 +740,72 @@ The Crawlers wizard validates these permissions on the App Registration during s - `vw_PendingRequestTimeline` - Aging pending requests - `vw_RequestResponseMetrics` - Aggregate approval statistics +## Repository Setup (One-Time) + +These steps are required once when creating or transferring the repository. They are not automated by CI. + +### GitHub Actions secrets + +| Secret | Required scopes | Purpose | +|--------|----------------|---------| +| `VERSION_BUMP_PAT` | `repo` (includes `contents:write`) | Lets `bump-version.yml` and `cut-release.yml` push commits directly to protected branches (`main`, `release/**`). The PAT owner **must have the admin role** on the repository so the bypass actor rule on `release/**` applies. | + +### Branch protection + +Run once after repo creation (requires `gh` CLI authenticated as admin): + +```bash +bash tools/setup-branch-protection.sh Fortigi/IdentityAtlas +``` + +This sets: +- `main` — PR required (1 approval), `PR Summary` check required, admins bypass +- `release/**` — PR required (0 approvals), `PR Summary` check required, no force-push, no deletion, admins bypass + +--- + ## Development Workflow ### Starting New Work +**Feature (not yet released):** ```bash git checkout main && git pull git checkout -b feature/ # e.g. feature/risk-score-export -# or +``` + +**Pre-release bugfix (bug is in main, not yet in a release):** +```bash +git checkout main && git pull git checkout -b bugfixes/ # e.g. bugfixes/fix-login-redirect ``` +**Production hotfix (bug is in a released version, customers are affected):** +```bash +# Step 1 — fix on the release branch +git checkout release/v5.2 && git pull +git checkout -b bugfixes/ +# ... make the fix, add changes/.md fragment, commit ... +gh pr create --base release/v5.2 --title "fix: ..." +# merge the PR → bump-version bumps patch, docker-publish pushes :latest + +# Step 2 — bring the fix into main via its own PR (main is protected, no direct commits) +git checkout main && git pull +git checkout -b bugfixes/-main +git cherry-pick # the fix commit only, not the version bump commit +gh pr create --base main --title "fix: ... (cherry-pick from release/v5.2)" +# merge the PR → bump-version bumps minor on main as normal +``` + +### Cutting a New Release + +When `main` is stable and ready to ship to customers: + +1. Go to **Actions → Cut Release Branch → Run workflow** +2. Enter the version, e.g. `5.3` (Major.Minor only) +3. The workflow creates `release/v5.3` from `main` and sets version to `5.3.0.0` +4. Merges to `release/v5.3` push `:latest` to customers + ### Making Changes 1. **Create/Edit** the relevant files @@ -764,17 +836,26 @@ gh pr create --base feature/foo-step-1 --title "step 2: ..." When a bottom PR merges, retarget the next one: `gh pr edit --base main`. -### Merging Feature/Bugfixes → Main (via PR) +### Merging to Main (feature / pre-release bugfix) -1. Open PR from `feature/` or `bugfixes/` into `main` (or into the previous stack branch) +1. Open PR from `feature/` or `bugfixes/` into `main` 2. Use the fragment content from `changes/.md` as the PR description 3. Requires 1 approval — merge when CI passes +4. After merge: `bump-version.yml` increments Minor + timestamp; `docker-publish.yml` pushes `:edge` -### Version Updates +### Merging to a Release Branch (production hotfix) -Version format: `Major.Minor.yyyyMMdd.HHmm` (e.g., `2.5.20260317.1430`) +1. Open PR from `bugfixes/` into `release/vX.Y` +2. Use the fragment content from `changes/.md` as the PR description +3. Merge when CI passes +4. After merge: `bump-version.yml` increments Patch; `docker-publish.yml` pushes `:latest` +5. Cherry-pick the fix to `main`: `git checkout main && git cherry-pick ` + +### Version Updates -See the **Branching & Versioning Strategy** section above for the full scheme. The Docker images are built and pushed by the `docker-publish.yml` GitHub Action on merge to `main`, tagged with both `latest` and the module version. +See the **Branching & Versioning Strategy** section above for the full scheme. +- `main` merges → `Major.Minor.yyyyMMdd.HHmm` → `:edge` Docker tag +- `release/*` merges → `Major.Minor.Patch.0` → `:latest` Docker tag ## User Workflow (Getting Started) diff --git a/README.md b/README.md index e3bbc757b..d599bc6c8 100644 --- a/README.md +++ b/README.md @@ -9,19 +9,29 @@ Permissions are scattered across identity systems, directories, and SaaS platfor **Prerequisites:** Docker and Docker Compose. ```bash -# 1. Download the production compose file +# 1. Download the compose file and environment template curl -O https://raw.githubusercontent.com/Fortigi/IdentityAtlas/main/docker-compose.prod.yml +curl -O https://raw.githubusercontent.com/Fortigi/IdentityAtlas/main/setup/config/.env.example -# 2. Start the stack +# 2. Create your .env file +cp .env.example .env +# For a quick local evaluation the defaults are fine. +# For any networked or production deployment, open .env and set: +# POSTGRES_PASSWORD= +# IDENTITY_ATLAS_MASTER_KEY= + +# 3. Start the stack (first run: ~2 min to pull images) docker compose -f docker-compose.prod.yml up -d -# 3. Open http://localhost:3001 +# 4. Open http://localhost:3001 # Go to Admin > Crawlers, then click "Load Demo Data" to explore with sample data, or # click "Add Crawler" to connect your Entra ID tenant. ``` The in-browser crawler wizard walks you through credentials, permission validation, object type selection, and scheduling — no PowerShell or command-line setup required. +> **Image channels:** The default pulls the latest stable release (`:latest`). To run the development build instead, set `IMAGE_TAG=edge` in your `.env`. See [Docker Setup](docs/architecture/docker-setup.md) for details. + --- ## What Identity Atlas Does diff --git a/app/ui/src/App.jsx b/app/ui/src/App.jsx index d4b95d59f..2e0411400 100644 --- a/app/ui/src/App.jsx +++ b/app/ui/src/App.jsx @@ -529,8 +529,13 @@ export default function App() { {/* Footer */} -
+
Identity Atlas{moduleVersion ? ` v${moduleVersion}` : ''} + {/^\d+\.\d+\.\d{8}\.\d{4}$/.test(moduleVersion) && ( + + edge + + )}
diff --git a/changes/feature-release-branch-strategy.md b/changes/feature-release-branch-strategy.md new file mode 100644 index 000000000..eaa57b4cc --- /dev/null +++ b/changes/feature-release-branch-strategy.md @@ -0,0 +1,10 @@ +- Added stable release branch model: `release/vX.Y` branches are cut from `main` and receive only bugfixes, giving customers a stable `:latest` Docker image that does not change when new features land on `main` +- New `cut-release.yml` workflow creates a release branch and sets the initial version (e.g. `5.2.0.0`) with one click from GitHub Actions +- Main branch builds now push the `:edge` Docker tag instead of `:latest`, so customers on `:latest` only receive intentional releases +- Production hotfixes merged to a release branch increment the patch version (`5.2.0.0` → `5.2.1.0`) and push `:latest` automatically +- `docker-compose.prod.yml` now supports an `IMAGE_TAG` environment variable to select the image channel (`latest`, `edge`, or a pinned version); customers get `latest` by default +- The footer now shows the running version; edge builds display a prominent amber "edge" badge so it is immediately obvious which channel is running +- README and Docker Setup documentation updated with step-by-step `.env` setup for both customers and developers, image channel selection guide, and a full environment variable reference +- Fixed PR checks (lint, unit tests, integration tests) to also run on pull requests targeting `release/**` branches, so hotfixes receive the same gate as feature PRs +- Fixed release cut workflow: the initial `X.Y.0.0` image is now published to `:latest` immediately after the release branch is created, not only after the first bugfix merges +- `release/**` branches are now protected: direct commits are blocked, a pull request is required, and the `PR Summary` status check must pass before merging; repository admins retain bypass rights so the automated version bump can still land diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 679f3f823..1b60721db 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -12,6 +12,11 @@ # Copy .env.example to .env and set your values, or pass them inline: # GRAPH_TENANT_ID=... GRAPH_CLIENT_ID=... GRAPH_CLIENT_SECRET=... docker compose -f docker-compose.prod.yml up # +# Image channel (IMAGE_TAG): +# latest — stable customer release (default, omit IMAGE_TAG or leave blank) +# edge — latest merged commit on main; may be unstable (set IMAGE_TAG=edge) +# 5.2.1.0 — pin to a specific release version +# # SECURITY: The default POSTGRES_PASSWORD is for local evaluation only. # For any networked or production deployment, set a strong password and an # explicit master key for the secrets vault: @@ -43,7 +48,7 @@ services: # ── Web (serves built frontend + API) ──────────────────────────────────────── web: - image: ghcr.io/fortigi/identity-atlas:latest + image: ghcr.io/fortigi/identity-atlas:${IMAGE_TAG:-latest} # The container runs as the unprivileged `node` user. The Docker socket # is owned by root:root 0660, so node needs GID 0 to read container stats. group_add: ["0"] @@ -81,7 +86,7 @@ services: # In v5 the worker container has NO database driver. It talks to the API # for everything (job pickup, ingest, progress reporting). worker: - image: ghcr.io/fortigi/identity-atlas-worker:latest + image: ghcr.io/fortigi/identity-atlas-worker:${IMAGE_TAG:-latest} environment: WEB_API_URL: "http://web:3001/api" # Optional global crawler API key — overridden per-job via WorkerConfig. diff --git a/docs/architecture/docker-setup.md b/docs/architecture/docker-setup.md index 5bcb2921e..77b2d1c13 100644 --- a/docs/architecture/docker-setup.md +++ b/docs/architecture/docker-setup.md @@ -9,13 +9,17 @@ Running Identity Atlas locally with Docker — three containers providing the fu The fastest way to try Identity Atlas — pulls pre-built images, no source code needed: ```bash -# Download the production compose file +# 1. Download the compose file and environment template curl -O https://raw.githubusercontent.com/Fortigi/IdentityAtlas/main/docker-compose.prod.yml +curl -O https://raw.githubusercontent.com/Fortigi/IdentityAtlas/main/setup/config/.env.example -# Start everything (first run: ~2 min to pull images) +# 2. Create your .env file +cp .env.example .env + +# 3. Start everything (first run: ~2 min to pull images) docker compose -f docker-compose.prod.yml up -d -# Open the UI +# 4. Open the UI open http://localhost:3001 ``` @@ -23,6 +27,38 @@ On first visit, the UI opens to the Dashboard. If no data is loaded yet, click * To connect your own Entra ID tenant, click **"Connect Entra ID"** on the Crawlers page and enter your App Registration credentials (Tenant ID, Client ID, Client Secret). +### The .env File + +`docker-compose.prod.yml` reads all configuration from a `.env` file in the same directory. The template has safe defaults for local evaluation — for anything networked or production, set these two variables: + +| Variable | Default | What to do | +|---|---|---| +| `POSTGRES_PASSWORD` | `identity_atlas_local` | **Change this** for any non-local deployment | +| `IDENTITY_ATLAS_MASTER_KEY` | *(auto-generated)* | Set an explicit value so you can back it up; if left blank the container generates one and saves it to the `job_data` volume | + +Full variable reference: [Environment Variables](#environment-variables). + +### Image Channels + +The compose file uses the `IMAGE_TAG` variable to select which build to pull: + +| `IMAGE_TAG` | What you get | Who should use it | +|---|---|---| +| *(unset or blank)* | `:latest` — last stable release | Customers and production deployments | +| `edge` | `:edge` — latest commit on `main`, may be unstable | Developers and testers who want the newest features | +| `5.2.1.0` | Exact pinned version, never auto-updates | Customers who want to control upgrade timing | + +The running version is always visible in the footer of the UI. Edge builds show an amber **edge** badge so it is immediately obvious which channel is running. + +```bash +# Run the stable release (default) +docker compose -f docker-compose.prod.yml up -d + +# Run the edge build +IMAGE_TAG=edge docker compose -f docker-compose.prod.yml up -d +# or set IMAGE_TAG=edge in your .env +``` + --- ## Developer Setup (From Source) @@ -69,6 +105,11 @@ happens inside the web container at startup via the migrations runner ```powershell cd c:\Source\GitHub\IdentityAtlas +# Create your .env file from the template +cp setup/config/.env.example .env +# IMAGE_TAG is ignored by the dev compose (it builds from source). +# You can leave the other defaults as-is for local development. + # Start the stack (first time takes ~3 min to build) docker compose up -d --build @@ -83,6 +124,8 @@ Start-Process http://localhost:3001 Start-Process http://localhost:3001/api/docs ``` +> **Note:** `docker-compose.yml` (dev) builds images from source — `IMAGE_TAG` has no effect. Use `docker-compose.prod.yml` with `IMAGE_TAG=edge` if you want to run the pre-built edge image without a local build. + ## Stopping ```powershell @@ -209,19 +252,60 @@ docker compose -f docker-compose.yml restart worker ### Environment Variables -Create `.env` from the template for secrets: +Copy the template once, then edit the values you need: -```powershell +```bash cp setup/config/.env.example .env -# Edit .env with your values ``` -| Variable | Purpose | -|---|---| -| `POSTGRES_PASSWORD` | PostgreSQL password. Both `docker-compose.yml` and `docker-compose.prod.yml` use PostgreSQL — SQL Server was dropped in v5. | -| `CRAWLER_API_KEY` | API key for the worker's crawler | -| `GRAPH_TENANT_ID` / `CLIENT_ID` / `CLIENT_SECRET` | For EntraID crawler | -| `LLM_PROVIDER` / `LLM_API_KEY` | For risk scoring (Anthropic or OpenAI) | +Both compose files (`docker-compose.yml` and `docker-compose.prod.yml`) read from `.env` in the project root. + +#### Image channel (`docker-compose.prod.yml` only) + +| Variable | Default | Description | +|---|---|---| +| `IMAGE_TAG` | *(blank → `latest`)* | Docker image tag to pull. Leave blank for the stable release, set `edge` for the latest dev build, or pin to a specific version like `5.2.1.0`. | + +#### Database + +| Variable | Default | Description | +|---|---|---| +| `POSTGRES_PASSWORD` | `identity_atlas_local` | PostgreSQL password. Safe for local evaluation; **change for any networked deployment**. | +| `POSTGRES_USER` | `identity_atlas` | PostgreSQL username. Rarely needs changing. | +| `POSTGRES_DB` | `identity_atlas` | Database name. Rarely needs changing. | + +#### Security + +| Variable | Default | Description | +|---|---|---| +| `IDENTITY_ATLAS_MASTER_KEY` | *(auto-generated)* | Master key for the AES-256-GCM secrets vault (LLM API keys, scraper credentials). If left blank, the container generates a key on first start and persists it to the `job_data` volume. **Set an explicit value for production** so the key can be backed up alongside other root secrets. | + +#### Authentication (optional) + +Identity Atlas defaults to no-auth (any browser can access the UI). To require Entra ID login: + +| Variable | Default | Description | +|---|---|---| +| `AUTH_ENABLED` | `false` | Set to `true` to require Entra ID authentication. | +| `AUTH_TENANT_ID` | — | Your Entra ID tenant ID. | +| `AUTH_CLIENT_ID` | — | App Registration client ID for the UI. | +| `AUTH_REQUIRED_ROLES` | — | Optional comma-separated list of app roles required to access the UI. | + +#### Crawler credentials (optional — can also configure via the in-browser wizard) + +| Variable | Default | Description | +|---|---|---| +| `CRAWLER_API_KEY` | *(auto-generated)* | API key the worker uses to authenticate with the API. Auto-generated on first start; override only if you need a fixed key. | +| `GRAPH_TENANT_ID` | — | Entra ID tenant ID for the Graph API crawler. | +| `GRAPH_CLIENT_ID` | — | App Registration client ID. | +| `GRAPH_CLIENT_SECRET` | — | App Registration client secret. | + +#### LLM / Risk Scoring (optional) + +| Variable | Default | Description | +|---|---|---| +| `LLM_PROVIDER` | — | `Anthropic`, `OpenAI`, or `AzureOpenAI`. Can also be configured per-tenant via Admin → LLM Settings. | +| `LLM_API_KEY` | — | API key for the selected LLM provider. | --- diff --git a/setup/config/.env.example b/setup/config/.env.example index 032f03333..7ae6dccd5 100644 --- a/setup/config/.env.example +++ b/setup/config/.env.example @@ -1,6 +1,12 @@ # Identity Atlas Docker — Local Environment Variables (v5) # Copy to .env and fill in your values. Never commit .env to git. +# Image channel — which Docker tag to pull (docker-compose.prod.yml only) +# latest — stable customer release (default, leave blank) +# edge — latest commit on main, may be unstable +# 5.2.1.0 — pin to a specific version +IMAGE_TAG= + # PostgreSQL (defaults match docker-compose.yml) POSTGRES_DB=identity_atlas POSTGRES_USER=identity_atlas diff --git a/tools/setup-branch-protection.sh b/tools/setup-branch-protection.sh new file mode 100644 index 000000000..417562e44 --- /dev/null +++ b/tools/setup-branch-protection.sh @@ -0,0 +1,115 @@ +#!/usr/bin/env bash +# ─── Branch Protection Setup ───────────────────────────────────────────────── +# Run once after creating or transferring the repository. +# Requires: gh CLI authenticated as a repository admin. +# +# What this configures: +# +# main (classic branch protection — already set, included here for docs) +# - Require PR with 1 approval before merging +# - Require "PR Summary" status check +# - Dismiss stale reviews on push +# - enforce_admins: false ← lets VERSION_BUMP_PAT push the version bump commit +# +# release/** (GitHub Ruleset — wildcard patterns need Rulesets API) +# - Require PR before merging (0 approvals needed) +# - Require "PR Summary" status check +# - Block direct pushes (non-fast-forward / force push) +# - Block branch deletion +# - Bypass: repository admins (actor_id=5) ← VERSION_BUMP_PAT owner must be admin +# ───────────────────────────────────────────────────────────────────────────── + +set -euo pipefail + +REPO="${1:-Fortigi/IdentityAtlas}" +echo "Configuring branch protection for: $REPO" + +# ── 1. main — classic branch protection ───────────────────────────────────── +echo "" +echo "Setting classic branch protection on main..." +gh api "repos/$REPO/branches/main/protection" \ + --method PUT \ + --input - <<'JSON' +{ + "required_status_checks": { + "strict": true, + "contexts": ["PR Summary"] + }, + "enforce_admins": false, + "required_pull_request_reviews": { + "dismiss_stale_reviews": true, + "require_code_owner_reviews": true, + "required_approving_review_count": 1, + "require_last_push_approval": false + }, + "restrictions": null, + "allow_force_pushes": false, + "allow_deletions": false, + "required_conversation_resolution": false +} +JSON +echo "✅ main branch protection set" + +# ── 2. release/** — GitHub Ruleset ────────────────────────────────────────── +echo "" +echo "Creating ruleset for release/** branches..." + +# Delete existing ruleset with the same name if it exists +EXISTING_ID=$(gh api "repos/$REPO/rulesets" | \ + python3 -c "import sys,json; rs=[r['id'] for r in json.load(sys.stdin) if r['name']=='Protect release branches']; print(rs[0] if rs else '')" 2>/dev/null || true) + +if [ -n "$EXISTING_ID" ]; then + echo " Removing existing ruleset (id=$EXISTING_ID)..." + gh api "repos/$REPO/rulesets/$EXISTING_ID" --method DELETE +fi + +gh api "repos/$REPO/rulesets" --method POST --input - <<'JSON' +{ + "name": "Protect release branches", + "target": "branch", + "enforcement": "active", + "conditions": { + "ref_name": { + "include": ["refs/heads/release/**"], + "exclude": [] + } + }, + "bypass_actors": [ + { + "actor_id": 5, + "actor_type": "RepositoryRole", + "bypass_mode": "always" + } + ], + "rules": [ + { "type": "deletion" }, + { "type": "non_fast_forward" }, + { + "type": "pull_request", + "parameters": { + "required_approving_review_count": 0, + "dismiss_stale_reviews_on_push": false, + "require_code_owner_review": false, + "require_last_push_approval": false, + "required_review_thread_resolution": false + } + }, + { + "type": "required_status_checks", + "parameters": { + "strict_required_status_checks_policy": false, + "required_status_checks": [ + { "context": "PR Summary" } + ] + } + } + ] +} +JSON +echo "✅ release/** ruleset created" + +echo "" +echo "Done. Branch protection summary:" +echo " main → PR required (1 approval) + PR Summary check" +echo " release/** → PR required (0 approvals) + PR Summary check + no force-push + no deletion" +echo " Bypass → Repository admins (the VERSION_BUMP_PAT owner must have admin role)" From 7ff0c84a6023671d91d5838ebf20c3ac3a0b57bd Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 18 Apr 2026 06:20:07 +0000 Subject: [PATCH 008/160] chore: bump version to 5.2.20260418.0620 --- CHANGES.md | 13 +++++++++++++ changes/feature-release-branch-strategy.md | 10 ---------- setup/IdentityAtlas.psd1 | 2 +- 3 files changed, 14 insertions(+), 11 deletions(-) delete mode 100644 changes/feature-release-branch-strategy.md diff --git a/CHANGES.md b/CHANGES.md index 87a424722..a5bc96b8d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,18 @@ ## Changes in this PR +- Added stable release branch model: `release/vX.Y` branches are cut from `main` and receive only bugfixes, giving customers a stable `:latest` Docker image that does not change when new features land on `main` +- New `cut-release.yml` workflow creates a release branch and sets the initial version (e.g. `5.2.0.0`) with one click from GitHub Actions +- Main branch builds now push the `:edge` Docker tag instead of `:latest`, so customers on `:latest` only receive intentional releases +- Production hotfixes merged to a release branch increment the patch version (`5.2.0.0` → `5.2.1.0`) and push `:latest` automatically +- `docker-compose.prod.yml` now supports an `IMAGE_TAG` environment variable to select the image channel (`latest`, `edge`, or a pinned version); customers get `latest` by default +- The footer now shows the running version; edge builds display a prominent amber "edge" badge so it is immediately obvious which channel is running +- README and Docker Setup documentation updated with step-by-step `.env` setup for both customers and developers, image channel selection guide, and a full environment variable reference +- Fixed PR checks (lint, unit tests, integration tests) to also run on pull requests targeting `release/**` branches, so hotfixes receive the same gate as feature PRs +- Fixed release cut workflow: the initial `X.Y.0.0` image is now published to `:latest` immediately after the release branch is created, not only after the first bugfix merges +- `release/**` branches are now protected: direct commits are blocked, a pull request is required, and the `PR Summary` status check must pass before merging; repository admins retain bypass rights so the automated version bump can still land + +## Changes in this PR + - Fixed Risk Scoring card on Dashboard to link directly to Admin → Risk Scoring subtab - Added Software Bill of Materials (SBOM) documentation listing all components, dependencies, and infrastructure elements - Fixed SBOM navigation entry in mkdocs.yml to ensure proper documentation site build diff --git a/changes/feature-release-branch-strategy.md b/changes/feature-release-branch-strategy.md deleted file mode 100644 index eaa57b4cc..000000000 --- a/changes/feature-release-branch-strategy.md +++ /dev/null @@ -1,10 +0,0 @@ -- Added stable release branch model: `release/vX.Y` branches are cut from `main` and receive only bugfixes, giving customers a stable `:latest` Docker image that does not change when new features land on `main` -- New `cut-release.yml` workflow creates a release branch and sets the initial version (e.g. `5.2.0.0`) with one click from GitHub Actions -- Main branch builds now push the `:edge` Docker tag instead of `:latest`, so customers on `:latest` only receive intentional releases -- Production hotfixes merged to a release branch increment the patch version (`5.2.0.0` → `5.2.1.0`) and push `:latest` automatically -- `docker-compose.prod.yml` now supports an `IMAGE_TAG` environment variable to select the image channel (`latest`, `edge`, or a pinned version); customers get `latest` by default -- The footer now shows the running version; edge builds display a prominent amber "edge" badge so it is immediately obvious which channel is running -- README and Docker Setup documentation updated with step-by-step `.env` setup for both customers and developers, image channel selection guide, and a full environment variable reference -- Fixed PR checks (lint, unit tests, integration tests) to also run on pull requests targeting `release/**` branches, so hotfixes receive the same gate as feature PRs -- Fixed release cut workflow: the initial `X.Y.0.0` image is now published to `:latest` immediately after the release branch is created, not only after the first bugfix merges -- `release/**` branches are now protected: direct commits are blocked, a pull request is required, and the `PR Summary` status check must pass before merging; repository admins retain bypass rights so the automated version bump can still land diff --git a/setup/IdentityAtlas.psd1 b/setup/IdentityAtlas.psd1 index 317dc3330..1d03ef92d 100644 --- a/setup/IdentityAtlas.psd1 +++ b/setup/IdentityAtlas.psd1 @@ -12,7 +12,7 @@ RootModule = '.\IdentityAtlas.psm1' # Version number of this module. -ModuleVersion = '5.1.20260417.1306' +ModuleVersion = '5.2.20260418.0620' # Supported PSEditions # CompatiblePSEditions = @() From db77ce1c5f494fdbcdb66db18259ed1267e36b31 Mon Sep 17 00:00:00 2001 From: Wim van den Heijkant Date: Sat, 18 Apr 2026 09:16:03 +0200 Subject: [PATCH 009/160] Fix Users/Resources filter dropdown after Postgres migration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit columnCache.js queried information_schema with lowercase table names ('principals', 'resources') but the v5 migration creates quoted PascalCase tables ("Principals", "Resources"). Postgres is case-sensitive on quoted identifiers, so every lookup returned zero columns and the filter dropdown collapsed to just the synthetic tag field. Also dropped the now-unused snakeToCamel helper — columns are already camelCase in the real schema — and the stale SYSTEM_COLS export (routes define their own local copies). Co-Authored-By: Claude Opus 4.7 (1M context) --- app/api/src/db/columnCache.js | 27 ++++++++----------- ...fixes-fix-user-resource-filter-dropdown.md | 1 + 2 files changed, 12 insertions(+), 16 deletions(-) create mode 100644 changes/bugfixes-fix-user-resource-filter-dropdown.md diff --git a/app/api/src/db/columnCache.js b/app/api/src/db/columnCache.js index 47bb697dc..377e630d5 100644 --- a/app/api/src/db/columnCache.js +++ b/app/api/src/db/columnCache.js @@ -5,12 +5,13 @@ // column. Both queries are cached for 5 minutes; an in-flight deduplication // promise prevents thundering-herd on cold cache. // -// In v5 the only tables are postgres `principals` and `resources` (snake_case). +// In v5 the only tables are postgres `Principals` and `Resources`. They are +// created with quoted PascalCase identifiers (see migrations/001_core_schema.sql) +// and the columns are also camelCase — information_schema lookups therefore +// need the exact case. +// // The legacy `GraphUsers` / `GraphGroups` paths are removed — they were the v3 // pre-universal-resource-model fallback and have been dead code since v3.1. -// -// Returned column shape stays in camelCase so the frontend doesn't need -// changes — we map snake_case → camelCase here. import * as db from './connection.js'; @@ -27,11 +28,6 @@ const FILTERABLE_TYPES = new Set([ // we only feed it information_schema output. const SAFE_IDENT_RE = /^[a-zA-Z0-9_]+$/; -// Convert postgres column name to camelCase for the API response -function snakeToCamel(s) { - return s.replace(/_([a-z0-9])/g, (_, c) => c.toUpperCase()); -} - // ─── Schema cache ─────────────────────────────────────────────── let principalColumnsCache = null; let principalColumnsCacheTime = 0; @@ -44,12 +40,12 @@ async function discoverColumns(table) { `SELECT column_name, data_type FROM information_schema.columns WHERE table_schema = 'public' AND table_name = $1 - AND column_name NOT IN ('id', 'system_id', 'extended_attributes') + AND column_name NOT IN ('id', 'systemId', 'extendedAttributes') ORDER BY ordinal_position`, [table] ); return r.rows.map(row => ({ - name: snakeToCamel(row.column_name), + name: row.column_name, rawName: row.column_name, type: row.data_type, })); @@ -60,7 +56,7 @@ export async function getPrincipalColumns(_pool) { if (principalColumnsCache && (now - principalColumnsCacheTime) < COLUMN_CACHE_TTL) { return principalColumnsCache; } - principalColumnsCache = await discoverColumns('principals'); + principalColumnsCache = await discoverColumns('Principals'); principalColumnsCacheTime = now; return principalColumnsCache; } @@ -70,7 +66,7 @@ export async function getResourceColumns(_pool) { if (resourceColumnsCache && (now - resourceColumnsCacheTime) < COLUMN_CACHE_TTL) { return resourceColumnsCache; } - resourceColumnsCache = await discoverColumns('resources'); + resourceColumnsCache = await discoverColumns('Resources'); resourceColumnsCacheTime = now; return resourceColumnsCache; } @@ -123,7 +119,7 @@ export async function getPrincipalColumnValues(_pool) { principalValuesInflight = (async () => { try { const cols = await getPrincipalColumns(null); - const result = await discoverColumnValues('principals', cols); + const result = await discoverColumnValues('Principals', cols); principalValuesCache = result; principalValuesCacheTime = Date.now(); return result; @@ -143,7 +139,7 @@ export async function getResourceColumnValues(_pool) { resourceValuesInflight = (async () => { try { const cols = await getResourceColumns(null); - const result = await discoverColumnValues('resources', cols); + const result = await discoverColumnValues('Resources', cols); resourceValuesCache = result; resourceValuesCacheTime = Date.now(); return result; @@ -159,4 +155,3 @@ export const getGroupColumnValues = getResourceColumnValues; export const getPrincipalOrUserColumnValues = getPrincipalColumnValues; export { FILTERABLE_TYPES }; -export const SYSTEM_COLS = new Set(['id', 'system_id', 'extended_attributes']); diff --git a/changes/bugfixes-fix-user-resource-filter-dropdown.md b/changes/bugfixes-fix-user-resource-filter-dropdown.md new file mode 100644 index 000000000..8ecdb02b1 --- /dev/null +++ b/changes/bugfixes-fix-user-resource-filter-dropdown.md @@ -0,0 +1 @@ +- Fixed the "Filters" dropdown on the Users and Resources pages — attribute fields (Department, Job Title, etc.) and their values are populated again. The list had regressed to showing only "User Tag" because column discovery was looking up table names in the wrong case after the PostgreSQL migration. From 8313cf0a775a4022c0612ca600eb942684b7b6a4 Mon Sep 17 00:00:00 2001 From: Wim van den Heijkant Date: Sat, 18 Apr 2026 09:28:51 +0200 Subject: [PATCH 010/160] Add regression tests for column discovery casing Pins the PostgreSQL table names ("Principals"/"Resources") and the camelCase system-column exclusion list that column discovery must use. These are the exact points where the filter-dropdown regression slipped in, so a future snake_case/lowercase change will fail CI instead of silently returning an empty column list. Tests are unit-only (mocked db.query) and run as part of the existing `unit-js` Vitest job on every PR. Co-Authored-By: Claude Opus 4.7 (1M context) --- app/api/src/db/columnCache.test.js | 134 ++++++++++++++++++ ...fixes-fix-user-resource-filter-dropdown.md | 1 + 2 files changed, 135 insertions(+) create mode 100644 app/api/src/db/columnCache.test.js diff --git a/app/api/src/db/columnCache.test.js b/app/api/src/db/columnCache.test.js new file mode 100644 index 000000000..a4d8c85d9 --- /dev/null +++ b/app/api/src/db/columnCache.test.js @@ -0,0 +1,134 @@ +// Regression tests for columnCache.js. +// +// The filter dropdown on the Users / Resources pages is populated from column +// discovery against information_schema. The v5 Postgres migration creates +// quoted-PascalCase tables ("Principals", "Resources") with camelCase columns; +// Postgres is case-sensitive on quoted identifiers, so a lowercase lookup +// silently returns zero rows and the UI dropdown collapses to just the +// synthetic tag field. These tests pin the casing so that regression can't +// slip back in. + +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +// We mock `./connection.js` with a query spy that each test can program. +// Vitest hoists vi.mock() above imports, so this runs before columnCache +// loads its `db` dependency. +const queryMock = vi.fn(); +vi.mock('./connection.js', () => ({ + query: (...args) => queryMock(...args), +})); + +// Helper: load a *fresh* copy of columnCache so the module-scoped caches +// don't leak state between tests. +async function freshModule() { + vi.resetModules(); + return await import('./columnCache.js'); +} + +beforeEach(() => { + queryMock.mockReset(); +}); + +describe('discoverColumns — table/column casing pinned to migrations', () => { + it('queries information_schema with PascalCase "Principals"', async () => { + queryMock.mockResolvedValue({ rows: [] }); + const mod = await freshModule(); + await mod.getPrincipalColumns(); + + expect(queryMock).toHaveBeenCalledTimes(1); + const [, params] = queryMock.mock.calls[0]; + expect(params).toEqual(['Principals']); + }); + + it('queries information_schema with PascalCase "Resources"', async () => { + queryMock.mockResolvedValue({ rows: [] }); + const mod = await freshModule(); + await mod.getResourceColumns(); + + const [, params] = queryMock.mock.calls[0]; + expect(params).toEqual(['Resources']); + }); + + it('excludes the camelCase system columns (not snake_case)', async () => { + queryMock.mockResolvedValue({ rows: [] }); + const mod = await freshModule(); + await mod.getPrincipalColumns(); + + const [sql] = queryMock.mock.calls[0]; + expect(sql).toMatch(/column_name NOT IN \('id', 'systemId', 'extendedAttributes'\)/); + expect(sql).not.toMatch(/system_id|extended_attributes/); + }); + + it('returns column metadata with camelCase names (no snake→camel conversion)', async () => { + queryMock.mockResolvedValue({ + rows: [ + { column_name: 'displayName', data_type: 'text' }, + { column_name: 'jobTitle', data_type: 'text' }, + ], + }); + const mod = await freshModule(); + const cols = await mod.getPrincipalColumns(); + + expect(cols).toEqual([ + { name: 'displayName', rawName: 'displayName', type: 'text' }, + { name: 'jobTitle', rawName: 'jobTitle', type: 'text' }, + ]); + }); +}); + +describe('discoverColumnValues — emits correctly-quoted PascalCase table name', () => { + // Each call to get{Principal,Resource}ColumnValues makes two queries: + // 1. discoverColumns (information_schema) + // 2. the UNION ALL over distinct values + // We program both responses in order. + function programSchemaThenValues(columns, valueRows) { + queryMock + .mockResolvedValueOnce({ rows: columns.map(c => ({ column_name: c.name, data_type: c.type })) }) + .mockResolvedValueOnce({ rows: valueRows }); + } + + it('Principals: SELECTs FROM "Principals" with double-quoted PascalCase', async () => { + programSchemaThenValues( + [{ name: 'department', type: 'text' }], + [{ col: 'department', val: 'Sales' }], + ); + const mod = await freshModule(); + const grouped = await mod.getPrincipalColumnValues(); + + const valuesSql = queryMock.mock.calls[1][0]; + expect(valuesSql).toMatch(/FROM "Principals"/); + expect(valuesSql).not.toMatch(/FROM "principals"/); + expect(grouped).toEqual({ department: ['Sales'] }); + }); + + it('Resources: SELECTs FROM "Resources" with double-quoted PascalCase', async () => { + programSchemaThenValues( + [{ name: 'resourceType', type: 'text' }], + [{ col: 'resourceType', val: 'Group' }], + ); + const mod = await freshModule(); + await mod.getResourceColumnValues(); + + const valuesSql = queryMock.mock.calls[1][0]; + expect(valuesSql).toMatch(/FROM "Resources"/); + expect(valuesSql).not.toMatch(/FROM "resources"/); + }); + + it('skips columns whose type is not in FILTERABLE_TYPES (e.g. jsonb, uuid)', async () => { + programSchemaThenValues( + [ + { name: 'displayName', type: 'text' }, + { name: 'extendedAttributes', type: 'jsonb' }, + { name: 'id', type: 'uuid' }, + ], + [], + ); + const mod = await freshModule(); + await mod.getPrincipalColumnValues(); + + const valuesSql = queryMock.mock.calls[1][0]; + expect(valuesSql).toMatch(/"displayName"/); + expect(valuesSql).not.toMatch(/"extendedAttributes"/); + expect(valuesSql).not.toMatch(/\buuid\b/); + }); +}); diff --git a/changes/bugfixes-fix-user-resource-filter-dropdown.md b/changes/bugfixes-fix-user-resource-filter-dropdown.md index 8ecdb02b1..7e78bce1a 100644 --- a/changes/bugfixes-fix-user-resource-filter-dropdown.md +++ b/changes/bugfixes-fix-user-resource-filter-dropdown.md @@ -1 +1,2 @@ - Fixed the "Filters" dropdown on the Users and Resources pages — attribute fields (Department, Job Title, etc.) and their values are populated again. The list had regressed to showing only "User Tag" because column discovery was looking up table names in the wrong case after the PostgreSQL migration. +- Added regression tests pinning the PostgreSQL table and column casing used by column discovery, so the same mismatch can't slip back in. From 43f1750e3bbc4891b0b1df3e20a9a6c61daa7aec Mon Sep 17 00:00:00 2001 From: Wim van den Heijkant Date: Sat, 18 Apr 2026 09:50:03 +0200 Subject: [PATCH 011/160] Allow filtering on extendedAttributes keys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Top-level scalar keys inside the Principals/Resources extendedAttributes JSONB column are now discovered alongside real columns and surfaced in the Users and Resources filter dropdowns under an `ext.` namespace (e.g. ext.userType, ext.onPremisesSyncEnabled, ext.extensionAttribute5). - columnCache.js — new discoverExtendedAttrValues() runs in parallel with the existing column-values scan and contributes ext. entries to the same cached result. Only string/number/boolean jsonb types are considered; object/array keys like signInActivity and groupTypes are skipped. Key names are validated with the same SAFE_IDENT_RE as everywhere else so the inlined JSON-path can't be an injection vector. - tags.js — buildFilterWhere now recognises `ext.` filter fields, validates the suffix against the same regex, and emits `"extendedAttributes"->>'key' = @param`. Real columns continue to go through the whitelist check. The helper is now exported so resources.js can reuse it instead of duplicating the filter loop. - resources.js — drops the inlined filter loop and calls the shared helper, picking up ext filter support for free. - useEntityPage.js — humanises `ext.` field names as " (ext)" so they're visually distinct from real columns in the field picker. Unit tests (vitest) cover ext-key discovery SQL shape, the scalar-type filter, unsafe-key rejection, and buildFilterWhere's ext-prefix path with injection-attempt fixtures. Co-Authored-By: Claude Opus 4.7 (1M context) --- app/api/src/db/columnCache.js | 61 +++++++++- app/api/src/db/columnCache.test.js | 101 ++++++++++++++--- app/api/src/routes/buildFilterWhere.test.js | 106 ++++++++++++++++++ app/api/src/routes/resources.js | 13 +-- app/api/src/routes/tags.js | 26 ++++- app/ui/src/hooks/useEntityPage.js | 19 +++- ...fixes-fix-user-resource-filter-dropdown.md | 1 + 7 files changed, 291 insertions(+), 36 deletions(-) create mode 100644 app/api/src/routes/buildFilterWhere.test.js diff --git a/app/api/src/db/columnCache.js b/app/api/src/db/columnCache.js index 377e630d5..dbbe42052 100644 --- a/app/api/src/db/columnCache.js +++ b/app/api/src/db/columnCache.js @@ -110,6 +110,55 @@ async function discoverColumnValues(table, columns) { return grouped; } +// Discover scalar top-level keys in the `extendedAttributes` JSONB column and +// their distinct values. The flat column list returned by `discoverColumns` +// deliberately excludes `extendedAttributes` (it's a blob, not directly +// filterable), but individual string/number/boolean keys INSIDE the blob are +// very useful filter fields — e.g. `userType`, `onPremisesSyncEnabled`, +// `extensionAttribute5`. They're surfaced under namespaced keys like +// `ext.userType` so the front end and `buildFilterWhere` can tell them apart +// from real columns and emit JSON-path SQL (`"extendedAttributes"->>'key'`). +// +// Object/array-valued keys (e.g. `signInActivity`, `groupTypes`) are skipped — +// matching on a serialized object is not a useful filter. +async function discoverExtendedAttrValues(table) { + if (!SAFE_IDENT_RE.test(table)) throw new Error(`Invalid table name: ${table}`); + + // Find distinct scalar top-level keys. We use jsonb_typeof on the value so + // we only keep keys whose typical content is something a user would filter + // on; if a key is mixed (string in some rows, object in others) we'd lose + // the object rows, but the filter still matches the scalar ones. + const keysRes = await db.query( + `SELECT DISTINCT key + FROM "${table}", jsonb_object_keys("extendedAttributes") AS key + WHERE "extendedAttributes" IS NOT NULL + AND jsonb_typeof("extendedAttributes"->key) IN ('string', 'number', 'boolean')` + ); + const keys = keysRes.rows.map(r => r.key).filter(k => SAFE_IDENT_RE.test(k)); + if (keys.length === 0) return {}; + + // One UNION ALL per key — same shape as discoverColumnValues. The + // `->> 'key'` form returns text for any scalar jsonb type, which is what + // we want: booleans become 'true'/'false', numbers become their printed form. + const parts = keys.map(k => + `SELECT 'ext.${k}' AS col, val FROM ( + SELECT DISTINCT "extendedAttributes"->>'${k}' AS val FROM "${table}" + WHERE "extendedAttributes" ? '${k}' + AND "extendedAttributes"->>'${k}' IS NOT NULL + AND "extendedAttributes"->>'${k}' <> '' + LIMIT 500 + ) t` + ); + + const r = await db.query(parts.join('\nUNION ALL\n') + '\nORDER BY col, val'); + const grouped = {}; + for (const row of r.rows) { + if (!grouped[row.col]) grouped[row.col] = []; + grouped[row.col].push(row.val); + } + return grouped; +} + export async function getPrincipalColumnValues(_pool) { const now = Date.now(); if (principalValuesCache && (now - principalValuesCacheTime) < COLUMN_CACHE_TTL) { @@ -119,7 +168,11 @@ export async function getPrincipalColumnValues(_pool) { principalValuesInflight = (async () => { try { const cols = await getPrincipalColumns(null); - const result = await discoverColumnValues('Principals', cols); + const [base, ext] = await Promise.all([ + discoverColumnValues('Principals', cols), + discoverExtendedAttrValues('Principals'), + ]); + const result = { ...base, ...ext }; principalValuesCache = result; principalValuesCacheTime = Date.now(); return result; @@ -139,7 +192,11 @@ export async function getResourceColumnValues(_pool) { resourceValuesInflight = (async () => { try { const cols = await getResourceColumns(null); - const result = await discoverColumnValues('Resources', cols); + const [base, ext] = await Promise.all([ + discoverColumnValues('Resources', cols), + discoverExtendedAttrValues('Resources'), + ]); + const result = { ...base, ...ext }; resourceValuesCache = result; resourceValuesCacheTime = Date.now(); return result; diff --git a/app/api/src/db/columnCache.test.js b/app/api/src/db/columnCache.test.js index a4d8c85d9..c134ac4d3 100644 --- a/app/api/src/db/columnCache.test.js +++ b/app/api/src/db/columnCache.test.js @@ -77,19 +77,27 @@ describe('discoverColumns — table/column casing pinned to migrations', () => { }); describe('discoverColumnValues — emits correctly-quoted PascalCase table name', () => { - // Each call to get{Principal,Resource}ColumnValues makes two queries: + // Each call to get{Principal,Resource}ColumnValues makes three queries in + // this order: // 1. discoverColumns (information_schema) - // 2. the UNION ALL over distinct values - // We program both responses in order. - function programSchemaThenValues(columns, valueRows) { + // 2. discoverColumnValues (UNION ALL over filterable columns) + // 3. discoverExtendedAttrValues — key discovery on the JSONB column + // 4. (optional) distinct-value UNION ALL over the ext keys from step 3 + // Tests program as many responses as they inspect; unused ones can be + // left as empty rows. + function programQueries(columnRows, valueRows, extKeyRows = [], extValueRows = []) { queryMock - .mockResolvedValueOnce({ rows: columns.map(c => ({ column_name: c.name, data_type: c.type })) }) - .mockResolvedValueOnce({ rows: valueRows }); + .mockResolvedValueOnce({ rows: columnRows }) + .mockResolvedValueOnce({ rows: valueRows }) + .mockResolvedValueOnce({ rows: extKeyRows }); + if (extKeyRows.length > 0) { + queryMock.mockResolvedValueOnce({ rows: extValueRows }); + } } it('Principals: SELECTs FROM "Principals" with double-quoted PascalCase', async () => { - programSchemaThenValues( - [{ name: 'department', type: 'text' }], + programQueries( + [{ column_name: 'department', data_type: 'text' }], [{ col: 'department', val: 'Sales' }], ); const mod = await freshModule(); @@ -102,8 +110,8 @@ describe('discoverColumnValues — emits correctly-quoted PascalCase table name' }); it('Resources: SELECTs FROM "Resources" with double-quoted PascalCase', async () => { - programSchemaThenValues( - [{ name: 'resourceType', type: 'text' }], + programQueries( + [{ column_name: 'resourceType', data_type: 'text' }], [{ col: 'resourceType', val: 'Group' }], ); const mod = await freshModule(); @@ -115,11 +123,11 @@ describe('discoverColumnValues — emits correctly-quoted PascalCase table name' }); it('skips columns whose type is not in FILTERABLE_TYPES (e.g. jsonb, uuid)', async () => { - programSchemaThenValues( + programQueries( [ - { name: 'displayName', type: 'text' }, - { name: 'extendedAttributes', type: 'jsonb' }, - { name: 'id', type: 'uuid' }, + { column_name: 'displayName', data_type: 'text' }, + { column_name: 'extendedAttributes', data_type: 'jsonb' }, + { column_name: 'id', data_type: 'uuid' }, ], [], ); @@ -132,3 +140,68 @@ describe('discoverColumnValues — emits correctly-quoted PascalCase table name' expect(valuesSql).not.toMatch(/\buuid\b/); }); }); + +describe('discoverExtendedAttrValues — surfaces JSONB keys as ext.', () => { + it('enumerates scalar JSONB keys and emits distinct values under ext.', async () => { + queryMock + // discoverColumns — keep tiny so we reach the ext phase quickly + .mockResolvedValueOnce({ rows: [{ column_name: 'department', data_type: 'text' }] }) + // discoverColumnValues — base UNION ALL + .mockResolvedValueOnce({ rows: [{ col: 'department', val: 'Sales' }] }) + // ext key discovery + .mockResolvedValueOnce({ rows: [{ key: 'userType' }, { key: 'onPremisesSyncEnabled' }] }) + // ext value UNION ALL + .mockResolvedValueOnce({ rows: [ + { col: 'ext.userType', val: 'Member' }, + { col: 'ext.userType', val: 'Guest' }, + { col: 'ext.onPremisesSyncEnabled', val: 'true' }, + ]}); + + const mod = await freshModule(); + const grouped = await mod.getPrincipalColumnValues(); + + expect(grouped['department']).toEqual(['Sales']); + expect(grouped['ext.userType']).toEqual(['Member', 'Guest']); + expect(grouped['ext.onPremisesSyncEnabled']).toEqual(['true']); + + // Ext key-discovery SQL must restrict to scalar jsonb types — that's what + // excludes objects (signInActivity) and arrays (groupTypes) from the list. + const keyDiscoverySql = queryMock.mock.calls[2][0]; + expect(keyDiscoverySql).toMatch(/jsonb_typeof.*IN \('string', 'number', 'boolean'\)/); + expect(keyDiscoverySql).toMatch(/FROM "Principals"/); + + // Ext value SQL must use the ->>'' form on the extendedAttributes + // column. If anyone changes it back to `->` (returning jsonb) string + // equality breaks for booleans/numbers. + const extValuesSql = queryMock.mock.calls[3][0]; + expect(extValuesSql).toMatch(/"extendedAttributes"->>'userType'/); + expect(extValuesSql).toMatch(/"extendedAttributes"->>'onPremisesSyncEnabled'/); + }); + + it('drops keys whose name contains unsafe characters (no SQL-injection vector)', async () => { + queryMock + .mockResolvedValueOnce({ rows: [] }) // discoverColumns — empty is fine + // No base values query because filterableCols is empty → discoverColumnValues returns {} + // Actually it WILL issue the UNION ALL only when filterableCols.length > 0, so skip it. + // But we still hit the ext key query: + .mockResolvedValueOnce({ rows: [ + { key: 'userType' }, + { key: "badKey'; DROP TABLE--" }, + { key: 'extension_deadbeef_sAMAccountName' }, + ]}) + .mockResolvedValueOnce({ rows: [ + { col: 'ext.userType', val: 'Member' }, + { col: 'ext.extension_deadbeef_sAMAccountName', val: 'jdoe' }, + ]}); + + const mod = await freshModule(); + await mod.getPrincipalColumnValues(); + + // Call sequence with an empty column list: schema, ext-key-discovery, + // ext-value UNION. The base-values query is skipped. + const extValuesSql = queryMock.mock.calls[2][0]; + expect(extValuesSql).toMatch(/'userType'/); + expect(extValuesSql).toMatch(/'extension_deadbeef_sAMAccountName'/); + expect(extValuesSql).not.toMatch(/DROP TABLE/); + }); +}); diff --git a/app/api/src/routes/buildFilterWhere.test.js b/app/api/src/routes/buildFilterWhere.test.js new file mode 100644 index 000000000..024ae51df --- /dev/null +++ b/app/api/src/routes/buildFilterWhere.test.js @@ -0,0 +1,106 @@ +// Unit tests for the shared `buildFilterWhere` helper. +// +// The helper is used by /api/users and /api/resources to build a +// parameterized WHERE clause from a JSON filter object. Two code paths are +// exercised separately — real columns (validated against a whitelist) and +// `ext.` filters on the `extendedAttributes` JSONB column (validated +// via regex because a JSON path key can't be parameter-bound). + +import { describe, it, expect } from 'vitest'; +import { buildFilterWhere } from './tags.js'; + +// Minimal stand-in for the mssql-compat `request` object: records every +// .input() call so we can assert on the parameter bindings. +function fakeRequest() { + const bound = {}; + return { + bound, + input(name, value) { bound[name] = value; return this; }, + }; +} + +describe('buildFilterWhere — real columns', () => { + it('emits parameterised equality on a valid column', () => { + const req = fakeRequest(); + const sql = buildFilterWhere(req, { department: 'Sales' }, new Set(['department']), 'u'); + expect(sql).toBe(' AND u."department"::text = @fl0'); + expect(req.bound).toEqual({ fl0: 'Sales' }); + }); + + it('silently drops fields that are not in the whitelist', () => { + const req = fakeRequest(); + const sql = buildFilterWhere(req, { nopeColumn: 'x' }, new Set(['department']), 'u'); + expect(sql).toBe(''); + expect(req.bound).toEqual({}); + }); + + it('skips empty / null / undefined values', () => { + const req = fakeRequest(); + const sql = buildFilterWhere( + req, + { department: '', jobTitle: null, companyName: undefined }, + new Set(['department', 'jobTitle', 'companyName']), + 'u', + ); + expect(sql).toBe(''); + expect(req.bound).toEqual({}); + }); + + it('uses the requested alias and param prefix', () => { + const req = fakeRequest(); + const sql = buildFilterWhere(req, { resourceType: 'Group' }, new Set(['resourceType']), 'r', 'bf'); + expect(sql).toBe(' AND r."resourceType"::text = @bf0'); + expect(req.bound).toEqual({ bf0: 'Group' }); + }); +}); + +describe('buildFilterWhere — extended-attribute filters', () => { + it('emits JSON-path SQL for ext. filters', () => { + const req = fakeRequest(); + const sql = buildFilterWhere(req, { 'ext.userType': 'Guest' }, new Set(), 'u'); + expect(sql).toBe(` AND u."extendedAttributes"->>'userType' = @fl0`); + expect(req.bound).toEqual({ fl0: 'Guest' }); + }); + + it('does NOT require ext keys to be in the column whitelist', () => { + const req = fakeRequest(); + // validColNames is intentionally empty — ext keys bypass whitelist + // because they're validated via regex instead. + const sql = buildFilterWhere(req, { 'ext.onPremisesSyncEnabled': 'true' }, new Set(), 'p'); + expect(sql).toBe(` AND p."extendedAttributes"->>'onPremisesSyncEnabled' = @fl0`); + }); + + it('rejects ext keys containing characters outside [a-zA-Z0-9_]', () => { + const req = fakeRequest(); + const sql = buildFilterWhere( + req, + { + "ext.badKey'; DROP TABLE--": 'x', + 'ext.bad-dash': 'x', + 'ext.bad.dot': 'x', + 'ext.normalKey': 'ok', + }, + new Set(), + 'u', + ); + // Only the safe key survives. + expect(sql).toBe(` AND u."extendedAttributes"->>'normalKey' = @fl0`); + expect(sql).not.toMatch(/DROP TABLE/); + expect(sql).not.toMatch(/bad-dash|bad\.dot/); + expect(req.bound).toEqual({ fl0: 'ok' }); + }); + + it('mixes real columns and ext keys with shared param counter', () => { + const req = fakeRequest(); + const sql = buildFilterWhere( + req, + { department: 'Sales', 'ext.userType': 'Member' }, + new Set(['department']), + 'u', + ); + // Both filters produced, each with its own @fl binding. + expect(sql).toMatch(/u\."department"::text = @fl0/); + expect(sql).toMatch(/u\."extendedAttributes"->>'userType' = @fl1/); + expect(req.bound).toEqual({ fl0: 'Sales', fl1: 'Member' }); + }); +}); diff --git a/app/api/src/routes/resources.js b/app/api/src/routes/resources.js index 16abfba87..cbdf2b20f 100644 --- a/app/api/src/routes/resources.js +++ b/app/api/src/routes/resources.js @@ -1,7 +1,7 @@ import { Router } from 'express'; import { timedRequest } from '../perf/sqlTimer.js'; import { getResourceColumns, getResourceColumnValues, FILTERABLE_TYPES } from '../db/columnCache.js'; -import { ensureTagTables } from './tags.js'; +import { ensureTagTables, buildFilterWhere } from './tags.js'; const router = Router(); const useSql = process.env.USE_SQL === 'true'; @@ -76,16 +76,7 @@ router.get('/resources', async (req, res) => { const cols = await getResourceColumns(p); const colNames = new Set(cols.map(c => c.name)); - let filterWhere = ''; - let idx = 0; - for (const [field, value] of Object.entries(attrFilters)) { - if (colNames.has(field) && value != null && String(value) !== '') { - const paramName = `fl${idx}`; - filterWhere += ` AND r."${field}"::text = @${paramName}`; - request.input(paramName, String(value)); - idx++; - } - } + const filterWhere = buildFilterWhere(request, attrFilters, colNames, 'r'); let where = '1=1'; if (search) { diff --git a/app/api/src/routes/tags.js b/app/api/src/routes/tags.js index 912ac1711..ac759d49c 100644 --- a/app/api/src/routes/tags.js +++ b/app/api/src/routes/tags.js @@ -20,13 +20,31 @@ let tablesReady = false; async function ensureTagTables(_pool) { tablesReady = true; } export { ensureTagTables }; -// Build parameterized WHERE clause from filters object, validating against actual columns. -function buildFilterWhere(requestObj, filters, validColNames, alias, paramPrefix = 'fl') { +// Build parameterized WHERE clause from filters object. +// +// Two kinds of filter keys are accepted: +// - Real column names — validated against `validColNames` to prevent SQL +// injection via field names, emitted as `alias."col"::text = @param`. +// - `ext.` — filters on a scalar value inside the `extendedAttributes` +// JSONB column. The suffix must match FILTER_KEY_RE so it's safe to +// inline (we can't parameter-bind a JSON path key). Emitted as +// `alias."extendedAttributes"->>'key' = @param`. +const FILTER_KEY_RE = /^[a-zA-Z0-9_]+$/; +const EXT_PREFIX = 'ext.'; +export function buildFilterWhere(requestObj, filters, validColNames, alias, paramPrefix = 'fl') { let where = ''; let idx = 0; for (const [field, value] of Object.entries(filters)) { - if (validColNames.has(field) && value != null && String(value) !== '') { - const paramName = `${paramPrefix}${idx}`; + if (value == null || String(value) === '') continue; + const paramName = `${paramPrefix}${idx}`; + + if (field.startsWith(EXT_PREFIX)) { + const key = field.slice(EXT_PREFIX.length); + if (!FILTER_KEY_RE.test(key)) continue; + where += ` AND ${alias}."extendedAttributes"->>'${key}' = @${paramName}`; + requestObj.input(paramName, String(value)); + idx++; + } else if (validColNames.has(field)) { where += ` AND ${alias}."${field}"::text = @${paramName}`; requestObj.input(paramName, String(value)); idx++; diff --git a/app/ui/src/hooks/useEntityPage.js b/app/ui/src/hooks/useEntityPage.js index 5e05c6ade..dcabf4751 100644 --- a/app/ui/src/hooks/useEntityPage.js +++ b/app/ui/src/hooks/useEntityPage.js @@ -242,14 +242,23 @@ export default function useEntityPage({ authFetch, entityType, listEndpoint, col const allOnPageSelected = items.length > 0 && selected.size === items.length; const hasAnyFilter = activeFilters.length > 0 || debouncedSearch; - // Build filterFields from availableColumns + // Build filterFields from availableColumns. Keys that start with `ext.` + // are extended-attribute filters (see api columnCache.js / buildFilterWhere) + // — we strip the prefix for display and tag the label with "(ext)" so + // they're visually distinct from real columns in the dropdown. const getFilterFields = useCallback((fieldLabels) => { + const humanize = (s) => s.replace(/([A-Z])/g, ' $1').replace(/^./, c => c.toUpperCase()).trim(); return availableColumns .filter(col => col.values && col.values.length >= 1 && col.values.length <= 500) - .map(col => ({ - key: col.column, - label: fieldLabels[col.column] || col.column.replace(/([A-Z])/g, ' $1').replace(/^./, s => s.toUpperCase()).trim(), - })); + .map(col => { + if (fieldLabels[col.column]) { + return { key: col.column, label: fieldLabels[col.column] }; + } + if (col.column.startsWith('ext.')) { + return { key: col.column, label: `${humanize(col.column.slice(4))} (ext)` }; + } + return { key: col.column, label: humanize(col.column) }; + }); }, [availableColumns]); const getOptionsForField = useCallback((fieldKey) => { diff --git a/changes/bugfixes-fix-user-resource-filter-dropdown.md b/changes/bugfixes-fix-user-resource-filter-dropdown.md index 7e78bce1a..8f18de178 100644 --- a/changes/bugfixes-fix-user-resource-filter-dropdown.md +++ b/changes/bugfixes-fix-user-resource-filter-dropdown.md @@ -1,2 +1,3 @@ - Fixed the "Filters" dropdown on the Users and Resources pages — attribute fields (Department, Job Title, etc.) and their values are populated again. The list had regressed to showing only "User Tag" because column discovery was looking up table names in the wrong case after the PostgreSQL migration. - Added regression tests pinning the PostgreSQL table and column casing used by column discovery, so the same mismatch can't slip back in. +- Added extended-attribute fields to the Users and Resources filter dropdowns. Keys stored inside the `extendedAttributes` JSON blob (e.g. `userType`, `onPremisesSyncEnabled`, `extensionAttribute5`, `city`, `country`) are now filterable via the UI, labelled with a "(ext)" suffix so they're distinguishable from regular columns. From f6ccbf305add5ec43b6d1cf31e300f5223ff625b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 18 Apr 2026 10:29:12 +0000 Subject: [PATCH 012/160] chore: bump version to 5.3.20260418.1029 --- CHANGES.md | 6 ++++++ changes/bugfixes-fix-user-resource-filter-dropdown.md | 3 --- setup/IdentityAtlas.psd1 | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) delete mode 100644 changes/bugfixes-fix-user-resource-filter-dropdown.md diff --git a/CHANGES.md b/CHANGES.md index a5bc96b8d..b2720a400 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,11 @@ ## Changes in this PR +- Fixed the "Filters" dropdown on the Users and Resources pages — attribute fields (Department, Job Title, etc.) and their values are populated again. The list had regressed to showing only "User Tag" because column discovery was looking up table names in the wrong case after the PostgreSQL migration. +- Added regression tests pinning the PostgreSQL table and column casing used by column discovery, so the same mismatch can't slip back in. +- Added extended-attribute fields to the Users and Resources filter dropdowns. Keys stored inside the `extendedAttributes` JSON blob (e.g. `userType`, `onPremisesSyncEnabled`, `extensionAttribute5`, `city`, `country`) are now filterable via the UI, labelled with a "(ext)" suffix so they're distinguishable from regular columns. + +## Changes in this PR + - Added stable release branch model: `release/vX.Y` branches are cut from `main` and receive only bugfixes, giving customers a stable `:latest` Docker image that does not change when new features land on `main` - New `cut-release.yml` workflow creates a release branch and sets the initial version (e.g. `5.2.0.0`) with one click from GitHub Actions - Main branch builds now push the `:edge` Docker tag instead of `:latest`, so customers on `:latest` only receive intentional releases diff --git a/changes/bugfixes-fix-user-resource-filter-dropdown.md b/changes/bugfixes-fix-user-resource-filter-dropdown.md deleted file mode 100644 index 8f18de178..000000000 --- a/changes/bugfixes-fix-user-resource-filter-dropdown.md +++ /dev/null @@ -1,3 +0,0 @@ -- Fixed the "Filters" dropdown on the Users and Resources pages — attribute fields (Department, Job Title, etc.) and their values are populated again. The list had regressed to showing only "User Tag" because column discovery was looking up table names in the wrong case after the PostgreSQL migration. -- Added regression tests pinning the PostgreSQL table and column casing used by column discovery, so the same mismatch can't slip back in. -- Added extended-attribute fields to the Users and Resources filter dropdowns. Keys stored inside the `extendedAttributes` JSON blob (e.g. `userType`, `onPremisesSyncEnabled`, `extensionAttribute5`, `city`, `country`) are now filterable via the UI, labelled with a "(ext)" suffix so they're distinguishable from regular columns. diff --git a/setup/IdentityAtlas.psd1 b/setup/IdentityAtlas.psd1 index 1d03ef92d..774e9f49f 100644 --- a/setup/IdentityAtlas.psd1 +++ b/setup/IdentityAtlas.psd1 @@ -12,7 +12,7 @@ RootModule = '.\IdentityAtlas.psm1' # Version number of this module. -ModuleVersion = '5.2.20260418.0620' +ModuleVersion = '5.3.20260418.1029' # Supported PSEditions # CompatiblePSEditions = @() From f86d036b954a66d98572fef49c417b77bd488b4c Mon Sep 17 00:00:00 2001 From: Wim van den Heijkant Date: Sat, 18 Apr 2026 15:54:01 +0200 Subject: [PATCH 013/160] Sync Entra service principals as Principals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Service principals (enterprise-app SPs, managed identities, AI agents) own a large share of role assignments in Entra and Azure but were not previously ingested by the Entra crawler. This adds a dedicated sync phase so they land in the Principals table alongside user accounts, which is a prerequisite for the forthcoming Azure RM crawler (role assignments there frequently target SPs). - tools/powershell-sdk/helpers/Get-FGServicePrincipalType.ps1 — new classifier implementing the CLAUDE.md taxonomy. Rules apply in order: (1) servicePrincipalType == 'ManagedIdentity', (2) well-known AI platform tags, (3) built-in + caller-supplied display-name heuristics, (4) default ServicePrincipal. WorkloadIdentity is out of scope — it can't be decided from an SP object alone. - tools/crawlers/entra-id/Start-EntraIDCrawler.ps1 — gated SP sync block behind the existing $SyncServicePrincipals switch. Fetches /servicePrincipals with a targeted $select, classifies each entry, and submits one full-sync batch per principalType (required because the ingest API's scoped delete takes one value at a time). Extra SP-specific fields (appId, publisherName, tags, etc.) go into extendedAttributes so they surface in the new ext.* filter dropdowns. New -AINamePatterns parameter lets operators extend the classifier with tenant-specific naming conventions. - app/api/src/routes/jobs.js — exposes 'servicePrincipals' as a selectable object type in the crawler wizard; Application.Read.All and Directory.Read.All are the gating permissions. - setup/docker/Invoke-CrawlerJob.ps1 — maps selectedObjects .servicePrincipals and the optional aiNamePatterns config onto the crawler parameters. - test/unit/IdentityAtlas.Tests.ps1 — seven Pester tests pinning the classification rules, including priority ordering (MI wins over AI tags), word-boundary guards on 'gpt' / 'bot' to avoid false positives like "GPTools", and custom-pattern support. Verified with Pester (145/145) and Vitest (112/112). Co-Authored-By: Claude Opus 4.7 (1M context) --- app/api/src/routes/jobs.js | 5 +- changes/feature-entraid-service-principals.md | 3 + setup/docker/Invoke-CrawlerJob.ps1 | 4 + test/unit/IdentityAtlas.Tests.ps1 | 84 +++++++++++++++++- .../entra-id/Start-EntraIDCrawler.ps1 | 86 +++++++++++++++++++ .../helpers/Get-FGServicePrincipalType.ps1 | 72 ++++++++++++++++ 6 files changed, 251 insertions(+), 3 deletions(-) create mode 100644 changes/feature-entraid-service-principals.md create mode 100644 tools/powershell-sdk/helpers/Get-FGServicePrincipalType.ps1 diff --git a/app/api/src/routes/jobs.js b/app/api/src/routes/jobs.js index b0fb9f347..3be978226 100644 --- a/app/api/src/routes/jobs.js +++ b/app/api/src/routes/jobs.js @@ -76,8 +76,8 @@ const PERMISSION_OBJECT_MAP = { 'User.Read.All': ['identity', 'context', 'usersGroupsMembers'], 'Group.Read.All': ['usersGroupsMembers'], 'GroupMember.Read.All': ['usersGroupsMembers'], - 'Directory.Read.All': ['directoryRoles'], - 'Application.Read.All': ['appsAppRoles'], + 'Directory.Read.All': ['directoryRoles', 'servicePrincipals'], + 'Application.Read.All': ['appsAppRoles', 'servicePrincipals'], 'PrivilegedEligibilitySchedule.Read.AzureADGroup': ['pim'], 'EntitlementManagement.Read.All': ['identityGovernance'], 'AccessReview.Read.All': ['identityGovernance'], @@ -91,6 +91,7 @@ const ENTRA_OBJECT_TYPES = [ { key: 'identity', label: 'Identity', description: 'Personal user accounts that are synced from HR' }, { key: 'context', label: 'Context', description: 'Auto-detected organizational structure from identity data' }, { key: 'usersGroupsMembers', label: 'Users & Groups & Members', description: 'All users, security groups, and group memberships' }, + { key: 'servicePrincipals', label: 'Service Principals', description: 'Non-human identities (enterprise app SPs, managed identities, AI agents)' }, { key: 'identityGovernance', label: 'Identity Governance', description: 'Access Packages, assignments, policies, reviews' }, { key: 'appsAppRoles', label: 'Apps & AppRoles', description: 'Application registrations and role assignments' }, { key: 'directoryRoles', label: 'Directory Roles', description: 'Entra ID directory role assignments' }, diff --git a/changes/feature-entraid-service-principals.md b/changes/feature-entraid-service-principals.md new file mode 100644 index 000000000..04f840980 --- /dev/null +++ b/changes/feature-entraid-service-principals.md @@ -0,0 +1,3 @@ +- Added Service Principals to the Entra ID crawler. When the "Service Principals" object type is selected, the crawler now pulls all Entra service principals (enterprise apps, managed identities, AI agents) and writes them to the Principals table alongside user accounts — unblocking future Azure RM role-assignment imports that reference these identities. +- New classification helper (`Get-FGServicePrincipalType`) tags each service principal as `ManagedIdentity`, `AIAgent`, or `ServicePrincipal` based on Graph's `servicePrincipalType`, well-known Microsoft AI platform tags (CopilotStudio, PowerVirtualAgents, AzureOpenAI, CognitiveServices), and display-name heuristics. Custom AI-name patterns can be supplied per crawler config via `aiNamePatterns`. +- Extra SP metadata (`appId`, `servicePrincipalType`, `publisherName`, `homepage`, `tags`, `servicePrincipalNames`, `notes`) is stored in `extendedAttributes` so it shows up in the Users filter dropdown and detail pages. diff --git a/setup/docker/Invoke-CrawlerJob.ps1 b/setup/docker/Invoke-CrawlerJob.ps1 index fa754a3be..98ba46a1e 100644 --- a/setup/docker/Invoke-CrawlerJob.ps1 +++ b/setup/docker/Invoke-CrawlerJob.ps1 @@ -131,6 +131,7 @@ switch ($JobType) { $crawlerParams['SyncResources'] = [bool]$objects['usersGroupsMembers'] $crawlerParams['SyncAssignments'] = [bool]$objects['usersGroupsMembers'] } + if ($objects.ContainsKey('servicePrincipals')) { $crawlerParams['SyncServicePrincipals'] = [bool]$objects['servicePrincipals'] } if ($objects.ContainsKey('identityGovernance')) { $crawlerParams['SyncGovernance'] = [bool]$objects['identityGovernance'] } if ($objects.ContainsKey('context')) { $crawlerParams['SyncContexts'] = [bool]$objects['context'] } if ($objects.ContainsKey('pim')) { $crawlerParams['SyncPim'] = [bool]$objects['pim'] } @@ -155,6 +156,9 @@ switch ($JobType) { if ($Config['customGroupAttributes']) { $crawlerParams['CustomGroupAttributes'] = @($Config['customGroupAttributes']) } + if ($Config['aiNamePatterns']) { + $crawlerParams['AINamePatterns'] = @($Config['aiNamePatterns']) + } # Identity filter if ($Config['identityFilter'] -and $Config['identityFilter']['attribute']) { diff --git a/test/unit/IdentityAtlas.Tests.ps1 b/test/unit/IdentityAtlas.Tests.ps1 index d21f83c61..e207e61a6 100644 --- a/test/unit/IdentityAtlas.Tests.ps1 +++ b/test/unit/IdentityAtlas.Tests.ps1 @@ -88,12 +88,94 @@ Describe 'Function Availability — Helpers (idempotent)' { It 'exports <_>' -ForEach @( 'Confirm-FGUser', 'Confirm-FGGroup', 'Confirm-FGGroupMember', 'Confirm-FGNotGroupMember', 'Confirm-FGAccessPackage', 'Confirm-FGAccessPackagePolicy', 'Confirm-FGAccessPackageResource', - 'Confirm-FGCatalog', 'Confirm-FGGroupInCatalog' + 'Confirm-FGCatalog', 'Confirm-FGGroupInCatalog', + 'Get-FGServicePrincipalType' ) { Get-Command $_ -ErrorAction SilentlyContinue | Should -Not -BeNullOrEmpty } } +# ─── Get-FGServicePrincipalType ─────────────────────────────────── +# Tests pin the classification taxonomy from CLAUDE.md. Any change to the +# ordering (e.g. Managed Identity must win over tag-based AI detection) needs +# a corresponding change here; otherwise crawler output silently shifts +# principalType labels and breaks risk-scoring heuristics downstream. +Describe 'Get-FGServicePrincipalType — classification rules' { + It 'classifies servicePrincipalType=ManagedIdentity as ManagedIdentity (even when tags look AI)' { + # Rule 1 is authoritative: MI must win over tag-based AI detection. + $sp = [pscustomobject]@{ + displayName = 'Copilot ghost tenant' + servicePrincipalType = 'ManagedIdentity' + tags = @('AzureOpenAI') + } + Get-FGServicePrincipalType -ServicePrincipal $sp | Should -Be 'ManagedIdentity' + } + + It 'classifies AI platform tags as AIAgent' { + foreach ($tag in @('CopilotStudio','PowerVirtualAgents','AzureOpenAI','CognitiveServices')) { + $sp = [pscustomobject]@{ + displayName = 'benign-sounding-sp' + servicePrincipalType = 'Application' + tags = @('SomeOtherTag', $tag) + } + Get-FGServicePrincipalType -ServicePrincipal $sp | + Should -Be 'AIAgent' -Because "tag '$tag' must trigger AIAgent" + } + } + + It 'does not match a displayName fragment against a tag-like unrelated name' { + # 'gptools' contains 'gpt' as substring but not as a word — the built-in + # pattern uses \bgpt\b. This guards against false positives on things + # like "GitOps Toolkit". + $sp = [pscustomobject]@{ + displayName = 'GPTools Support' + servicePrincipalType = 'Application' + tags = @() + } + Get-FGServicePrincipalType -ServicePrincipal $sp | Should -Be 'ServicePrincipal' + } + + It 'classifies AI displayNames as AIAgent (case-insensitive)' { + foreach ($name in @('Microsoft Copilot', 'my-OpenAI-proxy', 'Team Bot', 'GPT Assistant')) { + $sp = [pscustomobject]@{ + displayName = $name + servicePrincipalType = 'Application' + tags = @() + } + Get-FGServicePrincipalType -ServicePrincipal $sp | + Should -Be 'AIAgent' -Because "displayName '$name' should trigger AIAgent" + } + } + + It 'honours caller-supplied AINamePatterns' { + $sp = [pscustomobject]@{ + displayName = 'acme-agent-service' + servicePrincipalType = 'Application' + tags = @() + } + Get-FGServicePrincipalType -ServicePrincipal $sp -AINamePatterns @('acme-agent-') | + Should -Be 'AIAgent' + } + + It 'returns ServicePrincipal for an ordinary enterprise app' { + $sp = [pscustomobject]@{ + displayName = 'Jira Integration' + servicePrincipalType = 'Application' + tags = @('WindowsAzureActiveDirectoryIntegratedApp') + } + Get-FGServicePrincipalType -ServicePrincipal $sp | Should -Be 'ServicePrincipal' + } + + It 'handles an SP with no tags and no displayName gracefully' { + $sp = [pscustomobject]@{ + displayName = $null + servicePrincipalType = 'Application' + tags = $null + } + Get-FGServicePrincipalType -ServicePrincipal $sp | Should -Be 'ServicePrincipal' + } +} + Describe 'Function Availability — RiskScoring (v5 stubs)' { # In v5 these are stub functions that print a "not yet implemented" warning. # They still need to be exported so the module loads cleanly. diff --git a/tools/crawlers/entra-id/Start-EntraIDCrawler.ps1 b/tools/crawlers/entra-id/Start-EntraIDCrawler.ps1 index 428d6f942..42783b5bd 100644 --- a/tools/crawlers/entra-id/Start-EntraIDCrawler.ps1 +++ b/tools/crawlers/entra-id/Start-EntraIDCrawler.ps1 @@ -71,6 +71,11 @@ Param( # Custom group attributes to include in the sync (added to $select) [string[]]$CustomGroupAttributes = @(), + # Extra regex fragments applied to servicePrincipal.displayName to flag an + # SP as AIAgent. Combined with the built-in list ('copilot', 'openai', etc). + # Case-insensitive; use \b word boundaries if exactness matters. + [string[]]$AINamePatterns = @(), + # Identity filter: select which users are treated as identities # Format: @{ attribute='employeeId'; condition='isNotNull' } # or: @{ attribute='employeeType'; condition='equals'; value='Employee' } @@ -604,6 +609,87 @@ if ($SyncPrincipals) { } } +# ─── Sync Service Principals ───────────────────────────────────── +# Service principals are Entra ID's non-human identities — enterprise-app SPs, +# managed identities, AI agents (Copilot Studio / Azure OpenAI), etc. They own +# a large fraction of role assignments in Azure and M365, so we want them in +# the `Principals` table alongside human users. +# +# We classify each SP into one of the schema's principalType values via +# Get-FGServicePrincipalType (from tools/powershell-sdk/helpers). Each class +# gets its own full-sync batch because the ingest API's scoped-delete works +# on exactly one principalType value at a time. +if ($SyncServicePrincipals) { + Write-Host "`n[$(Get-Date -Format 'HH:mm:ss')] Syncing service principals..." -ForegroundColor Cyan + Update-CrawlerProgress -Step 'Syncing service principals' -Pct 18 -Detail 'Fetching from Microsoft Graph...' + + # `tags` and `servicePrincipalType` drive classification; `appId`, + # `appOwnerOrganizationId`, and `notes` go into extendedAttributes for + # downstream visibility. `accountEnabled` lives in its dedicated column. + $spSelectAttrs = @( + 'id','appId','displayName','servicePrincipalType','accountEnabled', + 'tags','appOwnerOrganizationId','createdDateTime','notes', + 'servicePrincipalNames','homepage','publisherName' + ) + $spSelect = $spSelectAttrs -join ',' + $sps = Invoke-FGGetRequest -URI "https://graph.microsoft.com/beta/servicePrincipals?`$select=$spSelect&`$top=999" + Update-CrawlerProgress -Detail "Classifying $($sps.Count) service principals..." + + # Bucket records by principalType so we can submit one scoped full-sync + # per type. An empty bucket is skipped entirely to avoid an unintended + # delete-everything-of-that-type against the DB. + $buckets = @{ + ServicePrincipal = New-Object System.Collections.ArrayList + ManagedIdentity = New-Object System.Collections.ArrayList + AIAgent = New-Object System.Collections.ArrayList + } + + foreach ($sp in $sps) { + $pt = Get-FGServicePrincipalType -ServicePrincipal $sp -AINamePatterns $AINamePatterns + + $rec = @{ + id = $sp.id + displayName = $sp.displayName + accountEnabled = [bool]$sp.accountEnabled + principalType = $pt + } + if ($sp.createdDateTime) { $rec['createdDateTime'] = $sp.createdDateTime } + + # Everything that isn't a first-class column but is useful for filters + # or risk signals lives in extendedAttributes. We stringify arrays + # (tags, servicePrincipalNames) because jsonb_typeof filters arrays out + # of the filter-dropdown discovery and a comma-joined string keeps the + # key discoverable. + $ext = @{} + if ($sp.appId) { $ext['appId'] = $sp.appId } + if ($sp.servicePrincipalType) { $ext['servicePrincipalType'] = $sp.servicePrincipalType } + if ($sp.appOwnerOrganizationId) { $ext['appOwnerOrganizationId'] = $sp.appOwnerOrganizationId } + if ($sp.publisherName) { $ext['publisherName'] = $sp.publisherName } + if ($sp.homepage) { $ext['homepage'] = $sp.homepage } + if ($sp.notes) { $ext['notes'] = $sp.notes } + if ($sp.tags -and $sp.tags.Count -gt 0) { + $ext['tags'] = ($sp.tags -join ',') + } + if ($sp.servicePrincipalNames -and $sp.servicePrincipalNames.Count -gt 0) { + $ext['servicePrincipalNames'] = ($sp.servicePrincipalNames -join ',') + } + if ($ext.Count -gt 0) { $rec['extendedAttributes'] = $ext } + + [void]$buckets[$pt].Add($rec) + } + + Write-Host (" Classified: {0} ServicePrincipal / {1} ManagedIdentity / {2} AIAgent" -f ` + $buckets.ServicePrincipal.Count, $buckets.ManagedIdentity.Count, $buckets.AIAgent.Count) -ForegroundColor Gray + + foreach ($pt in @('ServicePrincipal','ManagedIdentity','AIAgent')) { + $bucket = $buckets[$pt] + if ($bucket.Count -eq 0) { continue } + Update-CrawlerProgress -Detail "Uploading $($bucket.Count) $pt records..." + Send-IngestBatch -Endpoint 'ingest/principals' -SystemId $systemId -SyncMode 'full' ` + -Scope @{ principalType = $pt } -Records @($bucket) + } +} + # ─── Sync Resources (Groups) ───────────────────────────────────── if ($SyncResources) { Write-Host "`n[$(Get-Date -Format 'HH:mm:ss')] Syncing resources (groups)..." -ForegroundColor Cyan diff --git a/tools/powershell-sdk/helpers/Get-FGServicePrincipalType.ps1 b/tools/powershell-sdk/helpers/Get-FGServicePrincipalType.ps1 new file mode 100644 index 000000000..76adbb9ff --- /dev/null +++ b/tools/powershell-sdk/helpers/Get-FGServicePrincipalType.ps1 @@ -0,0 +1,72 @@ +function Get-FGServicePrincipalType { + <# + .SYNOPSIS + Classifies an Entra ID service principal into one of the principalType + values the Identity Atlas schema understands. + + .DESCRIPTION + Implements the detection taxonomy documented in CLAUDE.md. Applied in + priority order: + + 1. servicePrincipalType = 'ManagedIdentity' -> ManagedIdentity + 2. Tag contains one of the well-known AI platform markers + (CopilotStudio, PowerVirtualAgents, AzureOpenAI, CognitiveServices) + -> AIAgent + 3. displayName matches a built-in AI name heuristic or a caller- + supplied custom pattern -> AIAgent + 4. Default -> ServicePrincipal + + WorkloadIdentity (federated credentials) is intentionally out of scope + here because it can't be decided from a servicePrincipal object alone. + + .PARAMETER ServicePrincipal + The Graph service principal object (as returned by + /beta/servicePrincipals). Must have at least `servicePrincipalType`, + `tags`, and `displayName` fields populated if the caller wants + accurate classification. + + .PARAMETER AINamePatterns + Optional extra regex fragments to treat as AI-agent indicators. Matched + case-insensitively against displayName. Callers with domain-specific + naming ("pwc-bot-", "svc_ai_") pass them here. + + .OUTPUTS + [string] — one of: 'ManagedIdentity', 'AIAgent', 'ServicePrincipal' + #> + [CmdletBinding()] + [OutputType([string])] + Param( + [Parameter(Mandatory = $true)] + $ServicePrincipal, + + [Parameter(Mandatory = $false)] + [string[]]$AINamePatterns = @() + ) + + # Rule 1 — Managed Identity is authoritative: Graph tells us directly. + if ($ServicePrincipal.servicePrincipalType -eq 'ManagedIdentity') { + return 'ManagedIdentity' + } + + # Rule 2 — Well-known AI platform tags that Microsoft stamps on SPs. + # Keep this list narrow; speculative additions produce false positives that + # then propagate into risk scoring. + $AIPlatformTags = @('CopilotStudio', 'PowerVirtualAgents', 'AzureOpenAI', 'CognitiveServices') + if ($ServicePrincipal.tags) { + foreach ($t in $ServicePrincipal.tags) { + if ($AIPlatformTags -contains $t) { return 'AIAgent' } + } + } + + # Rule 3 — Name heuristics. Only applied if displayName is non-empty. + if ($ServicePrincipal.displayName) { + $builtInPatterns = @('copilot', 'openai', 'azure-ai', 'cognitive-service', '\bgpt\b', '\bbot\b') + $allPatterns = @($builtInPatterns) + @($AINamePatterns) + foreach ($pattern in $allPatterns) { + if ([string]::IsNullOrWhiteSpace($pattern)) { continue } + if ($ServicePrincipal.displayName -match "(?i)$pattern") { return 'AIAgent' } + } + } + + return 'ServicePrincipal' +} From 2fdc24b19b706c45a33a83a0499855e59f5f17b4 Mon Sep 17 00:00:00 2001 From: Wim van den Heijkant Date: Sat, 18 Apr 2026 16:03:41 +0200 Subject: [PATCH 014/160] Detect Entra Agent ID SPs as AIAgent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Microsoft Entra Agent ID (GA 2025) surfaces AI agents as service principals stamped with specific tags. Before this change our SP classifier missed them — Entra Agent ID-managed SPs would have landed in Principals as generic ServicePrincipal, losing the signal that they're non-human AI identities. Adds three markers to Get-FGServicePrincipalType: - AgenticInstance (exact tag — individual agent instance) - AgenticApp (exact tag — agentic application) - power-virtual-agents-* (prefix — PVA stamps per-instance GUIDs) Pester coverage extended with two new cases pinning the exact-tag path for AgenticInstance / AgenticApp and the prefix-match path for Power Virtual Agents. 9/9 classifier tests pass. No Graph endpoint changes — Entra Agent ID is already reachable via the existing /servicePrincipals fetch. Co-Authored-By: Claude Opus 4.7 (1M context) --- changes/feature-entraid-service-principals.md | 2 +- test/unit/IdentityAtlas.Tests.ps1 | 27 +++++++++++++++++++ .../helpers/Get-FGServicePrincipalType.ps1 | 15 ++++++++++- 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/changes/feature-entraid-service-principals.md b/changes/feature-entraid-service-principals.md index 04f840980..3dd7013b2 100644 --- a/changes/feature-entraid-service-principals.md +++ b/changes/feature-entraid-service-principals.md @@ -1,3 +1,3 @@ - Added Service Principals to the Entra ID crawler. When the "Service Principals" object type is selected, the crawler now pulls all Entra service principals (enterprise apps, managed identities, AI agents) and writes them to the Principals table alongside user accounts — unblocking future Azure RM role-assignment imports that reference these identities. -- New classification helper (`Get-FGServicePrincipalType`) tags each service principal as `ManagedIdentity`, `AIAgent`, or `ServicePrincipal` based on Graph's `servicePrincipalType`, well-known Microsoft AI platform tags (CopilotStudio, PowerVirtualAgents, AzureOpenAI, CognitiveServices), and display-name heuristics. Custom AI-name patterns can be supplied per crawler config via `aiNamePatterns`. +- New classification helper (`Get-FGServicePrincipalType`) tags each service principal as `ManagedIdentity`, `AIAgent`, or `ServicePrincipal` based on Graph's `servicePrincipalType`, well-known Microsoft AI platform tags (CopilotStudio, PowerVirtualAgents, AzureOpenAI, CognitiveServices), the Entra Agent ID markers (`AgenticInstance`, `AgenticApp`, `power-virtual-agents-*`), and display-name heuristics. Custom AI-name patterns can be supplied per crawler config via `aiNamePatterns`. - Extra SP metadata (`appId`, `servicePrincipalType`, `publisherName`, `homepage`, `tags`, `servicePrincipalNames`, `notes`) is stored in `extendedAttributes` so it shows up in the Users filter dropdown and detail pages. diff --git a/test/unit/IdentityAtlas.Tests.ps1 b/test/unit/IdentityAtlas.Tests.ps1 index e207e61a6..0a6ce3f59 100644 --- a/test/unit/IdentityAtlas.Tests.ps1 +++ b/test/unit/IdentityAtlas.Tests.ps1 @@ -123,6 +123,33 @@ Describe 'Get-FGServicePrincipalType — classification rules' { } } + It 'classifies Entra Agent ID tags (AgenticInstance, AgenticApp) as AIAgent' { + # Entra Agent ID (GA 2025) stamps SPs with these exact tags. These + # identities are first-class AI agents and must not be left as generic + # ServicePrincipal — risk scoring and UX both depend on the distinction. + foreach ($tag in @('AgenticInstance','AgenticApp')) { + $sp = [pscustomobject]@{ + displayName = 'some-agent-123' + servicePrincipalType = 'Application' + tags = @('WindowsAzureActiveDirectoryIntegratedApp', $tag) + } + Get-FGServicePrincipalType -ServicePrincipal $sp | + Should -Be 'AIAgent' -Because "Entra Agent ID tag '$tag' must trigger AIAgent" + } + } + + It 'classifies Power Virtual Agents tag prefix as AIAgent' { + # PVA stamps per-instance tags of the form `power-virtual-agents-` + # — we prefix-match because matching one-GUID-per-tag in a fixed list + # obviously doesn't work. + $sp = [pscustomobject]@{ + displayName = 'Copilot Studio flow host' + servicePrincipalType = 'Application' + tags = @('power-virtual-agents-3fa85f64-5717-4562-b3fc-2c963f66afa6') + } + Get-FGServicePrincipalType -ServicePrincipal $sp | Should -Be 'AIAgent' + } + It 'does not match a displayName fragment against a tag-like unrelated name' { # 'gptools' contains 'gpt' as substring but not as a word — the built-in # pattern uses \bgpt\b. This guards against false positives on things diff --git a/tools/powershell-sdk/helpers/Get-FGServicePrincipalType.ps1 b/tools/powershell-sdk/helpers/Get-FGServicePrincipalType.ps1 index 76adbb9ff..3c18a7fc1 100644 --- a/tools/powershell-sdk/helpers/Get-FGServicePrincipalType.ps1 +++ b/tools/powershell-sdk/helpers/Get-FGServicePrincipalType.ps1 @@ -51,10 +51,23 @@ function Get-FGServicePrincipalType { # Rule 2 — Well-known AI platform tags that Microsoft stamps on SPs. # Keep this list narrow; speculative additions produce false positives that # then propagate into risk scoring. - $AIPlatformTags = @('CopilotStudio', 'PowerVirtualAgents', 'AzureOpenAI', 'CognitiveServices') + # + # The exact-match list covers classic AI-related platform tags plus the + # Entra Agent ID markers introduced in 2025 (AgenticInstance, AgenticApp). + # Power Virtual Agents stamps a per-instance tag of the form + # `power-virtual-agents-`, so PVA is detected via prefix match. + $AIPlatformTags = @( + 'CopilotStudio', 'PowerVirtualAgents', 'AzureOpenAI', 'CognitiveServices', + 'AgenticInstance', 'AgenticApp' + ) + $AIPlatformTagPrefixes = @('power-virtual-agents-') if ($ServicePrincipal.tags) { foreach ($t in $ServicePrincipal.tags) { + if (-not $t) { continue } if ($AIPlatformTags -contains $t) { return 'AIAgent' } + foreach ($prefix in $AIPlatformTagPrefixes) { + if ($t.StartsWith($prefix)) { return 'AIAgent' } + } } } From ffaea49894306738c37d5403df1a6d63ec029554 Mon Sep 17 00:00:00 2001 From: Wim van den Heijkant Date: Sat, 18 Apr 2026 20:10:48 +0200 Subject: [PATCH 015/160] Add principalType sub-tabs to the Users page MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After enabling Service Principal sync the Principals table balloons — in local testing a mid-size tenant goes from ~4.5k users to 5.5k+ rows with SPs and managed identities mixed in. The Users page became hard to use because everything was in one list. Adds a sub-tab bar above the Tags / Filters rows with five options: All / Users / Service Principals / Managed Identities / AI Agents. Selecting a tab narrows the view to that principalType; 'All' leaves the list unfiltered. - useEntityPage gains a `baseFilters` option. Page-level filters are merged into the API `filtersObj` alongside the user-driven `activeFilters`, and page/selection reset when they change. User filters still win on key collision. - UsersPage passes `{ principalType: }` as baseFilters when the tab is not 'all', hides `principalType` from the ordinary filter dropdown to keep a single source of truth, and persists the tab in the URL hash (`#users?type=ServicePrincipal`) so reloads and deep links keep the same view — the same pattern the Admin page uses. - The new tab bar matches the Admin page's underlined-pill style. Verified against the live stack with a real tenant: All 1085 User 47 ServicePrincipal 981 AIAgent 50 ManagedIdentity 7 Co-Authored-By: Claude Opus 4.7 (1M context) --- app/ui/src/components/UsersPage.jsx | 74 ++++++++++++++++++- app/ui/src/hooks/useEntityPage.js | 22 ++++-- changes/feature-entraid-service-principals.md | 1 + 3 files changed, 89 insertions(+), 8 deletions(-) diff --git a/app/ui/src/components/UsersPage.jsx b/app/ui/src/components/UsersPage.jsx index 66c923f54..ebe588b09 100644 --- a/app/ui/src/components/UsersPage.jsx +++ b/app/ui/src/components/UsersPage.jsx @@ -1,4 +1,4 @@ -import { useMemo } from 'react'; +import { useMemo, useState, useEffect } from 'react'; import { useAuth } from '../auth/AuthGate'; import useEntityPage from '../hooks/useEntityPage'; import FilterBar from './FilterBar'; @@ -28,8 +28,51 @@ const TABLE_COLUMNS = [ { key: 'jobTitle', label: 'Job Title' }, ]; +// Sub-tabs for principalType. The Principals table is a universal identity +// store, so the "Users" page now lists more than just humans — splitting it +// by type keeps each view manageable when SP/MI/AIAgent sync is enabled. +// The tab label is what's shown; the value is matched against the column. +// 'all' is a sentinel for "no filter". +const PRINCIPAL_TYPE_TABS = [ + { key: 'all', label: 'All' }, + { key: 'User', label: 'Users' }, + { key: 'ServicePrincipal', label: 'Service Principals' }, + { key: 'ManagedIdentity', label: 'Managed Identities' }, + { key: 'AIAgent', label: 'AI Agents' }, +]; + +// Read/write the active sub-tab from the URL hash (?type=User on the users +// route). This keeps deep links working across reload, matching the pattern +// the Admin page uses for its own sub-tabs. +function readTypeFromHash() { + const hash = window.location.hash.replace('#', ''); + const q = hash.indexOf('?'); + const params = new URLSearchParams(q >= 0 ? hash.substring(q + 1) : ''); + const t = params.get('type'); + return t && PRINCIPAL_TYPE_TABS.some(tab => tab.key === t) ? t : 'all'; +} + +function writeTypeToHash(tab) { + const hash = window.location.hash.replace('#', ''); + const q = hash.indexOf('?'); + const page = q >= 0 ? hash.substring(0, q) : hash; + const params = new URLSearchParams(q >= 0 ? hash.substring(q + 1) : ''); + if (tab === 'all') params.delete('type'); else params.set('type', tab); + const qs = params.toString(); + window.history.replaceState(null, '', `#${page}${qs ? '?' + qs : ''}`); +} + export default function UsersPage({ onOpenDetail }) { const { authFetch } = useAuth(); + const [activeTypeTab, setActiveTypeTab] = useState(readTypeFromHash); + + useEffect(() => { writeTypeToHash(activeTypeTab); }, [activeTypeTab]); + + // Memoise so useEntityPage's filtersObj memo isn't busted every render. + const baseFilters = useMemo( + () => (activeTypeTab === 'all' ? null : { principalType: activeTypeTab }), + [activeTypeTab], + ); const ep = useEntityPage({ authFetch, @@ -37,9 +80,16 @@ export default function UsersPage({ onOpenDetail }) { listEndpoint: '/api/users', columnsEndpoint: '/api/user-columns-page', tagFilterKey: '__userTag', + baseFilters, }); - const filterFields = useMemo(() => ep.getFilterFields(FIELD_LABELS), [ep]); + // Hide `principalType` from the Filters dropdown — the sub-tabs control it. + // Leaving it in would create two ways to set the same value and surprise + // users when the two disagree. + const filterFields = useMemo( + () => ep.getFilterFields(FIELD_LABELS).filter(f => f.key !== 'principalType'), + [ep], + ); return (
@@ -49,6 +99,26 @@ export default function UsersPage({ onOpenDetail }) { {ep.total.toLocaleString()} total
+ {/* Principal-type sub-tabs. Matches the underlined-pills style used by + the Admin page's own sub-tab bar so the UX is consistent. */} +
+ +
+ {/* Tag management bar */}
Tags: diff --git a/app/ui/src/hooks/useEntityPage.js b/app/ui/src/hooks/useEntityPage.js index dcabf4751..c70f2cf5f 100644 --- a/app/ui/src/hooks/useEntityPage.js +++ b/app/ui/src/hooks/useEntityPage.js @@ -14,8 +14,11 @@ const PAGE_SIZE = 100; * @param {string} options.listEndpoint - API endpoint for entity list (e.g., '/api/users') * @param {string} options.columnsEndpoint - API endpoint for column discovery * @param {string} options.tagFilterKey - Key for tag filter (e.g., '__userTag' or '__groupTag') + * @param {object} [options.baseFilters] - Page-level filters always applied on top of user-driven ones + * (e.g. the Users-page principalType sub-tab). Keys with null/empty values are dropped. + * The caller is expected to memoise this object so its identity is stable per intended value. */ -export default function useEntityPage({ authFetch, entityType, listEndpoint, columnsEndpoint, tagFilterKey }) { +export default function useEntityPage({ authFetch, entityType, listEndpoint, columnsEndpoint, tagFilterKey, baseFilters }) { // Data state const [items, setItems] = useState([]); const [total, setTotal] = useState(0); @@ -51,7 +54,7 @@ export default function useEntityPage({ authFetch, entityType, listEndpoint, col const fetchVersion = useRef(0); // Reset page & selection when filters change - useEffect(() => { setPage(0); setSelected(new Set()); }, [debouncedSearch, activeFilters]); + useEffect(() => { setPage(0); setSelected(new Set()); }, [debouncedSearch, activeFilters, baseFilters]); // Fetch available columns for filter dropdowns useEffect(() => { @@ -74,11 +77,18 @@ export default function useEntityPage({ authFetch, entityType, listEndpoint, col useEffect(() => { fetchTags(); }, [fetchTags]); - // Build filters object for API + // Build filters object for API: merge user-driven `activeFilters` on top + // of page-level `baseFilters` (e.g. the Users-page principalType tab). User + // filters win on key collision — tabs aren't expected to overlap with + // user-configurable fields, but if they do, the explicit action wins. const filtersObj = useMemo(() => { - if (activeFilters.length === 0) return null; - return Object.fromEntries(activeFilters.map(f => [f.field, f.value])); - }, [activeFilters]); + const base = Object.fromEntries( + Object.entries(baseFilters || {}).filter(([, v]) => v != null && v !== '') + ); + const fromActive = Object.fromEntries(activeFilters.map(f => [f.field, f.value])); + const merged = { ...base, ...fromActive }; + return Object.keys(merged).length ? merged : null; + }, [activeFilters, baseFilters]); // Fetch items const fetchItems = useCallback(async () => { diff --git a/changes/feature-entraid-service-principals.md b/changes/feature-entraid-service-principals.md index 3dd7013b2..a3c7f5384 100644 --- a/changes/feature-entraid-service-principals.md +++ b/changes/feature-entraid-service-principals.md @@ -1,3 +1,4 @@ - Added Service Principals to the Entra ID crawler. When the "Service Principals" object type is selected, the crawler now pulls all Entra service principals (enterprise apps, managed identities, AI agents) and writes them to the Principals table alongside user accounts — unblocking future Azure RM role-assignment imports that reference these identities. - New classification helper (`Get-FGServicePrincipalType`) tags each service principal as `ManagedIdentity`, `AIAgent`, or `ServicePrincipal` based on Graph's `servicePrincipalType`, well-known Microsoft AI platform tags (CopilotStudio, PowerVirtualAgents, AzureOpenAI, CognitiveServices), the Entra Agent ID markers (`AgenticInstance`, `AgenticApp`, `power-virtual-agents-*`), and display-name heuristics. Custom AI-name patterns can be supplied per crawler config via `aiNamePatterns`. - Extra SP metadata (`appId`, `servicePrincipalType`, `publisherName`, `homepage`, `tags`, `servicePrincipalNames`, `notes`) is stored in `extendedAttributes` so it shows up in the Users filter dropdown and detail pages. +- Added principal-type sub-tabs to the Users page ("All / Users / Service Principals / Managed Identities / AI Agents") so the list stays navigable after an SP sync adds thousands of non-human identities. The active tab is preserved in the URL hash (`#users?type=AIAgent`). From 6c326e9d9083404bed32ee245467cf14210e7fd9 Mon Sep 17 00:00:00 2001 From: Wim van den Heijkant Date: Sat, 18 Apr 2026 20:22:53 +0200 Subject: [PATCH 016/160] Show principalType filter on the 'All' users tab MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit My previous commit hid principalType from the filter dropdown unconditionally. That's correct when a specific tab is active (the tab is the authoritative selector) but wrong on 'All' — where no type is pinned and users rightly expect to be able to add it as a regular filter. The hide now applies only on non-'All' tabs. Co-Authored-By: Claude Opus 4.7 (1M context) --- app/ui/src/components/UsersPage.jsx | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/app/ui/src/components/UsersPage.jsx b/app/ui/src/components/UsersPage.jsx index ebe588b09..777a90d50 100644 --- a/app/ui/src/components/UsersPage.jsx +++ b/app/ui/src/components/UsersPage.jsx @@ -83,12 +83,16 @@ export default function UsersPage({ onOpenDetail }) { baseFilters, }); - // Hide `principalType` from the Filters dropdown — the sub-tabs control it. - // Leaving it in would create two ways to set the same value and surprise - // users when the two disagree. + // Hide `principalType` from the Filters dropdown only when a specific + // sub-tab is active — the tab is the authoritative selector there, and + // having both would create two ways to set the same value. On the "All" + // tab no type is pinned, so leave `principalType` available as a regular + // filter option. const filterFields = useMemo( - () => ep.getFilterFields(FIELD_LABELS).filter(f => f.key !== 'principalType'), - [ep], + () => ep.getFilterFields(FIELD_LABELS).filter(f => + !(activeTypeTab !== 'all' && f.key === 'principalType') + ), + [ep, activeTypeTab], ); return ( From 8c0903cb3c7f2fd9dfb3b3ada26426dfaab9bcec Mon Sep 17 00:00:00 2001 From: Wim van den Heijkant Date: Sat, 18 Apr 2026 20:40:30 +0200 Subject: [PATCH 017/160] Raise ingest body limit to 50MB MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The crawler chunks at 5,000 records per batch. When extendedAttributes is populated — especially on Service Principal batches (appId, tags, servicePrincipalNames, publisherName, homepage, etc.) — a single batch commonly lands between 15 and 30 MB. The previous 10 MB cap aborted the crawl with HTTP 413 on any real-world SP-enabled tenant. 50 MB gives ~5x headroom over observed sizes without losing a sane upper bound. Record count is still independently capped at 50,000 by the ingest validator. The crawler's error line now also prints the payload size so the next 413 (or any other failure) surfaces the context needed to diagnose whether we're close to the limit again. Co-Authored-By: Claude Opus 4.7 (1M context) --- app/api/src/index.js | 7 ++++++- tools/crawlers/entra-id/Start-EntraIDCrawler.ps1 | 3 ++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/app/api/src/index.js b/app/api/src/index.js index 637ef04ed..ea9f47e49 100644 --- a/app/api/src/index.js +++ b/app/api/src/index.js @@ -255,7 +255,12 @@ app.use('/api', authMiddleware, jobsRouter); // Crawler self-service (API key auth) — /api/crawlers/whoami, /api/crawlers/rotate app.use('/api', crawlerAuthMiddleware, selfServiceCrawlersRouter); // Ingest endpoints (API key auth) — /api/ingest/* -app.use('/api/ingest', express.json({ limit: '10mb' })); // larger limit for ingest payloads +// Ingest body size cap. Crawler chunks at 5,000 records per batch; with +// extendedAttributes populated (SPs in particular carry appId, tags, +// servicePrincipalNames, publisherName, etc.) a typical batch can reach +// 20-30 MB. 50 MB gives ~5x headroom over real-world observed sizes while +// still keeping a sane upper bound on memory use per request. +app.use('/api/ingest', express.json({ limit: '50mb' })); app.use('/api', crawlerAuthMiddleware, ingestRouter); // In production, serve the frontend build output diff --git a/tools/crawlers/entra-id/Start-EntraIDCrawler.ps1 b/tools/crawlers/entra-id/Start-EntraIDCrawler.ps1 index 42783b5bd..7b7e09670 100644 --- a/tools/crawlers/entra-id/Start-EntraIDCrawler.ps1 +++ b/tools/crawlers/entra-id/Start-EntraIDCrawler.ps1 @@ -147,7 +147,8 @@ function Invoke-IngestAPI { continue } - Write-Host " ERROR: $Endpoint returned $statusCode after $attempt attempt(s)" -ForegroundColor Red + $payloadMB = [Math]::Round($json.Length / 1MB, 2) + Write-Host " ERROR: $Endpoint returned $statusCode after $attempt attempt(s) (payload: ${payloadMB} MB)" -ForegroundColor Red if ($responseBody) { Write-Host " Response: $responseBody" -ForegroundColor Yellow } else { From ee9f7cfa23149c987bdcf18e72af48ada1849093 Mon Sep 17 00:00:00 2001 From: Wim van den Heijkant Date: Sun, 19 Apr 2026 08:32:19 +0200 Subject: [PATCH 018/160] Add Excel Power Query workbook export MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lets a tenant admin go from "I want this data in Excel" to a working workbook in two clicks: Admin → Data → Excel Power Query Workbook → Generate token & download workbook. The downloaded .xlsx contains a Settings sheet with the API URL and a freshly-minted read-only token already filled in, plus one tab per object type with paginated Power Query M code ready to paste into Power Query Editor. Three pieces ship together because they only become useful in combination: 1. Read-only API tokens (`fgr_…`) - New `ReadApiKeys` table (migration 016). Stores SHA-256 hash + display prefix only; plaintext is shown to the operator exactly once at creation. - `authMiddleware` accepts `fgr_` bearers but only on GET requests and only on non-admin endpoints. POST and /api/admin/* with a read token both 403. A leaked workbook can read but never mutate and never reach token management itself. - Admin endpoints: list / create / revoke at /api/admin/read-tokens. 2. Bulk list endpoints - /api/assignments, /api/identity-members, /api/resource-relationships all paginated `?limit=1000&offset=N` (max 10 000 per page) with optional `?systemId` filter, returning the same `{data, total}` envelope every other list endpoint uses. 3. Excel workbook generator - POST /api/admin/data-export/workbook mints a token AND streams a ready-to-use .xlsx in a single response. Workbook contains README + Settings + 7 data sheets (Systems, Principals, Resources, Assignments, Identities, IdentityMembers, ResourceRelationships). - M code is pre-written and references the Settings sheet's BaseUrl/AuthToken named ranges, so token rotation = update one cell. /systems uses an array-shaped template; the rest use the paginated `{data,total}` walker. This is the MVP. A follow-up will replace the M-as-text presentation with a hand-built XLSX template that auto-loads queries on Excel open — the rest of the plumbing (token auth, bulk endpoints, admin UI, download endpoint) is identical. Tests: 16 new vitest cases (token format/hash determinism + workbook round-trip via exceljs). Manually smoke-tested end-to-end against the live local stack: token creation returned plaintext, used it as a bearer to read /users (7,911 rows), downloaded a 14 KB workbook with all 9 sheets and both named ranges intact. Co-Authored-By: Claude Opus 4.7 (1M context) --- app/api/package-lock.json | 816 ++++++++++++++++++ app/api/package.json | 1 + app/api/src/auth/readTokens.js | 88 ++ app/api/src/auth/readTokens.test.js | 74 ++ .../src/db/migrations/016_read_api_keys.sql | 29 + app/api/src/export/excelWorkbook.js | 151 ++++ app/api/src/export/excelWorkbook.test.js | 82 ++ app/api/src/export/queryTemplates.js | 91 ++ app/api/src/index.js | 6 + app/api/src/middleware/auth.js | 24 + app/api/src/routes/bulkLists.js | 126 +++ app/api/src/routes/dataExport.js | 105 +++ app/ui/src/components/AdminPage.jsx | 230 +++++ changes/feature-excel-powerquery-export.md | 4 + 14 files changed, 1827 insertions(+) create mode 100644 app/api/src/auth/readTokens.js create mode 100644 app/api/src/auth/readTokens.test.js create mode 100644 app/api/src/db/migrations/016_read_api_keys.sql create mode 100644 app/api/src/export/excelWorkbook.js create mode 100644 app/api/src/export/excelWorkbook.test.js create mode 100644 app/api/src/export/queryTemplates.js create mode 100644 app/api/src/routes/bulkLists.js create mode 100644 app/api/src/routes/dataExport.js create mode 100644 changes/feature-excel-powerquery-export.md diff --git a/app/api/package-lock.json b/app/api/package-lock.json index d85c49bf1..0723bb510 100644 --- a/app/api/package-lock.json +++ b/app/api/package-lock.json @@ -9,6 +9,7 @@ "version": "3.4.0", "dependencies": { "cors": "^2.8.5", + "exceljs": "^4.4.0", "express": "^4.21.0", "express-rate-limit": "^8.2.1", "helmet": "^8.1.0", @@ -415,6 +416,47 @@ "node": ">=12" } }, + "node_modules/@fast-csv/format": { + "version": "4.3.5", + "resolved": "https://registry.npmjs.org/@fast-csv/format/-/format-4.3.5.tgz", + "integrity": "sha512-8iRn6QF3I8Ak78lNAa+Gdl5MJJBM5vRHivFtMRUWINdevNo00K7OXxS2PshawLKTejVwieIlPmK5YlLu6w4u8A==", + "license": "MIT", + "dependencies": { + "@types/node": "^14.0.1", + "lodash.escaperegexp": "^4.1.2", + "lodash.isboolean": "^3.0.3", + "lodash.isequal": "^4.5.0", + "lodash.isfunction": "^3.0.9", + "lodash.isnil": "^4.0.0" + } + }, + "node_modules/@fast-csv/format/node_modules/@types/node": { + "version": "14.18.63", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.63.tgz", + "integrity": "sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ==", + "license": "MIT" + }, + "node_modules/@fast-csv/parse": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/@fast-csv/parse/-/parse-4.3.6.tgz", + "integrity": "sha512-uRsLYksqpbDmWaSmzvJcuApSEe38+6NQZBUsuAyMZKqHxH0g1wcJgsKUvN3WC8tewaqFjBMMGrkHmC+T7k8LvA==", + "license": "MIT", + "dependencies": { + "@types/node": "^14.0.1", + "lodash.escaperegexp": "^4.1.2", + "lodash.groupby": "^4.6.0", + "lodash.isfunction": "^3.0.9", + "lodash.isnil": "^4.0.0", + "lodash.isundefined": "^3.0.1", + "lodash.uniq": "^4.5.0" + } + }, + "node_modules/@fast-csv/parse/node_modules/@types/node": { + "version": "14.18.63", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.63.tgz", + "integrity": "sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ==", + "license": "MIT" + }, "node_modules/@jridgewell/sourcemap-codec": { "version": "1.5.5", "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", @@ -943,6 +985,75 @@ "integrity": "sha512-klpgFSWLW1ZEs8svjfb7g4qWY0YS5imI82dTg+QahUvJ8YqAY0P10Uk8tTyh9ZGuYEZEMaeJYCF5BFuX552hsw==", "license": "MIT" }, + "node_modules/archiver": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/archiver/-/archiver-5.3.2.tgz", + "integrity": "sha512-+25nxyyznAXF7Nef3y0EbBeqmGZgeN/BxHX29Rs39djAfaFalmQ89SE6CWyDCHzGL0yt/ycBtNOmGTW0FyGWNw==", + "license": "MIT", + "dependencies": { + "archiver-utils": "^2.1.0", + "async": "^3.2.4", + "buffer-crc32": "^0.2.1", + "readable-stream": "^3.6.0", + "readdir-glob": "^1.1.2", + "tar-stream": "^2.2.0", + "zip-stream": "^4.1.0" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/archiver-utils": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/archiver-utils/-/archiver-utils-2.1.0.tgz", + "integrity": "sha512-bEL/yUb/fNNiNTuUz979Z0Yg5L+LzLxGJz8x79lYmR54fmTIb6ob/hNQgkQnIUDWIFjZVQwl9Xs356I6BAMHfw==", + "license": "MIT", + "dependencies": { + "glob": "^7.1.4", + "graceful-fs": "^4.2.0", + "lazystream": "^1.0.0", + "lodash.defaults": "^4.2.0", + "lodash.difference": "^4.5.0", + "lodash.flatten": "^4.4.0", + "lodash.isplainobject": "^4.0.6", + "lodash.union": "^4.6.0", + "normalize-path": "^3.0.0", + "readable-stream": "^2.0.0" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/archiver-utils/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/archiver-utils/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/archiver-utils/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, "node_modules/argparse": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", @@ -974,12 +1085,77 @@ "node": ">=12" } }, + "node_modules/async": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", + "integrity": "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==", + "license": "MIT" + }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", "license": "MIT" }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/big-integer": { + "version": "1.6.52", + "resolved": "https://registry.npmjs.org/big-integer/-/big-integer-1.6.52.tgz", + "integrity": "sha512-QxD8cf2eVqJOOz63z6JIN9BzvVs/dlySa5HGSBH5xtR8dPteIRQnBxxKqkNTiT6jbDTF6jAfrd4oMcND9RGbQg==", + "license": "Unlicense", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/binary": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz", + "integrity": "sha512-D4H1y5KYwpJgK8wk1Cue5LLPgmwHKYSChkbspQg5JtVuR5ulGckxfR62H3AE9UDkdMC8yyXlqYihuz3Aqg2XZg==", + "license": "MIT", + "dependencies": { + "buffers": "~0.1.1", + "chainsaw": "~0.1.0" + }, + "engines": { + "node": "*" + } + }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "license": "MIT", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, + "node_modules/bluebird": { + "version": "3.4.7", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.4.7.tgz", + "integrity": "sha512-iD3898SR7sWVRHbiQv+sHUtHnMvC1o3nW5rAcqnq3uOn07DSAppZYUkIGslDz6gXC7HfunPe7YVBgoEJASPcHA==", + "license": "MIT" + }, "node_modules/body-parser": { "version": "1.20.4", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.4.tgz", @@ -1014,6 +1190,39 @@ "concat-map": "0.0.1" } }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "node_modules/buffer-crc32": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", + "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", + "license": "MIT", + "engines": { + "node": "*" + } + }, "node_modules/buffer-equal-constant-time": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", @@ -1026,6 +1235,23 @@ "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", "license": "MIT" }, + "node_modules/buffer-indexof-polyfill": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/buffer-indexof-polyfill/-/buffer-indexof-polyfill-1.0.2.tgz", + "integrity": "sha512-I7wzHwA3t1/lwXQh+A5PbNvJxgfo5r3xulgpYDB5zckTu/Z9oUK9biouBKQUjEqzaz3HnAT6TYoovmE+GqSf7A==", + "license": "MIT", + "engines": { + "node": ">=0.10" + } + }, + "node_modules/buffers": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz", + "integrity": "sha512-9q/rDEGSb/Qsvv2qvzIzdluL5k7AaJOTrw23z9reQthrbF7is4CtlT0DXyO1oei2DCp4uojjzQ7igaSHp1kAEQ==", + "engines": { + "node": ">=0.2.0" + } + }, "node_modules/busboy": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", @@ -1102,6 +1328,18 @@ "node": ">=18" } }, + "node_modules/chainsaw": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz", + "integrity": "sha512-75kWfWt6MEKNC8xYXIdRpDehRYY/tNSgwKaJq+dbbDcxORuVrrQ+SEHoWsniVn9XPYfP4gmdWIeDk/4YNp1rNQ==", + "license": "MIT/X11", + "dependencies": { + "traverse": ">=0.3.0 <0.4" + }, + "engines": { + "node": "*" + } + }, "node_modules/check-error": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/check-error/-/check-error-2.1.3.tgz", @@ -1112,6 +1350,21 @@ "node": ">= 16" } }, + "node_modules/compress-commons": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/compress-commons/-/compress-commons-4.1.2.tgz", + "integrity": "sha512-D3uMHtGc/fcO1Gt1/L7i1e33VOvD4A9hfQLP+6ewd+BvG/gQ84Yh4oftEhAdjSMgBgwGL+jsppT7JYNpo6MHHg==", + "license": "MIT", + "dependencies": { + "buffer-crc32": "^0.2.13", + "crc32-stream": "^4.0.2", + "normalize-path": "^3.0.0", + "readable-stream": "^3.6.0" + }, + "engines": { + "node": ">= 10" + } + }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -1222,6 +1475,37 @@ "url": "https://opencollective.com/express" } }, + "node_modules/crc-32": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.2.tgz", + "integrity": "sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==", + "license": "Apache-2.0", + "bin": { + "crc32": "bin/crc32.njs" + }, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/crc32-stream": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/crc32-stream/-/crc32-stream-4.0.3.tgz", + "integrity": "sha512-NT7w2JVU7DFroFdYkeq8cywxrgjPHWkdX1wjpRQXPX5Asews3tA+Ght6lddQO5Mkumffp3X7GEqku3epj2toIw==", + "license": "MIT", + "dependencies": { + "crc-32": "^1.2.0", + "readable-stream": "^3.4.0" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/dayjs": { + "version": "1.11.20", + "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.20.tgz", + "integrity": "sha512-YbwwqR/uYpeoP4pu043q+LTDLFBLApUP6VxRihdfNTqu4ubqMlGDLd6ErXhEgsyvY0K6nCs7nggYumAN+9uEuQ==", + "license": "MIT" + }, "node_modules/debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", @@ -1274,6 +1558,45 @@ "node": ">= 0.4" } }, + "node_modules/duplexer2": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/duplexer2/-/duplexer2-0.1.4.tgz", + "integrity": "sha512-asLFVfWWtJ90ZyOUHMqk7/S2w2guQKxUI2itj3d92ADHhxUSbCMGi1f1cBcJ7xM1To+pE/Khbwo1yuNbMEPKeA==", + "license": "BSD-3-Clause", + "dependencies": { + "readable-stream": "^2.0.2" + } + }, + "node_modules/duplexer2/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/duplexer2/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/duplexer2/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, "node_modules/ecdsa-sig-formatter": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", @@ -1298,6 +1621,15 @@ "node": ">= 0.8" } }, + "node_modules/end-of-stream": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", + "license": "MIT", + "dependencies": { + "once": "^1.4.0" + } + }, "node_modules/es-define-property": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", @@ -1399,6 +1731,26 @@ "node": ">= 0.6" } }, + "node_modules/exceljs": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/exceljs/-/exceljs-4.4.0.tgz", + "integrity": "sha512-XctvKaEMaj1Ii9oDOqbW/6e1gXknSY4g/aLCDicOXqBE4M0nRWkUu0PTp++UPNzoFY12BNHMfs/VadKIS6llvg==", + "license": "MIT", + "dependencies": { + "archiver": "^5.0.0", + "dayjs": "^1.8.34", + "fast-csv": "^4.3.1", + "jszip": "^3.10.1", + "readable-stream": "^3.6.0", + "saxes": "^5.0.1", + "tmp": "^0.2.0", + "unzipper": "^0.10.11", + "uuid": "^8.3.0" + }, + "engines": { + "node": ">=8.3.0" + } + }, "node_modules/expect-type": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.3.0.tgz", @@ -1473,6 +1825,19 @@ "express": ">= 4.11" } }, + "node_modules/fast-csv": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/fast-csv/-/fast-csv-4.3.6.tgz", + "integrity": "sha512-2RNSpuwwsJGP0frGsOmTb9oUF+VkFSM4SyLTDgwf2ciHWTarN0lQTC+F2f/t5J9QjW+c65VFIAAu85GsvMIusw==", + "license": "MIT", + "dependencies": { + "@fast-csv/format": "4.3.5", + "@fast-csv/parse": "4.3.6" + }, + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/finalhandler": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.2.tgz", @@ -1509,6 +1874,12 @@ "node": ">= 0.6" } }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", + "license": "MIT" + }, "node_modules/fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -1530,6 +1901,22 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/fstream": { + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/fstream/-/fstream-1.0.12.tgz", + "integrity": "sha512-WvJ193OHa0GHPEL+AycEJgxvBEwyfRkN1vhjca23OaPVMCaLCXTd5qAu82AjTcgP1UJmytkOKb63Ypde7raDIg==", + "deprecated": "This package is no longer supported.", + "license": "ISC", + "dependencies": { + "graceful-fs": "^4.1.2", + "inherits": "~2.0.0", + "mkdirp": ">=0.5 0", + "rimraf": "2" + }, + "engines": { + "node": ">=0.6" + } + }, "node_modules/function-bind": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", @@ -1609,6 +1996,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "license": "ISC" + }, "node_modules/has-symbols": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", @@ -1674,6 +2067,32 @@ "node": ">=0.10.0" } }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/immediate": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", + "integrity": "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==", + "license": "MIT" + }, "node_modules/inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", @@ -1752,6 +2171,48 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "license": "MIT" }, + "node_modules/jszip": { + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.10.1.tgz", + "integrity": "sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==", + "license": "(MIT OR GPL-3.0-or-later)", + "dependencies": { + "lie": "~3.3.0", + "pako": "~1.0.2", + "readable-stream": "~2.3.6", + "setimmediate": "^1.0.5" + } + }, + "node_modules/jszip/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/jszip/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/jszip/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, "node_modules/jwa": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", @@ -1812,17 +2273,104 @@ "safe-buffer": "^5.0.1" } }, + "node_modules/lazystream": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/lazystream/-/lazystream-1.0.1.tgz", + "integrity": "sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw==", + "license": "MIT", + "dependencies": { + "readable-stream": "^2.0.5" + }, + "engines": { + "node": ">= 0.6.3" + } + }, + "node_modules/lazystream/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/lazystream/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/lazystream/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/lie": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", + "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==", + "license": "MIT", + "dependencies": { + "immediate": "~3.0.5" + } + }, "node_modules/limiter": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/limiter/-/limiter-1.1.5.tgz", "integrity": "sha512-FWWMIEOxz3GwUI4Ts/IvgVy6LPvoMPgjMdQ185nN6psJyBJ4yOpzqm695/h5umdLJg2vW3GR5iG11MAkR2AzJA==" }, + "node_modules/listenercount": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/listenercount/-/listenercount-1.0.1.tgz", + "integrity": "sha512-3mk/Zag0+IJxeDrxSgaDPy4zZ3w05PRZeJNnlWhzFz5OkX49J4krc+A8X2d2M69vGMBEX0uyl8M+W+8gH+kBqQ==", + "license": "ISC" + }, "node_modules/lodash.clonedeep": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz", "integrity": "sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ==", "license": "MIT" }, + "node_modules/lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==", + "license": "MIT" + }, + "node_modules/lodash.difference": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.difference/-/lodash.difference-4.5.0.tgz", + "integrity": "sha512-dS2j+W26TQ7taQBGN8Lbbq04ssV3emRw4NY58WErlTO29pIqS0HmoT5aJ9+TUQ1N3G+JOZSji4eugsWwGp9yPA==", + "license": "MIT" + }, + "node_modules/lodash.escaperegexp": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/lodash.escaperegexp/-/lodash.escaperegexp-4.1.2.tgz", + "integrity": "sha512-TM9YBvyC84ZxE3rgfefxUWiQKLilstD6k7PTGt6wfbtXF8ixIJLOL3VYyV/z+ZiPLsVxAsKAFVwWlWeb2Y8Yyw==", + "license": "MIT" + }, + "node_modules/lodash.flatten": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz", + "integrity": "sha512-C5N2Z3DgnnKr0LOpv/hKCgKdb7ZZwafIrsesve6lmzvZIRZRGaZ/l6Q8+2W7NaT+ZwO3fFlSCzCzrDCFdJfZ4g==", + "license": "MIT" + }, + "node_modules/lodash.groupby": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.groupby/-/lodash.groupby-4.6.0.tgz", + "integrity": "sha512-5dcWxm23+VAoz+awKmBaiBvzox8+RqMgFhi7UvX9DHZr2HdxHXM/Wrf8cfKpsW37RNrvtPn6hSwNqurSILbmJw==", + "license": "MIT" + }, "node_modules/lodash.includes": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", @@ -1835,12 +2383,31 @@ "integrity": "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==", "license": "MIT" }, + "node_modules/lodash.isequal": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz", + "integrity": "sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ==", + "deprecated": "This package is deprecated. Use require('node:util').isDeepStrictEqual instead.", + "license": "MIT" + }, + "node_modules/lodash.isfunction": { + "version": "3.0.9", + "resolved": "https://registry.npmjs.org/lodash.isfunction/-/lodash.isfunction-3.0.9.tgz", + "integrity": "sha512-AirXNj15uRIMMPihnkInB4i3NHeb4iBtNg9WRWuK2o31S+ePwwNmDPaTL3o7dTJ+VXNZim7rFs4rxN4YU1oUJw==", + "license": "MIT" + }, "node_modules/lodash.isinteger": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz", "integrity": "sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==", "license": "MIT" }, + "node_modules/lodash.isnil": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/lodash.isnil/-/lodash.isnil-4.0.0.tgz", + "integrity": "sha512-up2Mzq3545mwVnMhTDMdfoG1OurpA/s5t88JmQX809eH3C8491iu2sfKhTfhQtKY78oPNhiaHJUpT/dUDAAtng==", + "license": "MIT" + }, "node_modules/lodash.isnumber": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz", @@ -1859,12 +2426,30 @@ "integrity": "sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==", "license": "MIT" }, + "node_modules/lodash.isundefined": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/lodash.isundefined/-/lodash.isundefined-3.0.1.tgz", + "integrity": "sha512-MXB1is3s899/cD8jheYYE2V9qTHwKvt+npCwpD+1Sxm3Q3cECXCiYHjeHWXNwr6Q0SOBPrYUDxendrO6goVTEA==", + "license": "MIT" + }, "node_modules/lodash.once": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz", "integrity": "sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==", "license": "MIT" }, + "node_modules/lodash.union": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.union/-/lodash.union-4.6.0.tgz", + "integrity": "sha512-c4pB2CdGrGdjMKYLA+XiRDO7Y0PRQbm/Gzg8qMj+QH+pFVAoTp5sBpO0odL3FjoPCGjK96p6qsP+yQoiLoOBcw==", + "license": "MIT" + }, + "node_modules/lodash.uniq": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz", + "integrity": "sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ==", + "license": "MIT" + }, "node_modules/loupe": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/loupe/-/loupe-3.2.1.tgz", @@ -2059,6 +2644,15 @@ "node": ">= 0.6" } }, + "node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", @@ -2101,6 +2695,12 @@ "wrappy": "1" } }, + "node_modules/pako": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", + "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==", + "license": "(MIT AND Zlib)" + }, "node_modules/parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -2370,6 +2970,63 @@ "node": ">= 0.8" } }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/readdir-glob": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/readdir-glob/-/readdir-glob-1.1.3.tgz", + "integrity": "sha512-v05I2k7xN8zXvPD9N+z/uhXPaj0sUFCe2rcWZIpBsqxfP7xXFQ0tipAd/wjj1YxWyWtUS5IDJpOG82JKt2EAVA==", + "license": "Apache-2.0", + "dependencies": { + "minimatch": "^5.1.0" + } + }, + "node_modules/readdir-glob/node_modules/brace-expansion": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.0.tgz", + "integrity": "sha512-TN1kCZAgdgweJhWWpgKYrQaMNHcDULHkWwQIspdtjV4Y5aurRdZpjAqn6yX3FPqTA9ngHCc4hJxMAMgGfve85w==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/readdir-glob/node_modules/minimatch": { + "version": "5.1.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.9.tgz", + "integrity": "sha512-7o1wEA2RyMP7Iu7GNba9vc0RWWGACJOCZBJX2GJWip0ikV+wcOsgVuY9uE8CPiyQhkGFSlhuSkZPavN7u1c2Fw==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/rimraf": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", + "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", + "deprecated": "Rimraf versions prior to v4 are no longer supported", + "license": "ISC", + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + } + }, "node_modules/rollup": { "version": "4.60.1", "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.60.1.tgz", @@ -2441,6 +3098,18 @@ "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", "license": "MIT" }, + "node_modules/saxes": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/saxes/-/saxes-5.0.1.tgz", + "integrity": "sha512-5LBh1Tls8c9xgGjw3QrMwETmTMVk0oFgvrFSvWx62llR2hcEInrKNZ2GZCCuuy2lvWrdl5jhbpeqc5hRYKFOcw==", + "license": "ISC", + "dependencies": { + "xmlchars": "^2.2.0" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/semver": { "version": "7.7.4", "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", @@ -2498,6 +3167,12 @@ "node": ">= 0.8.0" } }, + "node_modules/setimmediate": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", + "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==", + "license": "MIT" + }, "node_modules/setprototypeof": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", @@ -2633,6 +3308,15 @@ "node": ">=10.0.0" } }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, "node_modules/swagger-ui-dist": { "version": "5.32.1", "resolved": "https://registry.npmjs.org/swagger-ui-dist/-/swagger-ui-dist-5.32.1.tgz", @@ -2657,6 +3341,22 @@ "express": ">=4.0.0 || >=5.0.0-beta" } }, + "node_modules/tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "license": "MIT", + "dependencies": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/tinybench": { "version": "2.9.0", "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", @@ -2701,6 +3401,15 @@ "node": ">=14.0.0" } }, + "node_modules/tmp": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.5.tgz", + "integrity": "sha512-voyz6MApa1rQGUxT3E+BK7/ROe8itEx7vD8/HEvt4xwXucvQ5G5oeEiHkmHZJuBO21RpOf+YYm9MOivj709jow==", + "license": "MIT", + "engines": { + "node": ">=14.14" + } + }, "node_modules/toidentifier": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", @@ -2710,6 +3419,15 @@ "node": ">=0.6" } }, + "node_modules/traverse": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz", + "integrity": "sha512-iawgk0hLP3SxGKDfnDJf8wTz4p2qImnyihM5Hh/sGvQ3K37dPi/w8sRhdNIxYA1TwFwc5mDhIJq+O0RsvXBKdQ==", + "license": "MIT/X11", + "engines": { + "node": "*" + } + }, "node_modules/type-is": { "version": "1.6.18", "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", @@ -2744,6 +3462,54 @@ "node": ">= 0.8" } }, + "node_modules/unzipper": { + "version": "0.10.14", + "resolved": "https://registry.npmjs.org/unzipper/-/unzipper-0.10.14.tgz", + "integrity": "sha512-ti4wZj+0bQTiX2KmKWuwj7lhV+2n//uXEotUmGuQqrbVZSEGFMbI68+c6JCQ8aAmUWYvtHEz2A8K6wXvueR/6g==", + "license": "MIT", + "dependencies": { + "big-integer": "^1.6.17", + "binary": "~0.3.0", + "bluebird": "~3.4.1", + "buffer-indexof-polyfill": "~1.0.0", + "duplexer2": "~0.1.4", + "fstream": "^1.0.12", + "graceful-fs": "^4.2.2", + "listenercount": "~1.0.1", + "readable-stream": "~2.3.6", + "setimmediate": "~1.0.4" + } + }, + "node_modules/unzipper/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/unzipper/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/unzipper/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", @@ -2759,6 +3525,15 @@ "node": ">= 0.4.0" } }, + "node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", @@ -2990,6 +3765,12 @@ "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", "license": "ISC" }, + "node_modules/xmlchars": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz", + "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==", + "license": "MIT" + }, "node_modules/xtend": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", @@ -3018,6 +3799,41 @@ "json2yaml": "bin/json2yaml", "yaml2json": "bin/yaml2json" } + }, + "node_modules/zip-stream": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/zip-stream/-/zip-stream-4.1.1.tgz", + "integrity": "sha512-9qv4rlDiopXg4E69k+vMHjNN63YFMe9sZMrdlvKnCjlCRWeCBswPPMPUfx+ipsAWq1LXHe70RcbaHdJJpS6hyQ==", + "license": "MIT", + "dependencies": { + "archiver-utils": "^3.0.4", + "compress-commons": "^4.1.2", + "readable-stream": "^3.6.0" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/zip-stream/node_modules/archiver-utils": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/archiver-utils/-/archiver-utils-3.0.4.tgz", + "integrity": "sha512-KVgf4XQVrTjhyWmx6cte4RxonPLR9onExufI1jhvw/MQ4BB6IsZD5gT8Lq+u/+pRkWna/6JoHpiQioaqFP5Rzw==", + "license": "MIT", + "dependencies": { + "glob": "^7.2.3", + "graceful-fs": "^4.2.0", + "lazystream": "^1.0.0", + "lodash.defaults": "^4.2.0", + "lodash.difference": "^4.5.0", + "lodash.flatten": "^4.4.0", + "lodash.isplainobject": "^4.0.6", + "lodash.union": "^4.6.0", + "normalize-path": "^3.0.0", + "readable-stream": "^3.6.0" + }, + "engines": { + "node": ">= 10" + } } } } diff --git a/app/api/package.json b/app/api/package.json index 255b837a7..d15f4874e 100644 --- a/app/api/package.json +++ b/app/api/package.json @@ -14,6 +14,7 @@ }, "dependencies": { "cors": "^2.8.5", + "exceljs": "^4.4.0", "express": "^4.21.0", "express-rate-limit": "^8.2.1", "helmet": "^8.1.0", diff --git a/app/api/src/auth/readTokens.js b/app/api/src/auth/readTokens.js new file mode 100644 index 000000000..98ea820ca --- /dev/null +++ b/app/api/src/auth/readTokens.js @@ -0,0 +1,88 @@ +// Read-only API tokens (`fgr_…`) used by downstream tooling — chiefly the +// generated Excel Power Query workbook — to refresh the read API on +// auth-enabled deployments without an interactive sign-in. +// +// Storage: only the SHA-256 hash of the plaintext is kept in `ReadApiKeys`. +// We can use plain SHA-256 (no salt) because the plaintext is 32 random +// bytes of url-safe base64 — there's no dictionary to attack with rainbow +// tables, and a per-row salt would just complicate lookup without buying +// security against an attacker who already has the database. +// +// Plaintext is shown to the operator exactly once at creation; subsequent +// listings only show the prefix and a hash-of-the-prefix-style display. + +import crypto from 'crypto'; +import * as db from '../db/connection.js'; + +const TOKEN_PREFIX = 'fgr_'; +const TOKEN_RANDOM_BYTES = 32; +const PREFIX_DISPLAY_LEN = 12; // length of leading characters stored for display + +export function hashToken(plaintext) { + return crypto.createHash('sha256').update(plaintext, 'utf8').digest('hex'); +} + +// Generate a new plaintext token. The caller is responsible for showing it to +// the operator exactly once and persisting only the hash + display prefix. +export function generateToken() { + const random = crypto.randomBytes(TOKEN_RANDOM_BYTES).toString('base64url'); + return `${TOKEN_PREFIX}${random}`; +} + +// Return true if the bearer token looks like a read-API token. Cheap check +// to skip the JWT path quickly when called from the auth middleware. +export function isReadTokenFormat(bearer) { + return typeof bearer === 'string' && bearer.startsWith(TOKEN_PREFIX); +} + +// Insert a new read token. Returns the row + plaintext (for one-time display). +export async function createToken({ name, createdBy, expiresAt }) { + const plaintext = generateToken(); + const tokenHash = hashToken(plaintext); + const tokenPrefix = plaintext.slice(0, PREFIX_DISPLAY_LEN); + const r = await db.query( + `INSERT INTO "ReadApiKeys" ("name", "tokenHash", "tokenPrefix", "createdBy", "expiresAt") + VALUES ($1, $2, $3, $4, $5) + RETURNING id, name, "tokenPrefix", "createdAt", "createdBy", "expiresAt", "lastUsedAt", revoked`, + [name, tokenHash, tokenPrefix, createdBy || null, expiresAt || null] + ); + return { token: plaintext, row: r.rows[0] }; +} + +export async function listTokens() { + const r = await db.query( + `SELECT id, name, "tokenPrefix", "createdAt", "createdBy", "expiresAt", "lastUsedAt", revoked + FROM "ReadApiKeys" + ORDER BY "createdAt" DESC` + ); + return r.rows; +} + +export async function revokeToken(id) { + const r = await db.query( + `UPDATE "ReadApiKeys" SET revoked = TRUE WHERE id = $1 RETURNING id`, + [id] + ); + return r.rowCount > 0; +} + +// Look up an active token by its plaintext value (called by authMiddleware on +// every request that uses an `fgr_` bearer). Returns the row or null. Also +// updates lastUsedAt fire-and-forget — we don't await it because we don't want +// auth latency to depend on a write. +export async function findActiveByPlaintext(plaintext) { + const tokenHash = hashToken(plaintext); + const r = await db.query( + `SELECT id, name, "expiresAt", revoked + FROM "ReadApiKeys" + WHERE "tokenHash" = $1`, + [tokenHash] + ); + if (r.rows.length === 0) return null; + const row = r.rows[0]; + if (row.revoked) return null; + if (row.expiresAt && new Date(row.expiresAt) < new Date()) return null; + + db.query(`UPDATE "ReadApiKeys" SET "lastUsedAt" = now() WHERE id = $1`, [row.id]).catch(() => {}); + return row; +} diff --git a/app/api/src/auth/readTokens.test.js b/app/api/src/auth/readTokens.test.js new file mode 100644 index 000000000..96e93de05 --- /dev/null +++ b/app/api/src/auth/readTokens.test.js @@ -0,0 +1,74 @@ +// Unit tests for the read-only API token primitives. +// +// Lookup paths that hit the DB (createToken / findActiveByPlaintext) are +// covered indirectly by the live smoke tests in test/nightly. Here we only +// pin the pure helpers — the bits that decide whether a bearer string +// counts as a read token, and that hashing is deterministic. A regression +// in either is a security regression: the middleware would either accept +// the wrong shape of credential or fail to look one up. + +import { describe, it, expect, vi } from 'vitest'; + +// The module imports `../db/connection.js` — the lookup tests don't need a +// real DB so we mock it to a no-op pool. The pure helpers (hashToken, +// generateToken, isReadTokenFormat) don't actually call db at module load, +// only later, so the mock keeps the import chain happy. +vi.mock('../db/connection.js', () => ({ + query: vi.fn(), + queryOne: vi.fn(), +})); + +const { hashToken, generateToken, isReadTokenFormat } = await import('./readTokens.js'); + +describe('hashToken', () => { + it('returns a deterministic 64-char hex SHA-256', () => { + const a = hashToken('fgr_some-token-value'); + const b = hashToken('fgr_some-token-value'); + expect(a).toBe(b); + expect(a).toMatch(/^[0-9a-f]{64}$/); + }); + + it('produces different hashes for different inputs', () => { + expect(hashToken('fgr_a')).not.toBe(hashToken('fgr_b')); + }); +}); + +describe('generateToken', () => { + it('returns a token starting with the fgr_ prefix', () => { + expect(generateToken()).toMatch(/^fgr_/); + }); + + it('produces a high-entropy suffix (43+ url-safe base64 chars from 32 bytes)', () => { + const tok = generateToken(); + const suffix = tok.slice('fgr_'.length); + // 32 random bytes encode to 43 url-safe base64 chars (no padding). + expect(suffix.length).toBeGreaterThanOrEqual(43); + expect(suffix).toMatch(/^[A-Za-z0-9_-]+$/); + }); + + it('does not collide on rapid successive calls (sanity check on randomness)', () => { + const seen = new Set(); + for (let i = 0; i < 1000; i++) seen.add(generateToken()); + expect(seen.size).toBe(1000); + }); +}); + +describe('isReadTokenFormat', () => { + it('accepts strings that begin with fgr_', () => { + expect(isReadTokenFormat('fgr_anything')).toBe(true); + }); + + it('rejects crawler-format tokens (fgc_)', () => { + expect(isReadTokenFormat('fgc_anything')).toBe(false); + }); + + it('rejects JWTs (heuristically: anything not starting with fgr_)', () => { + expect(isReadTokenFormat('eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9...')).toBe(false); + }); + + it('rejects non-strings (defensive — middleware passes header.split() output)', () => { + expect(isReadTokenFormat(undefined)).toBe(false); + expect(isReadTokenFormat(null)).toBe(false); + expect(isReadTokenFormat(42)).toBe(false); + }); +}); diff --git a/app/api/src/db/migrations/016_read_api_keys.sql b/app/api/src/db/migrations/016_read_api_keys.sql new file mode 100644 index 000000000..feaa8bc96 --- /dev/null +++ b/app/api/src/db/migrations/016_read_api_keys.sql @@ -0,0 +1,29 @@ +-- Read-only API keys for downstream tooling (Excel Power Query, BI tools, etc). +-- +-- The existing crawler API keys (`fgc_…`) are only honoured by the +-- crawlerAuthMiddleware on /api/ingest and /api/crawlers/* routes — they do +-- NOT satisfy the JWT-based authMiddleware that guards the read API +-- (/api/users, /api/resources, etc). For a Power-Query workbook to refresh +-- against an auth-on deployment we need a credential that survives without a +-- signed-in user. This table stores those credentials. +-- +-- Format: tokens are issued as `fgr_<32-byte url-safe base64>`. Only the +-- SHA-256 hash is stored; the plaintext is shown to the operator exactly once +-- at creation time. Lookup by hash, never by id, so a stolen DB row can't be +-- replayed against the API. + +CREATE TABLE IF NOT EXISTS "ReadApiKeys" ( + "id" SERIAL PRIMARY KEY, + "name" TEXT NOT NULL, + "tokenHash" TEXT NOT NULL UNIQUE, + "tokenPrefix" TEXT NOT NULL, -- first 12 chars of plaintext, for display + "createdAt" TIMESTAMPTZ NOT NULL DEFAULT now(), + "createdBy" TEXT, + "lastUsedAt" TIMESTAMPTZ, + "expiresAt" TIMESTAMPTZ, -- NULL = no expiry + "revoked" BOOLEAN NOT NULL DEFAULT FALSE +); + +CREATE INDEX IF NOT EXISTS "ix_ReadApiKeys_tokenHash" + ON "ReadApiKeys" ("tokenHash") + WHERE "revoked" = FALSE; diff --git a/app/api/src/export/excelWorkbook.js b/app/api/src/export/excelWorkbook.js new file mode 100644 index 000000000..bc5a3396b --- /dev/null +++ b/app/api/src/export/excelWorkbook.js @@ -0,0 +1,151 @@ +// Excel workbook generator for the Power Query data export. +// +// MVP shape (v1): the workbook is a self-contained, opens-anywhere file +// with the API URL and read token already embedded in the Settings sheet +// and the M code for every object type printed verbatim on its own sheet. +// User opens the file → sees clear instructions → pastes the M code into +// Power Query Editor (Data → Get Data → Other Sources → Blank Query) → +// clicks Refresh. Token rotation = update one cell on the Settings sheet, +// no edits to the M code required. +// +// Why M-as-text instead of fully-embedded queries: hand-crafting the +// xl/queries/queries.xml that Excel auto-loads from is brittle without a +// real Excel install to validate against. A follow-up PR will swap this +// generator for a hand-built template + token-stamp approach so the user +// experience becomes a single Refresh click. The infrastructure (token +// auth, bulk endpoints, admin UI, download endpoint) is identical. + +import ExcelJS from 'exceljs'; +import { QUERIES } from './queryTemplates.js'; + +const HEADER_FILL = { type: 'pattern', pattern: 'solid', fgColor: { argb: 'FF1F2937' } }; +const HEADER_FONT = { color: { argb: 'FFFFFFFF' }, bold: true, size: 12 }; +const TOKEN_FILL = { type: 'pattern', pattern: 'solid', fgColor: { argb: 'FFFEF3C7' } }; +const M_FONT = { name: 'Consolas', size: 10 }; + +// Build the workbook with `BaseUrl` + `AuthToken` baked into the Settings +// sheet as defined names. Returns a Buffer ready to send as the response. +export async function generateWorkbook({ apiBaseUrl, token }) { + const wb = new ExcelJS.Workbook(); + wb.creator = 'Identity Atlas'; + wb.created = new Date(); + + buildReadMeSheet(wb); + buildSettingsSheet(wb, apiBaseUrl, token); + for (const q of QUERIES) { + buildQuerySheet(wb, q); + } + + return Buffer.from(await wb.xlsx.writeBuffer()); +} + +function buildReadMeSheet(wb) { + const sheet = wb.addWorksheet('README'); + sheet.columns = [{ width: 110 }]; + + const lines = [ + { text: 'Identity Atlas — Excel Power Query Workbook', style: { font: HEADER_FONT, fill: HEADER_FILL, alignment: { vertical: 'middle' } } }, + { text: '' }, + { text: 'How to use this workbook', style: { font: { bold: true, size: 12 } } }, + { text: '' }, + { text: '1. Check the Settings sheet — your API URL and read API key are already filled in.' }, + { text: ' If you ever need to point this workbook at a different deployment, or rotate the' }, + { text: ' token, edit those two cells. The named ranges feed every query automatically.' }, + { text: '' }, + { text: '2. For each data tab (Principals, Resources, Assignments, ...) the Power Query M' }, + { text: ' code is printed in cell A1. To turn it into live data:' }, + { text: ' - Open the Data tab on the Excel ribbon.' }, + { text: ' - Get Data → From Other Sources → Blank Query.' }, + { text: ' - In Power Query Editor: Home → Advanced Editor.' }, + { text: ' - Paste the M code from the sheet, then click Done → Close & Load.' }, + { text: ' - The query name in Power Query becomes the table name on this sheet.' }, + { text: '' }, + { text: '3. Refresh: Data → Refresh All. Or right-click a query → Refresh.' }, + { text: '' }, + { text: 'Security notes', style: { font: { bold: true, size: 12 } } }, + { text: '' }, + { text: 'The token in the Settings sheet is a read-only API key. It can only call read' }, + { text: 'endpoints (GET) and cannot reach any admin function. If the workbook is shared,' }, + { text: 'treat the token like a password and rotate it (Admin → Data → Read API Tokens).' }, + ]; + + lines.forEach((line, i) => { + const cell = sheet.getCell(i + 1, 1); + cell.value = line.text; + if (line.style?.font) cell.font = line.style.font; + if (line.style?.fill) cell.fill = line.style.fill; + if (line.style?.alignment) cell.alignment = line.style.alignment; + }); + sheet.getRow(1).height = 30; +} + +function buildSettingsSheet(wb, apiBaseUrl, token) { + const sheet = wb.addWorksheet('Settings'); + sheet.columns = [{ width: 18 }, { width: 80 }]; + + // Header row + const header = sheet.getRow(1); + header.values = ['Setting', 'Value']; + header.font = HEADER_FONT; + header.fill = HEADER_FILL; + header.height = 22; + + // BaseUrl + sheet.getCell('A2').value = 'BaseUrl'; + sheet.getCell('A2').font = { bold: true }; + sheet.getCell('B2').value = apiBaseUrl; + sheet.getCell('B2').fill = TOKEN_FILL; + + // AuthToken + sheet.getCell('A3').value = 'AuthToken'; + sheet.getCell('A3').font = { bold: true }; + sheet.getCell('B3').value = token; + sheet.getCell('B3').fill = TOKEN_FILL; + + // Instructions + sheet.getCell('A5').value = 'Notes'; + sheet.getCell('A5').font = { bold: true }; + sheet.getCell('B5').value = 'Edit the BaseUrl / AuthToken cells above to retarget this workbook. The named ranges feed every Power Query in the file.'; + sheet.getCell('B5').alignment = { wrapText: true }; + sheet.getRow(5).height = 40; + + // Defined names — these are what the M code references via + // Excel.CurrentWorkbook(){[Name="BaseUrl"]}[Content]{0}[Column1]. + // Defined names targeting a single cell return a one-row table with one + // column called Column1; that's why the M code uses [Column1] explicitly. + wb.definedNames.add(`'Settings'!$B$2`, 'BaseUrl'); + wb.definedNames.add(`'Settings'!$B$3`, 'AuthToken'); +} + +function buildQuerySheet(wb, query) { + const sheet = wb.addWorksheet(query.sheet); + sheet.columns = [{ width: 110 }]; + + // Header + const header = sheet.getRow(1); + header.values = [`${query.sheet} — Power Query M code`]; + header.font = HEADER_FONT; + header.fill = HEADER_FILL; + header.height = 22; + + // Endpoint hint row + sheet.getCell('A2').value = `Endpoint: GET ${query.endpoint} (paginated, returns { data, total })`; + sheet.getCell('A2').font = { italic: true, color: { argb: 'FF6B7280' } }; + + // Instructions + sheet.getCell('A4').value = 'Paste the M code below into Power Query: Data → Get Data → Other Sources → Blank Query → Advanced Editor.'; + sheet.getCell('A4').alignment = { wrapText: true }; + sheet.getRow(4).height = 36; + + // The M code itself, in a single cell with monospace font and wrap + const mCell = sheet.getCell('A6'); + mCell.value = query.m; + mCell.font = M_FONT; + mCell.alignment = { vertical: 'top', wrapText: true }; + + // Approximate row height by line count — exceljs can't auto-size based on + // wrapped content, so a fixed height with vertical-top alignment is the + // pragmatic compromise. + const lineCount = query.m.split('\n').length; + sheet.getRow(6).height = Math.max(15, lineCount * 13); +} diff --git a/app/api/src/export/excelWorkbook.test.js b/app/api/src/export/excelWorkbook.test.js new file mode 100644 index 000000000..3c7393e37 --- /dev/null +++ b/app/api/src/export/excelWorkbook.test.js @@ -0,0 +1,82 @@ +// Smoke tests for the Excel Power Query workbook generator. We don't try to +// validate the full XLSX schema — exceljs handles that — but we do pin the +// shape of the output so a regression in sheet naming, named ranges, or M +// content fails CI rather than producing a workbook that opens but has the +// wrong tabs / missing token. + +import { describe, it, expect, beforeAll } from 'vitest'; +import ExcelJS from 'exceljs'; +import { generateWorkbook } from './excelWorkbook.js'; +import { QUERIES } from './queryTemplates.js'; + +const FIXTURE = { + apiBaseUrl: 'http://localhost:3001/api', + token: 'fgr_unit-test-token-value', +}; + +describe('generateWorkbook', () => { + let buffer; + let wb; + + beforeAll(async () => { + buffer = await generateWorkbook(FIXTURE); + wb = new ExcelJS.Workbook(); + await wb.xlsx.load(buffer); + }); + + it('returns a non-empty Buffer', () => { + expect(Buffer.isBuffer(buffer)).toBe(true); + expect(buffer.length).toBeGreaterThan(2000); + }); + + it('opens cleanly with exceljs (round-trip)', () => { + expect(wb.worksheets.length).toBeGreaterThan(0); + }); + + it('includes the README and Settings sheets first', () => { + expect(wb.worksheets[0].name).toBe('README'); + expect(wb.worksheets[1].name).toBe('Settings'); + }); + + it('writes the supplied apiBaseUrl and token into the Settings sheet', () => { + const settings = wb.getWorksheet('Settings'); + expect(settings.getCell('B2').value).toBe(FIXTURE.apiBaseUrl); + expect(settings.getCell('B3').value).toBe(FIXTURE.token); + }); + + it('defines named ranges BaseUrl and AuthToken pointing at the Settings cells', () => { + // The M code references the names — if they go missing every query + // breaks at refresh time. Use exceljs's resolution API so we don't + // depend on the internal model representation. + const baseUrlRanges = wb.definedNames.getRanges('BaseUrl'); + const tokenRanges = wb.definedNames.getRanges('AuthToken'); + expect(baseUrlRanges?.ranges?.length || 0).toBeGreaterThan(0); + expect(tokenRanges?.ranges?.length || 0).toBeGreaterThan(0); + // Both names should resolve to a Settings-sheet cell — exact ref isn't + // load-bearing as long as the cell value is right (verified above). + expect(JSON.stringify(baseUrlRanges)).toMatch(/Settings/); + expect(JSON.stringify(tokenRanges)).toMatch(/Settings/); + }); + + it('emits one sheet per object type defined in queryTemplates', () => { + for (const q of QUERIES) { + expect(wb.getWorksheet(q.sheet)).toBeDefined(); + } + }); + + it('puts the M code on each query sheet, with the Excel.CurrentWorkbook lookups intact', () => { + for (const q of QUERIES) { + const sheet = wb.getWorksheet(q.sheet); + const cell = sheet.getCell('A6').value; + expect(typeof cell).toBe('string'); + // These two strings are the load-bearing parts of every query — they + // wire the named ranges to the Power Query at refresh time. If they + // vanish, the workbook is just a list of queries that ask the user + // for credentials interactively, which defeats the whole feature. + expect(cell).toContain('Excel.CurrentWorkbook(){[Name="BaseUrl"]}'); + expect(cell).toContain('Excel.CurrentWorkbook(){[Name="AuthToken"]}'); + // And it must reference the right endpoint + expect(cell).toContain(q.endpoint); + } + }); +}); diff --git a/app/api/src/export/queryTemplates.js b/app/api/src/export/queryTemplates.js new file mode 100644 index 000000000..819a518f4 --- /dev/null +++ b/app/api/src/export/queryTemplates.js @@ -0,0 +1,91 @@ +// Power Query M templates for the Excel data-export workbook. +// +// Each entry produces one Excel sheet with the M code printed in cell A1 +// (multi-line), ready to be pasted into Power Query Editor: +// Data → Get Data → Other Sources → Blank Query → Advanced Editor → paste. +// +// All queries pull credentials from the workbook's two named ranges +// (`BaseUrl`, `AuthToken`) instead of hard-coding them, so when the user +// rotates their token they only have to update one cell on the Settings +// sheet — the queries pick the new value up on the next refresh. +// +// Pagination strategy: every list endpoint returns `{ data, total }`. We +// fetch the first page to read `total`, then List.Generate pulls the rest +// in 1000-record steps and List.Combine flattens. PAGE_SIZE deliberately +// matches the bulkLists default so we never get throttled by a smaller cap. + +const PAGE_SIZE = 1000; + +// Single shared M function: paginate(endpoint, extraQuery) +// Returns the combined `data` array as a list of records. +// +// Heredoc-style template literal — we substitute PAGE_SIZE only. The user +// never edits this; they edit BaseUrl and AuthToken on the Settings sheet. +const PAGINATED_FETCH = ` +let + BaseUrl = Excel.CurrentWorkbook(){[Name="BaseUrl"]}[Content]{0}[Column1], + AuthToken = Excel.CurrentWorkbook(){[Name="AuthToken"]}[Content]{0}[Column1], + Headers = [#"Authorization" = "Bearer " & AuthToken], + PageSize = ${PAGE_SIZE}, + FetchPage = (offset as number) => + Json.Document(Web.Contents(BaseUrl, [ + RelativePath = "ENDPOINT_PATH", + Query = [limit = Text.From(PageSize), offset = Text.From(offset)], + Headers = Headers + ])), + First = FetchPage(0), + Total = First[total], + Pages = if Total <= PageSize then {First} + else List.Generate( + () => [page = First, off = 0], + each [off] < Total, + each [page = FetchPage([off] + PageSize), off = [off] + PageSize], + each [page] + ), + Combined = List.Combine(List.Transform(Pages, each _[data])), + Table = Table.FromList(Combined, Splitter.SplitByNothing(), null, null, ExtraValues.Error), + Expanded = Table.ExpandRecordColumn(Table, "Column1", Record.FieldNames(Combined{0})) +in + Expanded +`.trim(); + +function paginatedQuery(endpointPath) { + return PAGINATED_FETCH.replace('ENDPOINT_PATH', endpointPath); +} + +// `/api/systems` predates the {data,total} convention used by every other +// list endpoint — it returns a plain JSON array. We use a simpler M +// template so the Systems tab still works without changing the UI-facing +// API contract. +const ARRAY_FETCH = ` +let + BaseUrl = Excel.CurrentWorkbook(){[Name="BaseUrl"]}[Content]{0}[Column1], + AuthToken = Excel.CurrentWorkbook(){[Name="AuthToken"]}[Content]{0}[Column1], + Headers = [#"Authorization" = "Bearer " & AuthToken], + Source = Json.Document(Web.Contents(BaseUrl, [ + RelativePath = "ENDPOINT_PATH", + Headers = Headers + ])), + Table = Table.FromList(Source, Splitter.SplitByNothing(), null, null, ExtraValues.Error), + Expanded = if List.IsEmpty(Source) then Table + else Table.ExpandRecordColumn(Table, "Column1", Record.FieldNames(Source{0})) +in + Expanded +`.trim(); + +function arrayQuery(endpointPath) { + return ARRAY_FETCH.replace('ENDPOINT_PATH', endpointPath); +} + +// Each tab description follows the {sheet, endpoint, m} shape. The `sheet` +// becomes the Excel tab label and the named query name; `endpoint` is the +// API path inside `RelativePath` (so the workbook works against any host). +export const QUERIES = [ + { sheet: 'Systems', endpoint: 'systems', m: arrayQuery('systems') }, + { sheet: 'Principals', endpoint: 'users', m: paginatedQuery('users') }, + { sheet: 'Resources', endpoint: 'resources', m: paginatedQuery('resources') }, + { sheet: 'Assignments', endpoint: 'assignments', m: paginatedQuery('assignments') }, + { sheet: 'Identities', endpoint: 'identities', m: paginatedQuery('identities') }, + { sheet: 'IdentityMembers', endpoint: 'identity-members', m: paginatedQuery('identity-members') }, + { sheet: 'ResourceRelationships', endpoint: 'resource-relationships', m: paginatedQuery('resource-relationships') }, +]; diff --git a/app/api/src/index.js b/app/api/src/index.js index 637ef04ed..6d4062b17 100644 --- a/app/api/src/index.js +++ b/app/api/src/index.js @@ -32,6 +32,8 @@ import { crawlerAuthMiddleware } from './middleware/crawlerAuth.js'; import ingestRouter from './routes/ingest.js'; import jobsRouter from './routes/jobs.js'; import csvUploadsRouter from './routes/csvUploads.js'; +import dataExportRouter from './routes/dataExport.js'; +import bulkListsRouter from './routes/bulkLists.js'; import { loadAuthConfig, isAuthEnabled, getTenantId, getClientId } from './config/authConfig.js'; import swaggerUi from 'swagger-ui-express'; import YAML from 'yamljs'; @@ -246,6 +248,10 @@ app.use('/api', authMiddleware, riskScoringRunsRouter); app.use('/api', authMiddleware, correlationRulesetsRouter); app.use('/api', authMiddleware, csvUploadsRouter); app.use('/api', authMiddleware, governanceRouter); +// Bulk list endpoints used by Power Query / BI tools (read API keys honoured) +app.use('/api', authMiddleware, bulkListsRouter); +// Read API token CRUD + Excel workbook download (admin-scoped) +app.use('/api', authMiddleware, dataExportRouter); // ─── Crawler & job routes ─────────────────────────────────────── // Admin crawler management (Entra ID auth) — /api/admin/crawlers/* diff --git a/app/api/src/middleware/auth.js b/app/api/src/middleware/auth.js index c558bed86..8571dc545 100644 --- a/app/api/src/middleware/auth.js +++ b/app/api/src/middleware/auth.js @@ -12,6 +12,7 @@ import { getClientId, getRequiredRoles, } from '../config/authConfig.js'; +import { isReadTokenFormat, findActiveByPlaintext } from '../auth/readTokens.js'; // jwks-rsa's getSigningKey is callback-shaped. We need a stable function ref // that resolves the *current* client at call time so a hot reload picks up the @@ -44,6 +45,29 @@ export function authMiddleware(req, res, next) { if (token.startsWith('fgc_')) { return next(); } + + // Read-only API keys (`fgr_…`) are accepted on GET requests to non-admin + // endpoints — that's all downstream tooling (Excel Power Query, BI imports) + // needs and it keeps the blast radius of a leaked read token contained. + // Anything mutating or admin-scoped MUST come from a real signed-in user. + if (isReadTokenFormat(token)) { + if (req.method !== 'GET') { + return res.status(403).json({ error: 'Read API keys may only be used for GET requests' }); + } + if (req.path.startsWith('/api/admin/')) { + return res.status(403).json({ error: 'Read API keys cannot access admin endpoints' }); + } + findActiveByPlaintext(token).then(row => { + if (!row) return res.status(401).json({ error: 'Invalid, revoked, or expired read API key' }); + req.readToken = { id: row.id, name: row.name }; + next(); + }).catch(err => { + console.error('Read token lookup failed:', err.message); + res.status(500).json({ error: 'Authentication service error' }); + }); + return; + } + const tenantId = getTenantId(); const clientId = getClientId(); diff --git a/app/api/src/routes/bulkLists.js b/app/api/src/routes/bulkLists.js new file mode 100644 index 000000000..218b6f829 --- /dev/null +++ b/app/api/src/routes/bulkLists.js @@ -0,0 +1,126 @@ +// Bulk list endpoints for the join-table entities (ResourceAssignments, +// IdentityMembers, ResourceRelationships). The UI never needed flat +// listings of these — every existing route returns them scoped to a single +// resource/principal/identity. The Excel Power Query export does need the +// flat versions, so we add them here. +// +// All three follow the same shape: paginated (limit/offset, default 1000, +// max 10000), optional ?systemId filter, returns { data: [...], total: N }. +// Larger default page size than the entity list endpoints because Power +// Query is happiest when it can walk through fewer pages. + +import { Router } from 'express'; +import * as db from '../db/connection.js'; + +const router = Router(); + +const DEFAULT_LIMIT = 1000; +const MAX_LIMIT = 10000; + +function parsePaging(req) { + const limit = Math.min(Math.max(parseInt(req.query.limit, 10) || DEFAULT_LIMIT, 1), MAX_LIMIT); + const offset = Math.max(parseInt(req.query.offset, 10) || 0, 0); + const systemIdRaw = (req.query.systemId || '').toString().trim(); + const systemId = /^\d+$/.test(systemIdRaw) ? parseInt(systemIdRaw, 10) : null; + return { limit, offset, systemId }; +} + +// Run the data + count queries in parallel against a single WHERE clause that +// uses $1 for the optional systemId. Each table picks its own column set; the +// rest is mechanical. +async function runListAndCount({ table, alias, columns, orderBy, dataWhere, countWhere, systemId, limit, offset }) { + const dataParams = systemId !== null ? [systemId, limit, offset] : [limit, offset]; + const dataLimitOffset = systemId !== null ? '$2 OFFSET $3' : '$1 OFFSET $2'; + const countParams = systemId !== null ? [systemId] : []; + + const [list, count] = await Promise.all([ + db.query( + `SELECT ${columns} + FROM "${table}" ${alias} + ${dataWhere} + ORDER BY ${orderBy} + LIMIT ${dataLimitOffset}`, + dataParams + ), + db.queryOne( + `SELECT COUNT(*)::int AS total FROM "${table}" ${alias} ${countWhere}`, + countParams + ), + ]); + return { data: list.rows, total: count?.total || 0 }; +} + +// ─── GET /api/assignments ──────────────────────────────────────── +// Flat listing of "who has access to what" rows from ResourceAssignments. +router.get('/assignments', async (req, res) => { + const { limit, offset, systemId } = parsePaging(req); + try { + const result = await runListAndCount({ + table: 'ResourceAssignments', + alias: 'ra', + columns: `ra."resourceId", ra."principalId", ra."assignmentType", ra."systemId", + ra."principalType", ra."complianceState", ra."policyId", ra."state", + ra."assignmentStatus", ra."expirationDateTime", ra."extendedAttributes"`, + orderBy: `ra."resourceId", ra."principalId", ra."assignmentType"`, + dataWhere: systemId !== null ? `WHERE ra."systemId" = $1` : '', + countWhere: systemId !== null ? `WHERE ra."systemId" = $1` : '', + systemId, limit, offset, + }); + res.json(result); + } catch (err) { + console.error('GET /assignments failed:', err.message); + res.status(500).json({ error: 'Internal server error' }); + } +}); + +// ─── GET /api/identity-members ─────────────────────────────────── +// Flat listing of identity ↔ principal links. Optional systemId filters by +// the principal's home system (IdentityMembers itself doesn't carry it). +router.get('/identity-members', async (req, res) => { + const { limit, offset, systemId } = parsePaging(req); + try { + const where = systemId !== null + ? `WHERE EXISTS (SELECT 1 FROM "Principals" p + WHERE p.id = im."principalId" AND p."systemId" = $1)` + : ''; + const result = await runListAndCount({ + table: 'IdentityMembers', + alias: 'im', + columns: `im."identityId", im."principalId", im."isPrimary", + im."isHrAuthoritative", im."accountType", im."accountTypePattern", + im."accountEnabled", im."displayName"`, + orderBy: `im."identityId", im."principalId"`, + dataWhere: where, countWhere: where, + systemId, limit, offset, + }); + res.json(result); + } catch (err) { + console.error('GET /identity-members failed:', err.message); + res.status(500).json({ error: 'Internal server error' }); + } +}); + +// ─── GET /api/resource-relationships ───────────────────────────── +// Flat listing of parent↔child resource links (Contains, GrantsAccessTo). +router.get('/resource-relationships', async (req, res) => { + const { limit, offset, systemId } = parsePaging(req); + try { + const where = systemId !== null ? `WHERE rr."systemId" = $1` : ''; + const result = await runListAndCount({ + table: 'ResourceRelationships', + alias: 'rr', + columns: `rr."parentResourceId", rr."childResourceId", rr."relationshipType", + rr."systemId", rr."roleName", rr."roleOriginSystem", + rr."extendedAttributes"`, + orderBy: `rr."parentResourceId", rr."childResourceId", rr."relationshipType"`, + dataWhere: where, countWhere: where, + systemId, limit, offset, + }); + res.json(result); + } catch (err) { + console.error('GET /resource-relationships failed:', err.message); + res.status(500).json({ error: 'Internal server error' }); + } +}); + +export default router; diff --git a/app/api/src/routes/dataExport.js b/app/api/src/routes/dataExport.js new file mode 100644 index 000000000..2f8a91141 --- /dev/null +++ b/app/api/src/routes/dataExport.js @@ -0,0 +1,105 @@ +// Read-only API tokens + Excel Power Query workbook download. +// +// Two distinct flows live behind /api/admin/data-export/*: +// +// 1. Token CRUD — create / list / revoke read API keys (`fgr_…`). Used by +// operators who want to plug Identity Atlas into other tools (Power BI, +// curl, custom scripts). +// +// 2. Workbook download — convenience flow that creates a token AND returns +// a pre-stamped Excel workbook in a single request, so a data analyst +// can go from "I want my data in Excel" to a working pivot table in +// under a minute. +// +// Both flows are guarded by authMiddleware (mounted in index.js) and are +// admin-scoped — the auth middleware additionally rejects `fgr_` tokens for +// any /api/admin/* path, so a stolen read token can't mint more tokens. + +import { Router } from 'express'; +import { createToken, listTokens, revokeToken } from '../auth/readTokens.js'; +import { generateWorkbook } from '../export/excelWorkbook.js'; + +const router = Router(); + +// ─── GET /api/admin/read-tokens ───────────────────────────────── +router.get('/admin/read-tokens', async (_req, res) => { + try { + res.json(await listTokens()); + } catch (err) { + console.error('list read tokens failed:', err.message); + res.status(500).json({ error: 'Internal server error' }); + } +}); + +// ─── POST /api/admin/read-tokens ──────────────────────────────── +// Body: { name: string, expiresAt?: ISO date } +// Returns: { token: 'fgr_…' (one-time), row: { id, name, ... } } +router.post('/admin/read-tokens', async (req, res) => { + try { + const { name, expiresAt } = req.body || {}; + if (!name || typeof name !== 'string' || name.trim().length === 0) { + return res.status(400).json({ error: 'name is required' }); + } + if (name.length > 200) return res.status(400).json({ error: 'name too long' }); + if (expiresAt && Number.isNaN(Date.parse(expiresAt))) { + return res.status(400).json({ error: 'expiresAt must be a valid ISO timestamp' }); + } + const createdBy = req.user?.preferred_username || req.user?.email || 'unknown'; + const result = await createToken({ name: name.trim(), createdBy, expiresAt }); + res.status(201).json(result); + } catch (err) { + console.error('create read token failed:', err.message); + res.status(500).json({ error: 'Internal server error' }); + } +}); + +// ─── DELETE /api/admin/read-tokens/:id ────────────────────────── +router.delete('/admin/read-tokens/:id', async (req, res) => { + const id = parseInt(req.params.id, 10); + if (Number.isNaN(id)) return res.status(400).json({ error: 'invalid id' }); + try { + const ok = await revokeToken(id); + if (!ok) return res.status(404).json({ error: 'not found' }); + res.json({ ok: true }); + } catch (err) { + console.error('revoke read token failed:', err.message); + res.status(500).json({ error: 'Internal server error' }); + } +}); + +// ─── POST /api/admin/data-export/workbook ─────────────────────── +// One-click flow: create a new read token AND return the pre-stamped +// .xlsx in the same request. Body: { name?: string }. Default name is +// derived from the requesting user / timestamp. +router.post('/admin/data-export/workbook', async (req, res) => { + try { + const requestedName = (req.body?.name && String(req.body.name).trim()) || ''; + const createdBy = req.user?.preferred_username || req.user?.email || 'unknown'; + const tokenName = requestedName || `Excel workbook (${createdBy}, ${new Date().toISOString().slice(0, 10)})`; + + const { token } = await createToken({ name: tokenName.slice(0, 200), createdBy }); + + // The workbook embeds the API base URL so the same file works against + // whatever host actually generated it (compose stack, prod deployment, + // tunnel, etc). Honour X-Forwarded-* if present, otherwise fall back + // to the request host. + const proto = req.get('x-forwarded-proto') || req.protocol; + const host = req.get('x-forwarded-host') || req.get('host'); + const apiBaseUrl = `${proto}://${host}/api`; + + const buffer = await generateWorkbook({ apiBaseUrl, token }); + + const filename = `IdentityAtlas-${new Date().toISOString().slice(0, 10)}.xlsx`; + res.set({ + 'Content-Type': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + 'Content-Disposition': `attachment; filename="${filename}"`, + 'Content-Length': buffer.length, + }); + res.send(buffer); + } catch (err) { + console.error('workbook generation failed:', err.message); + res.status(500).json({ error: 'Internal server error' }); + } +}); + +export default router; diff --git a/app/ui/src/components/AdminPage.jsx b/app/ui/src/components/AdminPage.jsx index 21bcb6eb4..f796faf8f 100644 --- a/app/ui/src/components/AdminPage.jsx +++ b/app/ui/src/components/AdminPage.jsx @@ -656,6 +656,235 @@ function CorrelationSection() { return
{content()}
; } +// ── Power Query workbook + read-API tokens ──────────────────────── +// Lets a tenant admin mint read-only API tokens (`fgr_…`) and download a +// pre-stamped Excel workbook with those credentials baked in. Tokens are +// shown in plaintext exactly once at creation; everywhere else we display +// just the prefix. Revoking a token here flips the `revoked` flag in the DB +// — refresh attempts from any workbook holding that token start failing +// immediately. +function PowerQueryExportSection() { + const { authFetch } = useAuth(); + const [tokens, setTokens] = useState([]); + const [loading, setLoading] = useState(true); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(null); + const [newToken, setNewToken] = useState(null); // plaintext shown once + const [showCreate, setShowCreate] = useState(false); + const [name, setName] = useState(''); + + const refresh = useCallback(async () => { + try { + const r = await authFetch('/api/admin/read-tokens'); + if (r.ok) setTokens(await r.json()); + } catch (e) { setError(e.message); } + setLoading(false); + }, [authFetch]); + + useEffect(() => { refresh(); }, [refresh]); + + async function downloadWorkbook() { + setError(null); + setBusy(true); + try { + const r = await authFetch('/api/admin/data-export/workbook', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({}), + }); + if (!r.ok) { + const d = await r.json().catch(() => ({})); + throw new Error(d.error || 'Workbook generation failed'); + } + const blob = await r.blob(); + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = `IdentityAtlas-${new Date().toISOString().slice(0, 10)}.xlsx`; + a.click(); + URL.revokeObjectURL(url); + await refresh(); + } catch (e) { + setError(e.message); + } finally { + setBusy(false); + } + } + + async function createTokenOnly() { + if (!name.trim()) return; + setError(null); + setBusy(true); + try { + const r = await authFetch('/api/admin/read-tokens', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ name: name.trim() }), + }); + if (!r.ok) { + const d = await r.json().catch(() => ({})); + throw new Error(d.error || 'Token creation failed'); + } + const data = await r.json(); + setNewToken(data.token); // plaintext, shown once + setShowCreate(false); + setName(''); + await refresh(); + } catch (e) { + setError(e.message); + } finally { + setBusy(false); + } + } + + async function revoke(id) { + if (!confirm('Revoke this token? Workbooks using it will stop refreshing immediately.')) return; + setBusy(true); + try { + const r = await authFetch(`/api/admin/read-tokens/${id}`, { method: 'DELETE' }); + if (!r.ok) throw new Error('Revoke failed'); + await refresh(); + } catch (e) { + setError(e.message); + } finally { + setBusy(false); + } + } + + function fmtDate(s) { return s ? new Date(s).toLocaleString() : '—'; } + + return ( +
+
+

+ Download a pre-configured Excel workbook with Power Query M code for every + object type (Users, Resources, Assignments, etc). The workbook includes a + read-only API token so refreshing the data on any user's machine just + requires opening the file. +

+ +
+ + +
+ + {showCreate && ( +
+ setName(e.target.value)} + onKeyDown={e => e.key === 'Enter' && createTokenOnly()} + placeholder="Token name (e.g. 'PowerBI prod report')" + className="px-2 py-1 border border-gray-300 rounded text-sm flex-1" + /> + + +
+ )} + + {newToken && ( +
+

⚠ Copy this token now — it will not be shown again

+ {newToken} + + +
+ )} + + {error && ( +
+ {error} +
+ )} + +
+

Existing tokens

+ {loading ? ( +

Loading…

+ ) : tokens.length === 0 ? ( +

No tokens issued yet.

+ ) : ( + + + + + + + + + + + + + {tokens.map(t => ( + + + + + + + + + ))} + +
NamePrefixCreatedLast usedStatus
{t.name}{t.tokenPrefix}…{fmtDate(t.createdAt)}{fmtDate(t.lastUsedAt)} + {t.revoked + ? Revoked + : t.expiresAt && new Date(t.expiresAt) < new Date() + ? Expired + : Active} + + {!t.revoked && ( + + )} +
+ )} +
+
+
+ ); +} + // ── Curated Data section ────────────────────────────────────────── function CuratedDataSection() { @@ -1656,6 +1885,7 @@ export default function AdminPage({ onNavigate, onRefresh, onRiskScoresRefresh } {activeTab === 'data' && ( <> + diff --git a/changes/feature-excel-powerquery-export.md b/changes/feature-excel-powerquery-export.md new file mode 100644 index 000000000..47f642074 --- /dev/null +++ b/changes/feature-excel-powerquery-export.md @@ -0,0 +1,4 @@ +- Added a one-click Excel Power Query workbook download under **Admin → Data → Excel Power Query Workbook**. Clicking "Generate token & download workbook" mints a read-only API key, embeds it in the workbook's Settings sheet alongside the API URL, and streams the file back. Data analysts can drop the file on any machine and refresh from any deployment without ever editing M code. +- Tabs included: Systems, Principals, Resources, Assignments, Identities, IdentityMembers, ResourceRelationships. Each tab includes pre-written paginated Power Query M code that the user pastes into Power Query Editor (Data → Get Data → Other Sources → Blank Query). A future iteration will swap in a hand-built template that loads queries automatically — the rest of the plumbing is already in place. +- New read-only API token type (`fgr_…`). Tokens are accepted on GET requests to non-admin endpoints only — they cannot mutate data or reach any admin endpoint, even if leaked. Stored as SHA-256 hashes; plaintext is shown to the operator exactly once at creation. Tokens can be revoked from the same admin section, which immediately stops every workbook holding them from refreshing. +- New bulk list endpoints for the join tables that previously had no flat listing: `/api/assignments`, `/api/identity-members`, `/api/resource-relationships`. All are paginated (`?limit=1000&offset=N`, max 10 000 per page) with optional `?systemId=N` filter, returning `{ data, total }` like every other list endpoint. From 274eeb6f92342c8bf6f8fe0a8caa197e64a6825f Mon Sep 17 00:00:00 2001 From: Taeke Kooiker Date: Sun, 19 Apr 2026 10:03:13 +0200 Subject: [PATCH 019/160] bugfix: use portable [0-9] instead of \d in cut-release version regex \d is a GNU grep extension not supported on all POSIX grep implementations, causing valid inputs like "5.2" to fail the Major.Minor validation check. Closes #104 Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/cut-release.yml | 2 +- changes/fix-cut-release-version-validation.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changes/fix-cut-release-version-validation.md diff --git a/.github/workflows/cut-release.yml b/.github/workflows/cut-release.yml index a519de22c..7d3604f01 100644 --- a/.github/workflows/cut-release.yml +++ b/.github/workflows/cut-release.yml @@ -31,7 +31,7 @@ jobs: steps: - name: Validate version input run: | - if ! echo "${{ github.event.inputs.version }}" | grep -qE '^\d+\.\d+$'; then + if ! echo "${{ github.event.inputs.version }}" | grep -qE '^[0-9]+\.[0-9]+$'; then echo "::error::Version must be in Major.Minor format (e.g. 5.2)" exit 1 fi diff --git a/changes/fix-cut-release-version-validation.md b/changes/fix-cut-release-version-validation.md new file mode 100644 index 000000000..02b37b369 --- /dev/null +++ b/changes/fix-cut-release-version-validation.md @@ -0,0 +1 @@ +- Fixed the Cut Release workflow rejecting valid version inputs like "5.2" due to a non-portable regex From 62b0431c06e5c01ddeb96731c19e52b422b89982 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 19 Apr 2026 08:03:49 +0000 Subject: [PATCH 020/160] chore: bump version to 5.4.20260419.0803 --- CHANGES.md | 4 ++++ changes/fix-cut-release-version-validation.md | 1 - setup/IdentityAtlas.psd1 | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) delete mode 100644 changes/fix-cut-release-version-validation.md diff --git a/CHANGES.md b/CHANGES.md index b2720a400..b65102f90 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,9 @@ ## Changes in this PR +- Fixed the Cut Release workflow rejecting valid version inputs like "5.2" due to a non-portable regex + +## Changes in this PR + - Fixed the "Filters" dropdown on the Users and Resources pages — attribute fields (Department, Job Title, etc.) and their values are populated again. The list had regressed to showing only "User Tag" because column discovery was looking up table names in the wrong case after the PostgreSQL migration. - Added regression tests pinning the PostgreSQL table and column casing used by column discovery, so the same mismatch can't slip back in. - Added extended-attribute fields to the Users and Resources filter dropdowns. Keys stored inside the `extendedAttributes` JSON blob (e.g. `userType`, `onPremisesSyncEnabled`, `extensionAttribute5`, `city`, `country`) are now filterable via the UI, labelled with a "(ext)" suffix so they're distinguishable from regular columns. diff --git a/changes/fix-cut-release-version-validation.md b/changes/fix-cut-release-version-validation.md deleted file mode 100644 index 02b37b369..000000000 --- a/changes/fix-cut-release-version-validation.md +++ /dev/null @@ -1 +0,0 @@ -- Fixed the Cut Release workflow rejecting valid version inputs like "5.2" due to a non-portable regex diff --git a/setup/IdentityAtlas.psd1 b/setup/IdentityAtlas.psd1 index 774e9f49f..2080af711 100644 --- a/setup/IdentityAtlas.psd1 +++ b/setup/IdentityAtlas.psd1 @@ -12,7 +12,7 @@ RootModule = '.\IdentityAtlas.psm1' # Version number of this module. -ModuleVersion = '5.3.20260418.1029' +ModuleVersion = '5.4.20260419.0803' # Supported PSEditions # CompatiblePSEditions = @() From 86b7fb1bf5ceacd5f23ae5880683162b248955eb Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 19 Apr 2026 08:04:55 +0000 Subject: [PATCH 021/160] chore: bump version to 5.5.20260419.0804 --- CHANGES.md | 7 +++++++ changes/feature-entraid-service-principals.md | 4 ---- setup/IdentityAtlas.psd1 | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) delete mode 100644 changes/feature-entraid-service-principals.md diff --git a/CHANGES.md b/CHANGES.md index b65102f90..822ba2676 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,12 @@ ## Changes in this PR +- Added Service Principals to the Entra ID crawler. When the "Service Principals" object type is selected, the crawler now pulls all Entra service principals (enterprise apps, managed identities, AI agents) and writes them to the Principals table alongside user accounts — unblocking future Azure RM role-assignment imports that reference these identities. +- New classification helper (`Get-FGServicePrincipalType`) tags each service principal as `ManagedIdentity`, `AIAgent`, or `ServicePrincipal` based on Graph's `servicePrincipalType`, well-known Microsoft AI platform tags (CopilotStudio, PowerVirtualAgents, AzureOpenAI, CognitiveServices), the Entra Agent ID markers (`AgenticInstance`, `AgenticApp`, `power-virtual-agents-*`), and display-name heuristics. Custom AI-name patterns can be supplied per crawler config via `aiNamePatterns`. +- Extra SP metadata (`appId`, `servicePrincipalType`, `publisherName`, `homepage`, `tags`, `servicePrincipalNames`, `notes`) is stored in `extendedAttributes` so it shows up in the Users filter dropdown and detail pages. +- Added principal-type sub-tabs to the Users page ("All / Users / Service Principals / Managed Identities / AI Agents") so the list stays navigable after an SP sync adds thousands of non-human identities. The active tab is preserved in the URL hash (`#users?type=AIAgent`). + +## Changes in this PR + - Fixed the Cut Release workflow rejecting valid version inputs like "5.2" due to a non-portable regex ## Changes in this PR diff --git a/changes/feature-entraid-service-principals.md b/changes/feature-entraid-service-principals.md deleted file mode 100644 index a3c7f5384..000000000 --- a/changes/feature-entraid-service-principals.md +++ /dev/null @@ -1,4 +0,0 @@ -- Added Service Principals to the Entra ID crawler. When the "Service Principals" object type is selected, the crawler now pulls all Entra service principals (enterprise apps, managed identities, AI agents) and writes them to the Principals table alongside user accounts — unblocking future Azure RM role-assignment imports that reference these identities. -- New classification helper (`Get-FGServicePrincipalType`) tags each service principal as `ManagedIdentity`, `AIAgent`, or `ServicePrincipal` based on Graph's `servicePrincipalType`, well-known Microsoft AI platform tags (CopilotStudio, PowerVirtualAgents, AzureOpenAI, CognitiveServices), the Entra Agent ID markers (`AgenticInstance`, `AgenticApp`, `power-virtual-agents-*`), and display-name heuristics. Custom AI-name patterns can be supplied per crawler config via `aiNamePatterns`. -- Extra SP metadata (`appId`, `servicePrincipalType`, `publisherName`, `homepage`, `tags`, `servicePrincipalNames`, `notes`) is stored in `extendedAttributes` so it shows up in the Users filter dropdown and detail pages. -- Added principal-type sub-tabs to the Users page ("All / Users / Service Principals / Managed Identities / AI Agents") so the list stays navigable after an SP sync adds thousands of non-human identities. The active tab is preserved in the URL hash (`#users?type=AIAgent`). diff --git a/setup/IdentityAtlas.psd1 b/setup/IdentityAtlas.psd1 index 2080af711..d4b6aea59 100644 --- a/setup/IdentityAtlas.psd1 +++ b/setup/IdentityAtlas.psd1 @@ -12,7 +12,7 @@ RootModule = '.\IdentityAtlas.psm1' # Version number of this module. -ModuleVersion = '5.4.20260419.0803' +ModuleVersion = '5.5.20260419.0804' # Supported PSEditions # CompatiblePSEditions = @() From f9fa78c16272cf42cd9a7cff6ee3a0ade39ca43a Mon Sep 17 00:00:00 2001 From: Taeke Kooiker Date: Sun, 19 Apr 2026 10:08:15 +0200 Subject: [PATCH 022/160] feat: add About page with license and software BOM Adds an About tab showing the MIT license and a Software Bill of Materials table. The version string in the footer is now a clickable link that navigates to the About page. Closes #86 Co-Authored-By: Claude Sonnet 4.6 --- app/ui/src/App.jsx | 11 +- app/ui/src/components/AboutPage.jsx | 161 ++++++++++++++++++++++++++++ changes/feature-about-page-sbom.md | 2 + 3 files changed, 173 insertions(+), 1 deletion(-) create mode 100644 app/ui/src/components/AboutPage.jsx create mode 100644 changes/feature-about-page-sbom.md diff --git a/app/ui/src/App.jsx b/app/ui/src/App.jsx index 2e0411400..41a0289c0 100644 --- a/app/ui/src/App.jsx +++ b/app/ui/src/App.jsx @@ -22,6 +22,7 @@ const ContextDetailPage = lazy(() => import('./components/ContextDetailPage')); const IdentitiesPage = lazy(() => import('./components/IdentitiesPage')); const IdentityDetailPage = lazy(() => import('./components/IdentityDetailPage')); const AdminPage = lazy(() => import('./components/AdminPage')); +const AboutPage = lazy(() => import('./components/AboutPage')); // PerfPage and CrawlersPage are lazy-loaded inside AdminPage as sub-tabs. // const GovernancePage = lazy(() => import('./components/GovernancePage')); // temporarily disabled @@ -96,6 +97,7 @@ const ALL_NAV_TABS = [ { key: 'identities', label: 'Identities', feature: 'accountCorrelation', optional: true }, { key: 'org-chart', label: 'Org Chart', optional: true }, { key: 'admin', label: 'Admin' }, + { key: 'about', label: 'About' }, ]; export default function App() { @@ -496,6 +498,8 @@ export default function App() { ) : page === 'org-chart' ? ( + ) : page === 'about' ? ( + ) : page === 'performance' || page === 'crawlers' || page === 'admin' ? ( // Crawlers and Performance now live under Admin as sub-tabs. // Legacy #crawlers and #performance hashes redirect to the matching sub-tab. @@ -530,7 +534,12 @@ export default function App() { {/* Footer */}
- Identity Atlas{moduleVersion ? ` v${moduleVersion}` : ''} + {/^\d+\.\d+\.\d{8}\.\d{4}$/.test(moduleVersion) && ( edge diff --git a/app/ui/src/components/AboutPage.jsx b/app/ui/src/components/AboutPage.jsx new file mode 100644 index 000000000..9e09585ba --- /dev/null +++ b/app/ui/src/components/AboutPage.jsx @@ -0,0 +1,161 @@ +const MIT_LICENSE = `MIT License + +Copyright (c) 2025 Fortigi + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE.`; + +const SBOM_SECTIONS = [ + { + title: 'Infrastructure', + rows: [ + { name: 'PostgreSQL', version: '16-alpine', purpose: 'Database server', license: 'PostgreSQL License' }, + { name: 'PowerShell', version: '7.4 (ubuntu-22.04)',purpose: 'Crawler runtime and scripting engine', license: 'MIT' }, + { name: 'Node.js', version: 'Latest LTS', purpose: 'API server runtime', license: 'MIT' }, + { name: 'Docker', version: '20.10+', purpose: 'Container orchestration', license: 'Apache 2.0' }, + ], + }, + { + title: 'API Backend (Node.js)', + rows: [ + { name: 'express', version: '^4.21.0', purpose: 'Web application framework', license: 'MIT' }, + { name: 'pg', version: '^8.13.1', purpose: 'PostgreSQL client', license: 'MIT' }, + { name: 'pg-copy-streams', version: '^7.0.0', purpose: 'High-performance bulk import', license: 'MIT' }, + { name: 'helmet', version: '^8.1.0', purpose: 'Security headers middleware', license: 'MIT' }, + { name: 'express-rate-limit', version: '^8.2.1', purpose: 'Rate limiting protection', license: 'MIT' }, + { name: 'cors', version: '^2.8.5', purpose: 'Cross-Origin Resource Sharing', license: 'MIT' }, + { name: 'jsonwebtoken', version: '^9.0.2', purpose: 'JWT token validation', license: 'MIT' }, + { name: 'jwks-rsa', version: '^3.1.0', purpose: 'JWKS key retrieval for Entra ID', license: 'MIT' }, + { name: 'multer', version: '^1.4.5-lts.1', purpose: 'CSV upload handling', license: 'MIT' }, + { name: 'swagger-ui-express', version: '^5.0.1', purpose: 'API documentation UI', license: 'Apache 2.0' }, + { name: 'yamljs', version: '^0.3.0', purpose: 'YAML parsing for OpenAPI specs', license: 'MIT' }, + ], + }, + { + title: 'Frontend (React)', + rows: [ + { name: 'react', version: '^19.2.0', purpose: 'UI framework', license: 'MIT' }, + { name: 'react-dom', version: '^19.2.0', purpose: 'React DOM renderer', license: 'MIT' }, + { name: 'vite', version: '^7.3.1', purpose: 'Build tool and dev server', license: 'MIT' }, + { name: 'tailwindcss', version: '^4.1.18', purpose: 'Utility-first CSS framework', license: 'MIT' }, + { name: '@azure/msal-browser', version: '^4.12.0', purpose: 'Microsoft Authentication Library', license: 'MIT' }, + { name: '@dnd-kit/core', version: '^6.3.1', purpose: 'Drag-and-drop core', license: 'MIT' }, + { name: '@tanstack/react-virtual', version: '^3.13.18', purpose: 'Virtual scrolling for large tables', license: 'MIT' }, + { name: 'exceljs', version: '^4.4.0', purpose: 'Excel spreadsheet generation', license: 'MIT' }, + ], + }, +]; + +function SbomTable({ rows }) { + return ( +
+ + + + + + + + + + + {rows.map((row) => ( + + + + + + + ))} + +
PackageVersionPurposeLicense
{row.name}{row.version}{row.purpose}{row.license}
+
+ ); +} + +export default function AboutPage() { + return ( +
+ {/* Header */} +
+

About Identity Atlas

+

+ Identity Atlas is an open-source role-mining and identity governance platform built by{' '} + Fortigi. + It pulls authorization data from Microsoft Graph and other systems into a PostgreSQL database and + surfaces it through a React role-mining UI. +

+ +
+ + {/* License */} +
+

License

+
+          {MIT_LICENSE}
+        
+
+ + {/* Software BOM */} +
+

Software Bill of Materials

+

+ All direct dependencies use permissive open-source licenses (MIT, Apache 2.0, or PostgreSQL License). +

+
+ {SBOM_SECTIONS.map((section) => ( +
+

+ {section.title} +

+ +
+ ))} +
+
+
+ ); +} diff --git a/changes/feature-about-page-sbom.md b/changes/feature-about-page-sbom.md new file mode 100644 index 000000000..b825bb489 --- /dev/null +++ b/changes/feature-about-page-sbom.md @@ -0,0 +1,2 @@ +- Added an About page showing the MIT license text and Software Bill of Materials +- The version string in the footer is now a clickable link to the About page From 14ed7182485eca1fc4696e608a0f0e45eab28f0b Mon Sep 17 00:00:00 2001 From: Taeke Kooiker Date: Sun, 19 Apr 2026 10:10:40 +0200 Subject: [PATCH 023/160] fix: restore full copyright line in About page to match LICENSE file Co-Authored-By: Claude Sonnet 4.6 --- app/ui/src/components/AboutPage.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/ui/src/components/AboutPage.jsx b/app/ui/src/components/AboutPage.jsx index 9e09585ba..c455da908 100644 --- a/app/ui/src/components/AboutPage.jsx +++ b/app/ui/src/components/AboutPage.jsx @@ -1,6 +1,6 @@ const MIT_LICENSE = `MIT License -Copyright (c) 2025 Fortigi +Copyright (c) 2025 Wim van den Heijkant / Fortigi Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal From db805149d50f5961a379530a9ba1935e51ac0ae6 Mon Sep 17 00:00:00 2001 From: Taeke Kooiker Date: Sun, 19 Apr 2026 10:35:42 +0200 Subject: [PATCH 024/160] docs: documentation consistency pass - Add branching strategy reference page under docs/architecture - Add --pull always to all Quick Start commands (README, quickstart, docker-setup, index) - Add .env setup step to all Quick Start sections - Add tabbed Linux/macOS vs Windows code blocks in docker-setup and local-dev - Fix version pinning example to use release format (5.2.0.0) not edge timestamp - Fix history.md to link to branching-strategy.md instead of CLAUDE.md - Align local-dev.md with docker-setup.md (--build on first run, tabbed stop/reset) Co-Authored-By: Claude Sonnet 4.6 --- README.md | 5 +- changes/feature-docs-consistency-pass.md | 7 ++ docs/architecture/branching-strategy.md | 113 +++++++++++++++++++++++ docs/architecture/docker-setup.md | 107 +++++++++++++++------ docs/history.md | 2 +- docs/index.md | 10 +- docs/quickstart.md | 4 +- docs/ui/local-dev.md | 62 +++++++++++-- 8 files changed, 264 insertions(+), 46 deletions(-) create mode 100644 changes/feature-docs-consistency-pass.md create mode 100644 docs/architecture/branching-strategy.md diff --git a/README.md b/README.md index d599bc6c8..247183c52 100644 --- a/README.md +++ b/README.md @@ -20,8 +20,9 @@ cp .env.example .env # POSTGRES_PASSWORD= # IDENTITY_ATLAS_MASTER_KEY= -# 3. Start the stack (first run: ~2 min to pull images) -docker compose -f docker-compose.prod.yml up -d +# 3. Start the stack (first run: ~2 min to pull images; --pull always ensures +# Docker fetches the newest :latest instead of reusing a cached copy) +docker compose -f docker-compose.prod.yml up -d --pull always # 4. Open http://localhost:3001 # Go to Admin > Crawlers, then click "Load Demo Data" to explore with sample data, or diff --git a/changes/feature-docs-consistency-pass.md b/changes/feature-docs-consistency-pass.md new file mode 100644 index 000000000..efe4baba4 --- /dev/null +++ b/changes/feature-docs-consistency-pass.md @@ -0,0 +1,7 @@ +- Added branching and versioning strategy reference page under docs/architecture +- Added `--pull always` flag to Quick Start commands in README, quickstart, docker-setup, and index docs so users always get the newest image +- Added `.env` setup step (copy from template) to all Quick Start sections +- Added tabbed Linux/macOS and Windows code blocks throughout docker-setup and local-dev docs +- Fixed version pinning example in quickstart to use release format (5.2.0.0) instead of edge timestamp format +- Fixed history.md to link to the branching strategy doc instead of referencing CLAUDE.md +- Aligned local-dev.md with docker-setup.md: added --build note for first run, tabbed stop/reset commands diff --git a/docs/architecture/branching-strategy.md b/docs/architecture/branching-strategy.md new file mode 100644 index 000000000..4df5b6da3 --- /dev/null +++ b/docs/architecture/branching-strategy.md @@ -0,0 +1,113 @@ +# Branching and Versioning Strategy + +This document covers branch naming, PR rules, version format, and the changelog workflow for contributors. + +--- + +## Branch Model + +| Branch | Purpose | PR required? | Approval required? | +|--------|---------|-------------|-------------------| +| `main` | Stable trunk. Never commit directly. | Yes | Yes (at least 1) | +| `feature/` | All feature work. Created from `main`. Merged back to `main` via PR. | Yes | No | +| `bugfixes/` | Bug fixes. Created from `main`. Merged back to `main` via PR. | Yes | No | + +**Rules:** + +- `feature/` and `bugfixes/` branches must be branched off `main`. +- All merges to `main` go through a Pull Request — no direct pushes. +- Branch names: lowercase, hyphens. Examples: `feature/risk-score-export`, `bugfixes/fix-login-redirect`. +- **One issue per branch.** Each branch fixes exactly one issue or implements exactly one feature. Never combine unrelated fixes into a single branch or PR. + +--- + +## Starting New Work + +```bash +git checkout main && git pull +git checkout -b feature/ +# or +git checkout -b bugfixes/ +``` + +--- + +## Version Number Format + +``` +Major.Minor.yyyyMMdd.HHmm +``` + +Example: `5.2.20260420.1430` + +| Part | Meaning | +|------|---------| +| `Major` | Incremented manually for breaking changes (via a PR to `main`) | +| `Minor` | Auto-incremented by CI on every PR merge to `main` | +| `yyyyMMdd.HHmm` | Timestamp of the merge, set by CI | + +**Who updates what:** + +| Action | Who | When | +|--------|-----|------| +| `Minor` bump + timestamp | `bump-version.yml` GitHub Action | Automatically on every PR merge to `main` | +| `Major` bump | Developer, via PR | Only for breaking changes | +| Branch work | Nobody | Never touch `setup/IdentityAtlas.psd1` on a branch | + +--- + +## Changelog Fragments + +Every `feature/` or `bugfixes/` branch must include a changelog fragment. **Never edit `CHANGES.md` directly** — the `bump-version.yml` CI action merges all fragments on PR merge. + +**File:** `changes/.md` (e.g. `changes/fix-login-redirect.md`) + +**Format:** + +```markdown +- Fixed the login redirect when auth is enabled and no session exists +- Improved error message when tenant ID is missing +``` + +Write in user-facing language. One bullet per functional change. Add the file alongside the code change — don't batch at the end. + +--- + +## Merging to Main (via PR) + +1. Open a PR from `feature/` or `bugfixes/` into `main`. +2. Use the changelog fragment content as the PR description body. +3. Requires 1 approval and passing CI. +4. After merge, `bump-version.yml` automatically increments `Minor`, updates the timestamp, and merges all `changes/*.md` fragments into `CHANGES.md`. The `docker-publish.yml` action then builds and pushes Docker images tagged with the new version. + +--- + +## Stacked PRs + +For larger features, break the work into a stack of small focused PRs. Each PR targets the previous branch in the stack: + +```bash +# Step 1 — targets main +git checkout -b feature/foo-step-1 +gh pr create --base main --title "step 1: ..." + +# Step 2 — stacked on step 1 +git checkout -b feature/foo-step-2 +gh pr create --base feature/foo-step-1 --title "step 2: ..." +``` + +When a bottom PR merges, retarget the next one: `gh pr edit --base main`. + +--- + +## Image Channels + +The CI pipeline publishes Docker images on every merge to `main`: + +| Tag | Content | Who uses it | +|-----|---------|-------------| +| `:latest` | Last stable release | End users (default) | +| `:edge` | Latest commit on `main` | Testers and developers | +| `:5.2.0.0` | Exact pinned version | Production deployments | + +See [Docker Setup](docker-setup.md) for how to select a channel via `IMAGE_TAG`. diff --git a/docs/architecture/docker-setup.md b/docs/architecture/docker-setup.md index 77b2d1c13..e72e4d2a3 100644 --- a/docs/architecture/docker-setup.md +++ b/docs/architecture/docker-setup.md @@ -8,20 +8,43 @@ Running Identity Atlas locally with Docker — three containers providing the fu The fastest way to try Identity Atlas — pulls pre-built images, no source code needed: -```bash -# 1. Download the compose file and environment template -curl -O https://raw.githubusercontent.com/Fortigi/IdentityAtlas/main/docker-compose.prod.yml -curl -O https://raw.githubusercontent.com/Fortigi/IdentityAtlas/main/setup/config/.env.example +=== "Linux / macOS" -# 2. Create your .env file -cp .env.example .env + ```bash + # 1. Download the compose file and environment template + curl -O https://raw.githubusercontent.com/Fortigi/IdentityAtlas/main/docker-compose.prod.yml + curl -O https://raw.githubusercontent.com/Fortigi/IdentityAtlas/main/setup/config/.env.example -# 3. Start everything (first run: ~2 min to pull images) -docker compose -f docker-compose.prod.yml up -d + # 2. Create your .env file + cp .env.example .env -# 4. Open the UI -open http://localhost:3001 -``` + # 3. Start everything (--pull always fetches the newest :latest from the registry) + docker compose -f docker-compose.prod.yml up -d --pull always + + # 4. Open the UI + open http://localhost:3001 + ``` + +=== "Windows (PowerShell)" + + ```powershell + # 1. Download the compose file and environment template + Invoke-WebRequest ` + -Uri https://raw.githubusercontent.com/Fortigi/IdentityAtlas/main/docker-compose.prod.yml ` + -OutFile docker-compose.prod.yml + Invoke-WebRequest ` + -Uri https://raw.githubusercontent.com/Fortigi/IdentityAtlas/main/setup/config/.env.example ` + -OutFile .env.example + + # 2. Create your .env file + Copy-Item .env.example .env + + # 3. Start everything (--pull always fetches the newest :latest from the registry) + docker compose -f docker-compose.prod.yml up -d --pull always + + # 4. Open the UI + Start-Process http://localhost:3001 + ``` On first visit, the UI opens to the Dashboard. If no data is loaded yet, click **"Configure a crawler"** to go to Admin → Crawlers, then click **"Load Demo Data"** to populate the system with synthetic data (~30 seconds). After that, explore the Matrix, Users, Resources, and other pages. @@ -52,10 +75,10 @@ The running version is always visible in the footer of the UI. Edge builds show ```bash # Run the stable release (default) -docker compose -f docker-compose.prod.yml up -d +docker compose -f docker-compose.prod.yml up -d --pull always # Run the edge build -IMAGE_TAG=edge docker compose -f docker-compose.prod.yml up -d +IMAGE_TAG=edge docker compose -f docker-compose.prod.yml up -d --pull always # or set IMAGE_TAG=edge in your .env ``` @@ -102,27 +125,53 @@ happens inside the web container at startup via the migrations runner ## Quick Start (Developer) -```powershell -cd c:\Source\GitHub\IdentityAtlas +=== "Linux / macOS" -# Create your .env file from the template -cp setup/config/.env.example .env -# IMAGE_TAG is ignored by the dev compose (it builds from source). -# You can leave the other defaults as-is for local development. + ```bash + cd /path/to/IdentityAtlas -# Start the stack (first time takes ~3 min to build) -docker compose up -d --build + # Create your .env file from the template + cp setup/config/.env.example .env + # IMAGE_TAG is ignored by the dev compose (it builds from source). + # You can leave the other defaults as-is for local development. -# Verify -docker compose ps -# Expected: postgres (healthy), web (up), worker (up) + # Start the stack (first time takes ~3 min to build) + docker compose up -d --build -# Open the UI — click "Load Demo Data" on the Crawlers page -Start-Process http://localhost:3001 + # Verify + docker compose ps + # Expected: postgres (healthy), web (up), worker (up) -# Open Swagger docs -Start-Process http://localhost:3001/api/docs -``` + # Open the UI — click "Load Demo Data" on the Crawlers page + open http://localhost:3001 + + # Open Swagger docs + open http://localhost:3001/api/docs + ``` + +=== "Windows (PowerShell)" + + ```powershell + cd C:\path\to\IdentityAtlas + + # Create your .env file from the template + Copy-Item setup/config/.env.example .env + # IMAGE_TAG is ignored by the dev compose (it builds from source). + # You can leave the other defaults as-is for local development. + + # Start the stack (first time takes ~3 min to build) + docker compose up -d --build + + # Verify + docker compose ps + # Expected: postgres (healthy), web (up), worker (up) + + # Open the UI — click "Load Demo Data" on the Crawlers page + Start-Process http://localhost:3001 + + # Open Swagger docs + Start-Process http://localhost:3001/api/docs + ``` > **Note:** `docker-compose.yml` (dev) builds images from source — `IMAGE_TAG` has no effect. Use `docker-compose.prod.yml` with `IMAGE_TAG=edge` if you want to run the pre-built edge image without a local build. diff --git a/docs/history.md b/docs/history.md index 32cc9e832..6dc2ca882 100644 --- a/docs/history.md +++ b/docs/history.md @@ -89,7 +89,7 @@ The v5 rewrite was executed in a single ~2-week feature branch (`feature/univers ## Version numbers -v5 resets the major version. `ModuleVersion` follows `Major.Minor.yyyyMMdd.HHmm` per the branching strategy in CLAUDE.md. The `CHANGELOG.md` at the repo root is the authoritative per-release history; this document is the narrative. +v5 resets the major version. `ModuleVersion` follows `Major.Minor.yyyyMMdd.HHmm` per the [branching and versioning strategy](architecture/branching-strategy.md). The `CHANGELOG.md` at the repo root is the authoritative per-release history; this document is the narrative. ## Acknowledgements diff --git a/docs/index.md b/docs/index.md index 9d8f50f68..f5c6a6abc 100644 --- a/docs/index.md +++ b/docs/index.md @@ -60,11 +60,15 @@ A 4-layer scoring engine that classifies principals by risk without sending sens **Prerequisite:** Docker. ```bash -# Download the production compose file +# Download the compose file and environment template curl -O https://raw.githubusercontent.com/Fortigi/IdentityAtlas/main/docker-compose.prod.yml +curl -O https://raw.githubusercontent.com/Fortigi/IdentityAtlas/main/setup/config/.env.example -# Start the stack -docker compose -f docker-compose.prod.yml up -d +# Create your .env file (defaults are fine for local evaluation) +cp .env.example .env + +# Start the stack (--pull always fetches the newest :latest from the registry) +docker compose -f docker-compose.prod.yml up -d --pull always ``` Open [http://localhost:3001](http://localhost:3001) → click **"Load Demo Data"** for instant gratification, or **"Connect Entra ID"** to wire up your own tenant via the in-browser wizard. diff --git a/docs/quickstart.md b/docs/quickstart.md index c516c7358..fa8a728bd 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -116,8 +116,8 @@ image: ghcr.io/fortigi/identity-atlas-worker:latest with the explicit version tag: ```yaml -image: ghcr.io/fortigi/identity-atlas:5.0.20260411.1955 -image: ghcr.io/fortigi/identity-atlas-worker:5.0.20260411.1955 +image: ghcr.io/fortigi/identity-atlas:5.2.0.0 +image: ghcr.io/fortigi/identity-atlas-worker:5.2.0.0 ``` Both images are always published with the same version tag, so they'll stay in sync. diff --git a/docs/ui/local-dev.md b/docs/ui/local-dev.md index beb0c30b6..9253b487b 100644 --- a/docs/ui/local-dev.md +++ b/docs/ui/local-dev.md @@ -21,9 +21,41 @@ The PostgreSQL port `5432` is exposed to the host for direct database access dur ## Start the Stack -```bash -docker compose up -d -``` +First, create your `.env` file from the template (only needed once): + +=== "Linux / macOS" + + ```bash + cp setup/config/.env.example .env + ``` + +=== "Windows (PowerShell)" + + ```powershell + Copy-Item setup/config/.env.example .env + ``` + +Then start the stack. Use `--build` on the first run (or after pulling new commits) to build the images from source: + +=== "Linux / macOS" + + ```bash + # First run or after code changes — builds images from source (~3 min) + docker compose up -d --build + + # Subsequent runs (images already built) + docker compose up -d + ``` + +=== "Windows (PowerShell)" + + ```powershell + # First run or after code changes — builds images from source (~3 min) + docker compose up -d --build + + # Subsequent runs (images already built) + docker compose up -d + ``` This starts: @@ -52,13 +84,25 @@ The worker runs every minute and picks up queued jobs. Live progress is shown on ## Stopping and Resetting -```bash -# Stop the stack (data persists in the postgres_data volume) -docker compose down +=== "Linux / macOS" -# Stop and delete all data (full reset) -docker compose down -v -``` + ```bash + # Stop the stack (data persists in the postgres_data volume) + docker compose down + + # Stop and delete all data (full reset) + docker compose down -v + ``` + +=== "Windows (PowerShell)" + + ```powershell + # Stop the stack (data persists in the postgres_data volume) + docker compose down + + # Stop and delete all data (full reset) + docker compose down -v + ``` ## Building the Image Manually From 990f5cf67b2ea1c929706621791b67c74996dfff Mon Sep 17 00:00:00 2001 From: Taeke Kooiker Date: Sun, 19 Apr 2026 10:58:33 +0200 Subject: [PATCH 025/160] feat: replace release branches with tag-based release model MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hotfixes to a released version now branch from the release tag (git checkout -b bugfixes/fix-foo v5.2.0) rather than from a release/vX.Y branch. This means a hotfix ships only the fix — features already merged to main are not included. Changes: - cut-release.yml: creates a vX.Y.Z tag on main HEAD instead of a release branch. Input is now Major.Minor.Patch (e.g. 5.2.0). - cut-hotfix.yml (new): tags the HEAD of a hotfix branch as a new patch version, triggering docker-publish. - docker-publish.yml: triggers on v* tag pushes. Derives the version from the tag name (v5.2.0 → 5.2.0.0) instead of reading psd1. - bump-version.yml: simplified — release branch logic removed entirely. - setup-branch-protection.sh: removes release/** ruleset, adds cleanup step for any existing legacy ruleset. - docs/architecture/branching-strategy.md: rewritten for the new model. - CLAUDE.md: branching table, version scheme, and workflow sections updated. Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/bump-version.yml | 70 +++-------- .github/workflows/cut-hotfix.yml | 78 ++++++++++++ .github/workflows/cut-release.yml | 92 +++++++------- .github/workflows/docker-publish.yml | 56 ++++++--- CLAUDE.md | 101 ++++++++------- changes/feature-tag-based-releases.md | 4 + docs/architecture/branching-strategy.md | 159 ++++++++++++++++++++++++ tools/setup-branch-protection.sh | 76 +++-------- 8 files changed, 406 insertions(+), 230 deletions(-) create mode 100644 .github/workflows/cut-hotfix.yml create mode 100644 changes/feature-tag-based-releases.md create mode 100644 docs/architecture/branching-strategy.md diff --git a/.github/workflows/bump-version.yml b/.github/workflows/bump-version.yml index 2a1e76bbf..fcf9c036c 100644 --- a/.github/workflows/bump-version.yml +++ b/.github/workflows/bump-version.yml @@ -1,17 +1,18 @@ # ─── Version Bump + Changelog Merge on PR Merge ────────────────────────────── -# On every PR merge this workflow: +# On every PR merge to main this workflow: # 1. Collects all fragment files from changes/*.md and prepends them to # CHANGES.md, then deletes the fragments. -# 2. Bumps the version in setup/IdentityAtlas.psd1: -# - main: increments Minor, updates timestamp → Major.Minor.yyyyMMdd.HHmm -# - release/v*: increments Patch → Major.Minor.Patch.0 +# 2. Increments Minor and updates the timestamp → Major.Minor.yyyyMMdd.HHmm # # Branches never touch CHANGES.md or the version file directly — each branch # creates a uniquely named fragment file in changes/ instead, so merge # conflicts on those two files are eliminated. # # After the commit lands, docker-publish.yml is triggered via workflow_run -# so Docker images are always tagged with the new (post-bump) version. +# so Docker images are always tagged with the new (post-bump) version (:edge). +# +# Releases are handled separately via git tags (Actions → Cut Release), +# not by this workflow. # ───────────────────────────────────────────────────────────────────────────── name: Bump version on PR merge @@ -21,7 +22,6 @@ on: types: [closed] branches: - main - - 'release/**' jobs: bump-version: @@ -31,27 +31,14 @@ jobs: contents: write steps: - - name: Determine target branch - id: branch - run: | - TARGET="${{ github.event.pull_request.base.ref }}" - echo "name=$TARGET" >> "$GITHUB_OUTPUT" - if [[ "$TARGET" == release/* ]]; then - echo "type=release" >> "$GITHUB_OUTPUT" - else - echo "type=main" >> "$GITHUB_OUTPUT" - fi - - uses: actions/checkout@v4 with: - ref: ${{ steps.branch.outputs.name }} + ref: main token: ${{ secrets.VERSION_BUMP_PAT }} - name: Merge changelog fragments and bump version shell: pwsh run: | - $branchType = "${{ steps.branch.outputs.type }}" - # ── 1. Collect changelog fragments from changes/*.md ────────────── $fragments = Get-ChildItem changes/*.md -ErrorAction SilentlyContinue | Sort-Object Name if ($fragments) { @@ -65,39 +52,20 @@ jobs: Write-Host "No changelog fragments found in changes/ -- skipping CHANGES.md update" } - # ── 2. Bump version in setup/IdentityAtlas.psd1 ─────────────────── + # ── 2. Increment Minor + update timestamp → Major.Minor.yyyyMMdd.HHmm $content = Get-Content setup/IdentityAtlas.psd1 -Raw - - if ($branchType -eq 'release') { - # Release branch: increment Patch → Major.Minor.Patch.0 - if ($content -match "ModuleVersion\s*=\s*'(\d+)\.(\d+)\.(\d+)\.(\d+)'") { - $major = $Matches[1] - $minor = $Matches[2] - $patch = [int]$Matches[3] + 1 - $newVer = "$major.$minor.$patch.0" - $content = $content -replace "ModuleVersion\s*=\s*'\d+\.\d+\.\d+\.\d+'", "ModuleVersion = '$newVer'" - Set-Content setup/IdentityAtlas.psd1 $content -NoNewline - Write-Host "Bumped to $newVer (release patch)" - "NEW_VERSION=$newVer" | Out-File -Append $env:GITHUB_ENV - } else { - Write-Error "Could not find ModuleVersion in setup/IdentityAtlas.psd1" - exit 1 - } + if ($content -match "ModuleVersion\s*=\s*'(\d+)\.(\d+)\.\d+\.\d+'") { + $major = $Matches[1] + $minor = [int]$Matches[2] + 1 + $stamp = (Get-Date -Format 'yyyyMMdd.HHmm') + $newVer = "$major.$minor.$stamp" + $content = $content -replace "ModuleVersion\s*=\s*'\d+\.\d+\.\d+\.\d+'", "ModuleVersion = '$newVer'" + Set-Content setup/IdentityAtlas.psd1 $content -NoNewline + Write-Host "Bumped to $newVer" + "NEW_VERSION=$newVer" | Out-File -Append $env:GITHUB_ENV } else { - # Main branch: increment Minor, update timestamp → Major.Minor.yyyyMMdd.HHmm - if ($content -match "ModuleVersion\s*=\s*'(\d+)\.(\d+)\.\d+\.\d+'") { - $major = $Matches[1] - $minor = [int]$Matches[2] + 1 - $stamp = (Get-Date -Format 'yyyyMMdd.HHmm') - $newVer = "$major.$minor.$stamp" - $content = $content -replace "ModuleVersion\s*=\s*'\d+\.\d+\.\d+\.\d+'", "ModuleVersion = '$newVer'" - Set-Content setup/IdentityAtlas.psd1 $content -NoNewline - Write-Host "Bumped to $newVer (main dev build)" - "NEW_VERSION=$newVer" | Out-File -Append $env:GITHUB_ENV - } else { - Write-Error "Could not find ModuleVersion in setup/IdentityAtlas.psd1" - exit 1 - } + Write-Error "Could not find ModuleVersion in setup/IdentityAtlas.psd1" + exit 1 } - name: Commit and push diff --git a/.github/workflows/cut-hotfix.yml b/.github/workflows/cut-hotfix.yml new file mode 100644 index 000000000..a26620a91 --- /dev/null +++ b/.github/workflows/cut-hotfix.yml @@ -0,0 +1,78 @@ +# ─── Cut a Hotfix Release ───────────────────────────────────────────────────── +# Manually triggered. Tags the HEAD of a hotfix branch with a new patch version, +# triggering docker-publish to build :latest + :X.Y.Z.0. +# +# Usage: +# 1. git checkout -b bugfixes/fix-foo v5.2.0 ← branch from the release tag +# 2. Fix the bug, push the branch +# 3. Go to Actions → Cut Hotfix → Run workflow +# 4. Enter the branch name and new version (e.g. "5.2.1") +# 5. After the hotfix ships, open a PR to cherry-pick the fix into main +# ───────────────────────────────────────────────────────────────────────────── + +name: Cut Hotfix + +on: + workflow_dispatch: + inputs: + branch: + description: 'Hotfix branch name (e.g. bugfixes/fix-login-crash)' + required: true + version: + description: 'New version (major.minor.patch, e.g. "5.2.1")' + required: true + +jobs: + cut-hotfix: + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - name: Validate version input + run: | + if ! echo "${{ github.event.inputs.version }}" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$'; then + echo "::error::Version must be in Major.Minor.Patch format (e.g. 5.2.1)" + exit 1 + fi + + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.inputs.branch }} + token: ${{ secrets.VERSION_BUMP_PAT }} + + - name: Create and push hotfix tag + run: | + VERSION="${{ github.event.inputs.version }}" + TAG="v${VERSION}" + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git tag -a "$TAG" -m "Hotfix release $TAG" + git push origin "$TAG" + echo "TAG=$TAG" >> "$GITHUB_ENV" + echo "VERSION=$VERSION" >> "$GITHUB_ENV" + COMMIT=$(git rev-parse --short HEAD) + echo "COMMIT=$COMMIT" >> "$GITHUB_ENV" + echo "✅ Tagged ${TAG} on ${COMMIT}" + + - name: Post run summary + run: | + cat >> "$GITHUB_STEP_SUMMARY" << EOF + ## ✅ Hotfix tagged + + | | | + |---|---| + | **Tag** | \`${TAG}\` | + | **Version** | \`${VERSION}.0\` | + | **Branch** | \`${{ github.event.inputs.branch }}\` | + | **Commit** | \`${COMMIT}\` | + + docker-publish is now building \`:latest\` + \`:${VERSION}.0\` — check the [Actions tab](../../actions) for progress. + + ### Next step — cherry-pick to main + \`\`\`bash + git checkout main && git pull + git cherry-pick ${COMMIT} + gh pr create --base main --title "fix: cherry-pick hotfix from ${TAG}" + \`\`\` + EOF diff --git a/.github/workflows/cut-release.yml b/.github/workflows/cut-release.yml index 7d3604f01..45e48d686 100644 --- a/.github/workflows/cut-release.yml +++ b/.github/workflows/cut-release.yml @@ -1,24 +1,26 @@ -# ─── Cut a Release Branch ───────────────────────────────────────────────────── -# Manually triggered. Creates release/vX.Y from main, sets the version to -# X.Y.0.0, then triggers docker-publish to build and push the initial -# :latest image for that version. +# ─── Cut a Release ──────────────────────────────────────────────────────────── +# Manually triggered. Tags the current main HEAD with vX.Y.Z and lets +# docker-publish (triggered by the tag push) build and push :latest + :X.Y.Z.0. # # Usage: -# 1. Go to Actions → Cut Release Branch → Run workflow -# 2. Enter the version (e.g. "5.2" — major.minor only, no patch) -# 3. The workflow creates release/v5.2, sets ModuleVersion = 5.2.0.0, -# and publishes ghcr.io/fortigi/identity-atlas:latest + :5.2.0.0 -# 4. Bugfixes targeting that release are PRed into release/v5.2 -# 5. Each merge bumps patch and republishes: 5.2.0.0 → 5.2.1.0 → 5.2.2.0 ... +# 1. Go to Actions → Cut Release → Run workflow +# 2. Enter the version (e.g. "5.2.0" — major.minor.patch) +# 3. The workflow creates tag v5.2.0 on the current main HEAD +# 4. docker-publish builds and pushes :latest + :5.2.0.0 +# +# Hotfixes (shipping a bugfix without including features already on main): +# 1. git checkout -b bugfixes/fix-foo v5.2.0 ← branch from the tag, not main +# 2. Fix the bug, push the branch +# 3. Run Actions → Cut Hotfix with the branch name and new version (e.g. 5.2.1) # ───────────────────────────────────────────────────────────────────────────── -name: Cut Release Branch +name: Cut Release on: workflow_dispatch: inputs: version: - description: 'Release version (major.minor only, e.g. "5.2")' + description: 'Release version (major.minor.patch, e.g. "5.2.0")' required: true jobs: @@ -26,13 +28,12 @@ jobs: runs-on: ubuntu-latest permissions: contents: write - actions: write steps: - name: Validate version input run: | - if ! echo "${{ github.event.inputs.version }}" | grep -qE '^[0-9]+\.[0-9]+$'; then - echo "::error::Version must be in Major.Minor format (e.g. 5.2)" + if ! echo "${{ github.event.inputs.version }}" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$'; then + echo "::error::Version must be in Major.Minor.Patch format (e.g. 5.2.0)" exit 1 fi @@ -40,42 +41,37 @@ jobs: with: ref: main token: ${{ secrets.VERSION_BUMP_PAT }} - fetch-depth: 0 - - name: Create release branch and set version - shell: pwsh - run: | - $version = "${{ github.event.inputs.version }}" - $branch = "release/v$version" - $newVer = "$version.0.0" - - # Create and push the branch - git checkout -b $branch - Write-Host "Created branch $branch" - - # Set version to Major.Minor.0.0 - $content = Get-Content setup/IdentityAtlas.psd1 -Raw - $content = $content -replace "ModuleVersion\s*=\s*'\d+\.\d+\.\d+\.\d+'", "ModuleVersion = '$newVer'" - Set-Content setup/IdentityAtlas.psd1 $content -NoNewline - Write-Host "Set ModuleVersion to $newVer" - - "NEW_VERSION=$newVer" | Out-File -Append $env:GITHUB_ENV - "BRANCH=$branch" | Out-File -Append $env:GITHUB_ENV - - - name: Commit and push + - name: Create and push release tag run: | + VERSION="${{ github.event.inputs.version }}" + TAG="v${VERSION}" git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" - git add setup/IdentityAtlas.psd1 - git commit -m "chore: cut release branch, set version to ${NEW_VERSION}" - git push origin "${BRANCH}" - echo "✅ Release branch ${BRANCH} created at version ${NEW_VERSION}" + git tag -a "$TAG" -m "Release $TAG" + git push origin "$TAG" + echo "TAG=$TAG" >> "$GITHUB_ENV" + echo "VERSION=$VERSION" >> "$GITHUB_ENV" + echo "✅ Tagged ${TAG} on $(git rev-parse --short HEAD)" - - name: Trigger docker-publish for initial release image + - name: Post run summary run: | - gh workflow run docker-publish.yml \ - --ref main \ - --field branch="${BRANCH}" - echo "✅ docker-publish triggered for ${BRANCH} (will publish :latest + :${NEW_VERSION})" - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + cat >> "$GITHUB_STEP_SUMMARY" << EOF + ## ✅ Release tagged + + | | | + |---|---| + | **Tag** | \`${TAG}\` | + | **Version** | \`${VERSION}.0\` | + | **Commit** | \`$(git rev-parse --short HEAD)\` | + + docker-publish is now building \`:latest\` + \`:${VERSION}.0\` — check the [Actions tab](../../actions) for progress. + + ### Hotfix workflow (if a bug is found in this release) + \`\`\`bash + git checkout -b bugfixes/fix-foo ${TAG} # branch from the tag, not main + # fix the bug, commit, push + git push origin bugfixes/fix-foo + # then run Actions → Cut Hotfix + \`\`\` + EOF diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 25aecadd2..8283f97c2 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -8,9 +8,9 @@ # ghcr.io/fortigi/identity-atlas — Node.js API + React frontend (web service) # ghcr.io/fortigi/identity-atlas-worker — PowerShell worker (crawlers, risk scoring) # -# Tags pushed depend on the source branch: -# main: edge + Major.Minor.yyyyMMdd.HHmm (dev builds, not :latest) -# release/v*: latest + Major.Minor.Patch.0 (stable customer releases) +# Tags pushed depend on the trigger: +# main (via bump-version): :edge + Major.Minor.yyyyMMdd.HHmm +# v* tag push: :latest + Major.Minor.Patch.0 # ───────────────────────────────────────────────────────────────────────────── name: Publish Docker Images @@ -21,11 +21,13 @@ on: types: [completed] branches: - main - - 'release/**' + push: + tags: + - 'v*' workflow_dispatch: inputs: - branch: - description: 'Branch to build from (main or release/vX.Y)' + ref: + description: 'Branch or tag to build from (e.g. main, v5.2.0)' required: true default: 'main' @@ -38,39 +40,55 @@ jobs: runs-on: ubuntu-latest if: > github.event_name == 'workflow_dispatch' || + github.event_name == 'push' || github.event.workflow_run.conclusion == 'success' permissions: contents: read packages: write steps: - - name: Determine source branch and image tags + - name: Determine ref and channel id: config run: | - if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then - BRANCH="${{ github.event.inputs.branch }}" + EVENT="${{ github.event_name }}" + if [ "$EVENT" = "workflow_dispatch" ]; then + REF="${{ github.event.inputs.ref }}" + elif [ "$EVENT" = "push" ]; then + # Tag push: github.ref_name = v5.2.0 + REF="${{ github.ref_name }}" else - BRANCH="${{ github.event.workflow_run.head_branch }}" + # workflow_run from bump-version on main + REF="${{ github.event.workflow_run.head_branch }}" fi - echo "branch=$BRANCH" >> "$GITHUB_OUTPUT" + echo "ref=$REF" >> "$GITHUB_OUTPUT" - if [[ "$BRANCH" == release/* ]]; then + if [[ "$REF" == v* ]]; then + # Release tag: v5.2.0 → version 5.2.0.0 + VERSION="${REF#v}.0" echo "channel=release" >> "$GITHUB_OUTPUT" + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + echo "use_tag_version=true" >> "$GITHUB_OUTPUT" else echo "channel=main" >> "$GITHUB_OUTPUT" + echo "use_tag_version=false" >> "$GITHUB_OUTPUT" fi - uses: actions/checkout@v4 with: - ref: ${{ steps.config.outputs.branch }} + ref: ${{ steps.config.outputs.ref }} - - name: Extract version from manifest + - name: Extract version id: version shell: bash run: | - version=$(grep -oP "ModuleVersion\s*=\s*'\K[^']+" setup/IdentityAtlas.psd1) - echo "version=$version" >> "$GITHUB_OUTPUT" - echo "Module version: $version" + if [ "${{ steps.config.outputs.use_tag_version }}" = "true" ]; then + echo "version=${{ steps.config.outputs.version }}" >> "$GITHUB_OUTPUT" + echo "Version from tag: ${{ steps.config.outputs.version }}" + else + version=$(grep -oP "ModuleVersion\s*=\s*'\K[^']+" setup/IdentityAtlas.psd1) + echo "version=$version" >> "$GITHUB_OUTPUT" + echo "Version from psd1: $version" + fi - name: Log in to GitHub Container Registry uses: docker/login-action@v3 @@ -163,7 +181,7 @@ jobs: run: docker compose -f docker-compose.qa.yml down -v 2>/dev/null || true # ── Push images (only reached if smoke test passed) ───────────── - - name: Push web image (release → latest) + - name: Push web image (release tag → latest) if: steps.config.outputs.channel == 'release' uses: docker/build-push-action@v6 with: @@ -177,7 +195,7 @@ jobs: cache-from: type=gha,scope=web cache-to: type=gha,mode=max,scope=web - - name: Push worker image (release → latest) + - name: Push worker image (release tag → latest) if: steps.config.outputs.channel == 'release' uses: docker/build-push-action@v6 with: diff --git a/CLAUDE.md b/CLAUDE.md index 1ea512cb2..753767111 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -32,47 +32,46 @@ Identity Atlas is a Docker-deployed application that pulls authorization data fr | Branch | Purpose | PR required? | Approval required? | |--------|---------|-------------|-------------------| | `main` | Integration trunk. Never commit directly. Merges push `:edge` Docker tag. | Yes | Yes (at least 1) | -| `release/vX.Y` | Stable customer release line. Cut from `main` via `cut-release.yml`. Merges push `:latest` Docker tag. | Yes | No | | `feature/` | All feature work. Created from `main`. Merged back to `main` via PR. | Yes (to `main`) | No | -| `bugfixes/` | Bug fixes. Branch from `release/vX.Y` for production hotfixes, or from `main` for pre-release fixes. | Yes | No | +| `bugfixes/` | Bug fixes. Branch from `main` for pre-release fixes; branch from a **release tag** for hotfixes. | Yes (to `main`) | No | **Rules:** - `feature/` branches must be branched off `main`. -- `bugfixes/` branches branch from **`release/vX.Y`** when fixing a production issue (customers are affected), or from `main` when fixing something not yet released. -- Production bugfixes merged to `release/vX.Y` must also be cherry-picked to `main` so the fix is included in future feature releases. -- All merges go through a Pull Request — no direct pushes to `main` or `release/*` ever. +- `bugfixes/` branches branch from `main` for pre-release fixes. For hotfixes to an already-released version, branch from the release tag: `git checkout -b bugfixes/fix-foo v5.2.0`. +- Hotfix commits must be cherry-picked back to `main` via a separate PR so the fix is included in future releases. +- All merges go through a Pull Request — no direct pushes to `main` ever. - Branch names: `feature/` or `bugfixes/` (lowercase, hyphens). Example: `feature/risk-score-export`, `bugfixes/fix-login-redirect`. -- When starting work, always create a new branch. Never work directly on `main` or `release/*`. +- When starting work, always create a new branch. Never work directly on `main`. - **One issue per branch.** Each branch must fix exactly one issue or implement exactly one feature. Never combine fixes for separate, unrelated issues into a single branch or PR. Exception: if a single code change genuinely resolves more than one issue (e.g. the same root cause), both issue numbers may be referenced in the commit and PR — but this should be rare and the connection must be explicit. ### Version Number Scheme Two formats, both 4-part (PowerShell-compatible): -| Branch | Version format | Example | Docker tag pushed | -|--------|---------------|---------|-------------------| -| `main` | `Major.Minor.yyyyMMdd.HHmm` | `5.3.20260419.1430` | `:edge` | -| `release/vX.Y` | `Major.Minor.Patch.0` | `5.2.1.0` | `:latest` | +| Context | Version format | Example | Docker tag pushed | +|---------|---------------|---------|-------------------| +| `main` dev builds | `Major.Minor.yyyyMMdd.HHmm` | `5.3.20260419.1430` | `:edge` | +| Release tags (`v*`) | `Major.Minor.Patch.0` | `5.2.1.0` | `:latest` | | `feature/*` / `bugfixes/*` | — | — | Nobody | -The timestamp format on `main` makes dev builds instantly recognisable. The semantic `Patch.0` format on release branches gives customers a clear upgrade path. +The timestamp format on `main` makes dev builds instantly recognisable. Release versions use `Major.Minor.Patch.0` (patch increments for each hotfix). **Who updates versions:** -| Branch | Who updates it | When | -|--------|---------------|------| -| `main` | `bump-version.yml` (automated) | Every PR merge — increments `Minor`, updates timestamp | -| `release/vX.Y` | `bump-version.yml` (automated) | Every PR merge — increments `Patch` (e.g. `5.2.0.0` → `5.2.1.0`) | +| Context | Who updates it | When | +|---------|---------------|------| +| `main` dev builds | `bump-version.yml` (automated) | Every PR merge — increments `Minor`, updates timestamp | +| Release tags | `cut-release.yml` / `cut-hotfix.yml` (automated) | When you run Actions → Cut Release or Cut Hotfix | | `feature/*` / `bugfixes/*` | **Nobody** | Never touch `setup/IdentityAtlas.psd1` on a branch | **How to apply:** 1. **Starting a feature or pre-release bugfix branch**: Branch from `main`. Leave `setup/IdentityAtlas.psd1` untouched. -2. **Starting a production hotfix branch**: Branch from `release/vX.Y`. Leave `setup/IdentityAtlas.psd1` untouched. +2. **Starting a hotfix branch**: Branch from the release tag (`git checkout -b bugfixes/fix-foo v5.2.0`). Leave `setup/IdentityAtlas.psd1` untouched. 3. **After any code change on a branch**: Add bullets to `changes/.md`. Do not edit `CHANGES.md` or `ModuleVersion`. 4. **When merging → main via PR**: `bump-version.yml` increments Minor + timestamp. `docker-publish.yml` builds and pushes `:edge` + versioned tag. -5. **When merging → release/vX.Y via PR**: `bump-version.yml` increments Patch. `docker-publish.yml` builds and pushes `:latest` + versioned tag. -6. **Cutting a new release**: Run the `cut-release.yml` workflow (Actions → Cut Release Branch → enter `Major.Minor`). It creates `release/vX.Y` from `main` and sets the version to `X.Y.0.0`. +5. **Cutting a release**: Run Actions → Cut Release, enter `Major.Minor.Patch` (e.g. `5.2.0`). Tags `v5.2.0` on current `main` HEAD; `docker-publish.yml` pushes `:latest` + `:5.2.0.0`. +6. **Shipping a hotfix**: Run Actions → Cut Hotfix, enter the branch name and new version (e.g. `5.2.1`). Tags `v5.2.1` on the hotfix branch HEAD; `docker-publish.yml` pushes `:latest` + `:5.2.1.0`. 7. **Major version bump**: Edit `setup/IdentityAtlas.psd1` manually on `main` (via a PR) for a breaking change. Increment `Major`, reset `Minor` to `0`. ### Changelog fragments (replaces direct CHANGES.md edits) @@ -748,7 +747,7 @@ These steps are required once when creating or transferring the repository. They | Secret | Required scopes | Purpose | |--------|----------------|---------| -| `VERSION_BUMP_PAT` | `repo` (includes `contents:write`) | Lets `bump-version.yml` and `cut-release.yml` push commits directly to protected branches (`main`, `release/**`). The PAT owner **must have the admin role** on the repository so the bypass actor rule on `release/**` applies. | +| `VERSION_BUMP_PAT` | `repo` (includes `contents:write`) | Lets `bump-version.yml`, `cut-release.yml`, and `cut-hotfix.yml` push tags and commits to `main`. | ### Branch protection @@ -760,7 +759,7 @@ bash tools/setup-branch-protection.sh Fortigi/IdentityAtlas This sets: - `main` — PR required (1 approval), `PR Summary` check required, admins bypass -- `release/**` — PR required (0 approvals), `PR Summary` check required, no force-push, no deletion, admins bypass +- Tags are immutable by default in GitHub — no extra protection needed --- @@ -774,38 +773,22 @@ git checkout main && git pull git checkout -b feature/ # e.g. feature/risk-score-export ``` -**Pre-release bugfix (bug is in main, not yet in a release):** +**Pre-release bugfix (bug is in main, not yet released):** ```bash git checkout main && git pull git checkout -b bugfixes/ # e.g. bugfixes/fix-login-redirect ``` -**Production hotfix (bug is in a released version, customers are affected):** +**Hotfix (bug is in a released version — ship the fix without including unreleased features):** ```bash -# Step 1 — fix on the release branch -git checkout release/v5.2 && git pull -git checkout -b bugfixes/ -# ... make the fix, add changes/.md fragment, commit ... -gh pr create --base release/v5.2 --title "fix: ..." -# merge the PR → bump-version bumps patch, docker-publish pushes :latest - -# Step 2 — bring the fix into main via its own PR (main is protected, no direct commits) -git checkout main && git pull -git checkout -b bugfixes/-main -git cherry-pick # the fix commit only, not the version bump commit -gh pr create --base main --title "fix: ... (cherry-pick from release/v5.2)" -# merge the PR → bump-version bumps minor on main as normal +# Branch from the release tag, not from main +git checkout -b bugfixes/ v5.2.0 +# ... make the fix, add changes/.md fragment, commit, push ... +git push origin bugfixes/ +# Then run Actions → Cut Hotfix with the branch name and new version (e.g. 5.2.1) +# After the hotfix ships, cherry-pick the fix to main via a separate PR ``` -### Cutting a New Release - -When `main` is stable and ready to ship to customers: - -1. Go to **Actions → Cut Release Branch → Run workflow** -2. Enter the version, e.g. `5.3` (Major.Minor only) -3. The workflow creates `release/v5.3` from `main` and sets version to `5.3.0.0` -4. Merges to `release/v5.3` push `:latest` to customers - ### Making Changes 1. **Create/Edit** the relevant files @@ -843,19 +826,35 @@ When a bottom PR merges, retarget the next one: `gh pr edit --base main 3. Requires 1 approval — merge when CI passes 4. After merge: `bump-version.yml` increments Minor + timestamp; `docker-publish.yml` pushes `:edge` -### Merging to a Release Branch (production hotfix) +### Cutting a Release -1. Open PR from `bugfixes/` into `release/vX.Y` -2. Use the fragment content from `changes/.md` as the PR description -3. Merge when CI passes -4. After merge: `bump-version.yml` increments Patch; `docker-publish.yml` pushes `:latest` -5. Cherry-pick the fix to `main`: `git checkout main && git cherry-pick ` +When `main` is stable and ready to ship to customers: + +1. Go to **Actions → Cut Release → Run workflow** +2. Enter the version: `Major.Minor.Patch` (e.g. `5.2.0`) +3. The workflow creates tag `v5.2.0` on the current `main` HEAD +4. `docker-publish.yml` triggers automatically and pushes `:latest` + `:5.2.0.0` + +### Hotfix Releases (shipping a fix without unreleased features) + +```bash +# 1. Branch from the release tag — NOT from main +git checkout -b bugfixes/fix-foo v5.2.0 + +# 2. Fix, commit, push +git push origin bugfixes/fix-foo +``` + +3. Go to **Actions → Cut Hotfix → Run workflow** +4. Enter the branch name and new version (e.g. `5.2.1`) +5. `docker-publish.yml` triggers on the new tag and pushes `:latest` + `:5.2.1.0` +6. Cherry-pick the fix to `main`: open a PR from a cherry-pick branch into `main` ### Version Updates See the **Branching & Versioning Strategy** section above for the full scheme. - `main` merges → `Major.Minor.yyyyMMdd.HHmm` → `:edge` Docker tag -- `release/*` merges → `Major.Minor.Patch.0` → `:latest` Docker tag +- Release tags (`v*`) → `Major.Minor.Patch.0` → `:latest` Docker tag ## User Workflow (Getting Started) diff --git a/changes/feature-tag-based-releases.md b/changes/feature-tag-based-releases.md new file mode 100644 index 000000000..9b3796a57 --- /dev/null +++ b/changes/feature-tag-based-releases.md @@ -0,0 +1,4 @@ +- Replaced long-lived release branches with git tags for release management — hotfixes now ship only the fix, without features already merged to main +- Added "Cut Release" workflow: tags vX.Y.Z on main HEAD, triggers :latest publish +- Added "Cut Hotfix" workflow: tags a hotfix branch commit as a new patch version, triggers :latest publish +- Removed release branch concept — no more "Compare & pull request" banner confusion after cutting a release diff --git a/docs/architecture/branching-strategy.md b/docs/architecture/branching-strategy.md new file mode 100644 index 000000000..5914eb1b3 --- /dev/null +++ b/docs/architecture/branching-strategy.md @@ -0,0 +1,159 @@ +# Branching and Versioning Strategy + +This document covers branch naming, PR rules, version format, and the release workflow for contributors. + +--- + +## Branch Model + +| Branch | Purpose | PR required? | Approval required? | +|--------|---------|-------------|-------------------| +| `main` | Stable trunk. Never commit directly. | Yes | Yes (at least 1) | +| `feature/` | All feature work. Created from `main`. Merged back to `main` via PR. | Yes | No | +| `bugfixes/` | Bug fixes. Branch from `main` for pre-release fixes; branch from a **release tag** for hotfixes. | Yes (to `main`) | No | + +**Rules:** + +- `feature/` branches must be branched off `main`. +- `bugfixes/` branches branch from `main` for pre-release fixes. For hotfixes to an already-released version, branch from the release tag (e.g. `git checkout -b bugfixes/fix-foo v5.2.0`). +- All merges to `main` go through a Pull Request — no direct pushes. +- Branch names: lowercase, hyphens. Examples: `feature/risk-score-export`, `bugfixes/fix-login-redirect`. +- **One issue per branch.** Each branch fixes exactly one issue or implements exactly one feature. Never combine unrelated fixes into a single branch or PR. + +--- + +## Starting New Work + +```bash +# Feature or pre-release bugfix — branch from main +git checkout main && git pull +git checkout -b feature/ +# or +git checkout -b bugfixes/ + +# Hotfix to a released version — branch from the release tag +git checkout -b bugfixes/ v5.2.0 +``` + +--- + +## Version Number Format + +Two version formats, both PowerShell-compatible (4-part): + +| Context | Format | Example | +|---------|--------|---------| +| `main` dev builds | `Major.Minor.yyyyMMdd.HHmm` | `5.3.20260419.1430` | +| Release tags | `Major.Minor.Patch.0` | `5.2.1.0` | + +**Who updates what:** + +| Action | Who | When | +|--------|-----|------| +| `Minor` bump + timestamp | `bump-version.yml` GitHub Action | Automatically on every PR merge to `main` | +| Release version | `cut-release.yml` GitHub Action | When you run Actions → Cut Release | +| Hotfix version | `cut-hotfix.yml` GitHub Action | When you run Actions → Cut Hotfix | +| `Major` bump | Developer, via PR to `main` | Only for breaking changes | +| Branch work | Nobody | Never touch `setup/IdentityAtlas.psd1` on a branch | + +--- + +## Changelog Fragments + +Every `feature/` or `bugfixes/` branch must include a changelog fragment. **Never edit `CHANGES.md` directly** — the `bump-version.yml` CI action merges all fragments on PR merge. + +**File:** `changes/.md` (e.g. `changes/fix-login-redirect.md`) + +**Format:** + +```markdown +- Fixed the login redirect when auth is enabled and no session exists +- Improved error message when tenant ID is missing +``` + +Write in user-facing language. One bullet per functional change. Add the file alongside the code change — don't batch at the end. + +--- + +## Merging to Main (via PR) + +1. Open a PR from `feature/` or `bugfixes/` into `main`. +2. Use the changelog fragment content as the PR description body. +3. Requires 1 approval and passing CI. +4. After merge, `bump-version.yml` automatically increments `Minor`, updates the timestamp, and merges all `changes/*.md` fragments into `CHANGES.md`. The `docker-publish.yml` action then builds and pushes Docker images tagged `:edge`. + +--- + +## Cutting a Release + +When `main` is stable and ready to ship to customers: + +1. Go to **Actions → Cut Release → Run workflow** +2. Enter the version: `Major.Minor.Patch` (e.g. `5.2.0`) +3. The workflow creates tag `v5.2.0` on the current `main` HEAD +4. `docker-publish.yml` triggers automatically on the tag push and builds `:latest` + `:5.2.0.0` + +Customers who track `:latest` will receive the new version automatically on their next `docker compose pull`. + +--- + +## Hotfix Releases + +To ship a bugfix without including features that are already on `main`: + +```bash +# 1. Branch from the release tag, not from main +git checkout -b bugfixes/fix-login-crash v5.2.0 + +# 2. Fix the bug, commit +git add ... +git commit -m "fix: ..." + +# 3. Push the branch +git push origin bugfixes/fix-login-crash +``` + +Then: + +4. Go to **Actions → Cut Hotfix → Run workflow** +5. Enter the branch name (`bugfixes/fix-login-crash`) and new version (`5.2.1`) +6. The workflow creates tag `v5.2.1` on the HEAD of your branch +7. `docker-publish.yml` builds `:latest` + `:5.2.1.0` + +After the hotfix ships, open a PR to cherry-pick the fix into `main`: + +```bash +git checkout main && git pull +git cherry-pick +gh pr create --base main --title "fix: cherry-pick hotfix from v5.2.1" +``` + +--- + +## Stacked PRs + +For larger features, break the work into a stack of small focused PRs. Each PR targets the previous branch in the stack: + +```bash +# Step 1 — targets main +git checkout -b feature/foo-step-1 +gh pr create --base main --title "step 1: ..." + +# Step 2 — stacked on step 1 +git checkout -b feature/foo-step-2 +gh pr create --base feature/foo-step-1 --title "step 2: ..." +``` + +When a bottom PR merges, retarget the next one: `gh pr edit --base main`. + +--- + +## Image Channels + +| Tag | Content | Who uses it | +|-----|---------|-------------| +| `:latest` | Last stable release (from a `v*` tag) | End users (default) | +| `:edge` | Latest commit on `main` | Testers and developers | +| `:5.2.1.0` | Exact pinned version | Production deployments needing controlled upgrades | + +See [Docker Setup](docker-setup.md) for how to select a channel via `IMAGE_TAG`. diff --git a/tools/setup-branch-protection.sh b/tools/setup-branch-protection.sh index 417562e44..9aecd95c2 100644 --- a/tools/setup-branch-protection.sh +++ b/tools/setup-branch-protection.sh @@ -5,18 +5,15 @@ # # What this configures: # -# main (classic branch protection — already set, included here for docs) +# main (classic branch protection) # - Require PR with 1 approval before merging # - Require "PR Summary" status check # - Dismiss stale reviews on push # - enforce_admins: false ← lets VERSION_BUMP_PAT push the version bump commit # -# release/** (GitHub Ruleset — wildcard patterns need Rulesets API) -# - Require PR before merging (0 approvals needed) -# - Require "PR Summary" status check -# - Block direct pushes (non-fast-forward / force push) -# - Block branch deletion -# - Bypass: repository admins (actor_id=5) ← VERSION_BUMP_PAT owner must be admin +# Release model uses git tags (v5.2.0, v5.2.1, ...) rather than long-lived +# release branches. Hotfix branches (bugfixes/*) are short-lived and deleted +# after cherry-picking to main — no special protection needed. # ───────────────────────────────────────────────────────────────────────────── set -euo pipefail @@ -24,7 +21,7 @@ set -euo pipefail REPO="${1:-Fortigi/IdentityAtlas}" echo "Configuring branch protection for: $REPO" -# ── 1. main — classic branch protection ───────────────────────────────────── +# ── main — classic branch protection ──────────────────────────────────────── echo "" echo "Setting classic branch protection on main..." gh api "repos/$REPO/branches/main/protection" \ @@ -50,66 +47,23 @@ gh api "repos/$REPO/branches/main/protection" \ JSON echo "✅ main branch protection set" -# ── 2. release/** — GitHub Ruleset ────────────────────────────────────────── +# ── Remove legacy release/** ruleset if it exists ─────────────────────────── echo "" -echo "Creating ruleset for release/** branches..." - -# Delete existing ruleset with the same name if it exists +echo "Checking for legacy release/** ruleset..." EXISTING_ID=$(gh api "repos/$REPO/rulesets" | \ python3 -c "import sys,json; rs=[r['id'] for r in json.load(sys.stdin) if r['name']=='Protect release branches']; print(rs[0] if rs else '')" 2>/dev/null || true) if [ -n "$EXISTING_ID" ]; then - echo " Removing existing ruleset (id=$EXISTING_ID)..." + echo " Removing legacy release/** ruleset (id=$EXISTING_ID)..." gh api "repos/$REPO/rulesets/$EXISTING_ID" --method DELETE + echo " ✅ Legacy ruleset removed" +else + echo " No legacy ruleset found — nothing to remove" fi -gh api "repos/$REPO/rulesets" --method POST --input - <<'JSON' -{ - "name": "Protect release branches", - "target": "branch", - "enforcement": "active", - "conditions": { - "ref_name": { - "include": ["refs/heads/release/**"], - "exclude": [] - } - }, - "bypass_actors": [ - { - "actor_id": 5, - "actor_type": "RepositoryRole", - "bypass_mode": "always" - } - ], - "rules": [ - { "type": "deletion" }, - { "type": "non_fast_forward" }, - { - "type": "pull_request", - "parameters": { - "required_approving_review_count": 0, - "dismiss_stale_reviews_on_push": false, - "require_code_owner_review": false, - "require_last_push_approval": false, - "required_review_thread_resolution": false - } - }, - { - "type": "required_status_checks", - "parameters": { - "strict_required_status_checks_policy": false, - "required_status_checks": [ - { "context": "PR Summary" } - ] - } - } - ] -} -JSON -echo "✅ release/** ruleset created" - echo "" echo "Done. Branch protection summary:" -echo " main → PR required (1 approval) + PR Summary check" -echo " release/** → PR required (0 approvals) + PR Summary check + no force-push + no deletion" -echo " Bypass → Repository admins (the VERSION_BUMP_PAT owner must have admin role)" +echo " main → PR required (1 approval) + PR Summary check + no force-push" +echo " tags → No branch protection needed (tags are immutable by default)" +echo "" +echo "Release model: git tags (v5.2.0, v5.2.1, ...) via Actions → Cut Release / Cut Hotfix" From 5fe62d8a54f4f462ab137d391ad109fd93aa4d4f Mon Sep 17 00:00:00 2001 From: Taeke Kooiker Date: Sun, 19 Apr 2026 11:05:08 +0200 Subject: [PATCH 026/160] docs: update release model documentation for tag-based releases - branching-strategy.md: rewritten to reflect tag-based releases (vX.Y.Z tags instead of release/vX.Y branches), hotfix workflow, and updated image channels table showing when :latest vs :edge publish - docker-setup.md: Image Channels table now explains :latest is published when a release tag is cut, not on every main merge; added pinned version example command - quickstart.md: added Image Channels section explaining :latest vs :edge clearly; fixed version format in "Checking the running version" section (stable shows Major.Minor.Patch.0, edge shows timestamp format) Co-Authored-By: Claude Sonnet 4.6 --- docs/architecture/branching-strategy.md | 85 ++++++++++++++++++------- docs/architecture/docker-setup.md | 11 ++-- docs/quickstart.md | 29 +++++++-- 3 files changed, 94 insertions(+), 31 deletions(-) diff --git a/docs/architecture/branching-strategy.md b/docs/architecture/branching-strategy.md index 4df5b6da3..0bd8934ef 100644 --- a/docs/architecture/branching-strategy.md +++ b/docs/architecture/branching-strategy.md @@ -1,6 +1,6 @@ # Branching and Versioning Strategy -This document covers branch naming, PR rules, version format, and the changelog workflow for contributors. +This document covers branch naming, PR rules, version format, and the release workflow for contributors. --- @@ -10,11 +10,13 @@ This document covers branch naming, PR rules, version format, and the changelog |--------|---------|-------------|-------------------| | `main` | Stable trunk. Never commit directly. | Yes | Yes (at least 1) | | `feature/` | All feature work. Created from `main`. Merged back to `main` via PR. | Yes | No | -| `bugfixes/` | Bug fixes. Created from `main`. Merged back to `main` via PR. | Yes | No | +| `bugfixes/` | Bug fixes. Branch from `main` for pre-release fixes; branch from a **release tag** for hotfixes. | Yes (to `main`) | No | **Rules:** -- `feature/` and `bugfixes/` branches must be branched off `main`. +- `feature/` branches must be branched off `main`. +- `bugfixes/` branches branch from `main` for pre-release fixes. For hotfixes to an already-released version, branch from the release tag: `git checkout -b bugfixes/fix-foo v5.2.0`. +- Hotfix commits must be cherry-picked back to `main` via a separate PR so the fix is included in future releases. - All merges to `main` go through a Pull Request — no direct pushes. - Branch names: lowercase, hyphens. Examples: `feature/risk-score-export`, `bugfixes/fix-login-redirect`. - **One issue per branch.** Each branch fixes exactly one issue or implements exactly one feature. Never combine unrelated fixes into a single branch or PR. @@ -24,34 +26,35 @@ This document covers branch naming, PR rules, version format, and the changelog ## Starting New Work ```bash +# Feature or pre-release bugfix — branch from main git checkout main && git pull git checkout -b feature/ # or git checkout -b bugfixes/ + +# Hotfix to a released version — branch from the release tag, NOT from main +git checkout -b bugfixes/ v5.2.0 ``` --- ## Version Number Format -``` -Major.Minor.yyyyMMdd.HHmm -``` - -Example: `5.2.20260420.1430` +Two formats, both 4-part (PowerShell-compatible): -| Part | Meaning | -|------|---------| -| `Major` | Incremented manually for breaking changes (via a PR to `main`) | -| `Minor` | Auto-incremented by CI on every PR merge to `main` | -| `yyyyMMdd.HHmm` | Timestamp of the merge, set by CI | +| Context | Format | Example | +|---------|--------|---------| +| `main` dev builds (`:edge`) | `Major.Minor.yyyyMMdd.HHmm` | `5.3.20260419.1430` | +| Release tags (`:latest`) | `Major.Minor.Patch.0` | `5.2.1.0` | **Who updates what:** | Action | Who | When | |--------|-----|------| | `Minor` bump + timestamp | `bump-version.yml` GitHub Action | Automatically on every PR merge to `main` | -| `Major` bump | Developer, via PR | Only for breaking changes | +| Release version | `cut-release.yml` GitHub Action | When you run Actions → Cut Release | +| Hotfix version | `cut-hotfix.yml` GitHub Action | When you run Actions → Cut Hotfix | +| `Major` bump | Developer, via PR to `main` | Only for breaking changes | | Branch work | Nobody | Never touch `setup/IdentityAtlas.psd1` on a branch | --- @@ -78,7 +81,47 @@ Write in user-facing language. One bullet per functional change. Add the file al 1. Open a PR from `feature/` or `bugfixes/` into `main`. 2. Use the changelog fragment content as the PR description body. 3. Requires 1 approval and passing CI. -4. After merge, `bump-version.yml` automatically increments `Minor`, updates the timestamp, and merges all `changes/*.md` fragments into `CHANGES.md`. The `docker-publish.yml` action then builds and pushes Docker images tagged with the new version. +4. After merge, `bump-version.yml` automatically increments `Minor`, updates the timestamp, and merges all `changes/*.md` fragments into `CHANGES.md`. The `docker-publish.yml` action then builds and pushes Docker images tagged `:edge`. + +--- + +## Cutting a Release + +When `main` is stable and ready to ship to customers: + +1. Go to **Actions → Cut Release → Run workflow** +2. Enter the version: `Major.Minor.Patch` (e.g. `5.2.0`) +3. The workflow creates tag `v5.2.0` on the current `main` HEAD +4. `docker-publish.yml` triggers automatically on the tag push and builds `:latest` + `:5.2.0.0` + +Customers who track `:latest` will receive the new version on their next `docker compose pull`. + +--- + +## Hotfix Releases + +To ship a bugfix without including features already on `main`: + +```bash +# 1. Branch from the release tag — NOT from main +git checkout -b bugfixes/fix-login-crash v5.2.0 + +# 2. Fix the bug, commit, push +git push origin bugfixes/fix-login-crash +``` + +3. Go to **Actions → Cut Hotfix → Run workflow** +4. Enter the branch name (`bugfixes/fix-login-crash`) and new version (`5.2.1`) +5. The workflow creates tag `v5.2.1` on the HEAD of your branch +6. `docker-publish.yml` builds `:latest` + `:5.2.1.0` + +After the hotfix ships, open a PR to cherry-pick the fix into `main`: + +```bash +git checkout main && git pull +git cherry-pick +gh pr create --base main --title "fix: cherry-pick hotfix from v5.2.1" +``` --- @@ -102,12 +145,10 @@ When a bottom PR merges, retarget the next one: `gh pr edit --base main ## Image Channels -The CI pipeline publishes Docker images on every merge to `main`: - -| Tag | Content | Who uses it | -|-----|---------|-------------| -| `:latest` | Last stable release | End users (default) | -| `:edge` | Latest commit on `main` | Testers and developers | -| `:5.2.0.0` | Exact pinned version | Production deployments | +| Tag | Published when | Who uses it | +|-----|---------------|-------------| +| `:latest` | A release tag (`v5.2.0`) is created via Actions → Cut Release or Cut Hotfix | Customers (default) | +| `:edge` | Every PR merges to `main` | Developers and testers | +| `:5.2.0.0` | Same time as `:latest` — exact pinned version | Production deployments needing controlled upgrades | See [Docker Setup](docker-setup.md) for how to select a channel via `IMAGE_TAG`. diff --git a/docs/architecture/docker-setup.md b/docs/architecture/docker-setup.md index e72e4d2a3..d166f20ac 100644 --- a/docs/architecture/docker-setup.md +++ b/docs/architecture/docker-setup.md @@ -67,9 +67,9 @@ The compose file uses the `IMAGE_TAG` variable to select which build to pull: | `IMAGE_TAG` | What you get | Who should use it | |---|---|---| -| *(unset or blank)* | `:latest` — last stable release | Customers and production deployments | -| `edge` | `:edge` — latest commit on `main`, may be unstable | Developers and testers who want the newest features | -| `5.2.1.0` | Exact pinned version, never auto-updates | Customers who want to control upgrade timing | +| *(unset or blank)* | `:latest` — last stable release, published when a release tag is cut | Customers and production deployments | +| `edge` | `:edge` — latest commit on `main`, updated on every PR merge, may include unreleased features | Developers and testers | +| `5.2.1.0` | Exact pinned version, never auto-updates | Production deployments needing controlled upgrade timing | The running version is always visible in the footer of the UI. Edge builds show an amber **edge** badge so it is immediately obvious which channel is running. @@ -77,9 +77,12 @@ The running version is always visible in the footer of the UI. Edge builds show # Run the stable release (default) docker compose -f docker-compose.prod.yml up -d --pull always -# Run the edge build +# Run the edge build (latest merged to main, may be unstable) IMAGE_TAG=edge docker compose -f docker-compose.prod.yml up -d --pull always # or set IMAGE_TAG=edge in your .env + +# Run a specific pinned version +IMAGE_TAG=5.2.0.0 docker compose -f docker-compose.prod.yml up -d --pull always ``` --- diff --git a/docs/quickstart.md b/docs/quickstart.md index fa8a728bd..3d5631aef 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -74,11 +74,30 @@ Open the UI at [http://localhost:3001](http://localhost:3001) and the Admin → --- -## Upgrading to a new version +## Image channels + +Identity Atlas publishes two channels: + +| Channel | Tag | Updated when | Use it for | +|---------|-----|-------------|-----------| +| **Stable** | `:latest` | A new release is cut (e.g. `v5.2.0`) | Customers and production — default | +| **Edge** | `:edge` | Every PR merges to `main` | Developers and testers who want unreleased features | + +Both channels also publish an exact version tag (`:5.2.0.0`, `:5.2.1.0`) at the same time as `:latest`, so you can pin to a specific build. + +```bash +# Default: pull the latest stable release +docker compose -f docker-compose.prod.yml up -d --pull always -Identity Atlas publishes new images to `ghcr.io/fortigi/identity-atlas{,-worker}` on every push to `main`. The `:latest` tag always points at the newest build, and each build also gets a version-stamped tag (`5.0.yyyyMMdd.HHmm`) for reproducible deployments. +# Edge: latest commit on main (may include unreleased features) +IMAGE_TAG=edge docker compose -f docker-compose.prod.yml up -d --pull always +``` + +--- + +## Upgrading to a new version -To upgrade an existing deployment to the newest version: +To upgrade an existing deployment to the newest stable release: === "Linux / macOS" @@ -98,8 +117,8 @@ The database volume is preserved across upgrades — any data you have loaded st Three ways to see which version is currently deployed: -1. **Dashboard** — open [http://localhost:3001](http://localhost:3001); the Version card on the right shows `v5.0.yyyyMMdd.HHmm`. -2. **API endpoint** — `Invoke-RestMethod http://localhost:3001/api/version` (or `curl` on Linux/macOS). Returns `{ "version": "5.0.yyyyMMdd.HHmm" }`. +1. **Dashboard** — open [http://localhost:3001](http://localhost:3001); the Version card in the footer shows the version. Stable releases show `v5.2.0.0`; edge builds show `v5.3.20260419.1430` with an amber **edge** badge. +2. **API endpoint** — `Invoke-RestMethod http://localhost:3001/api/version` (or `curl` on Linux/macOS). Returns `{ "version": "5.2.0.0" }`. 3. **Docker directly** — `docker compose -f docker-compose.prod.yml images` lists the image tag each container is running. Compare that against the newest tag on [ghcr.io/fortigi/identity-atlas](https://github.com/Fortigi/IdentityAtlas/pkgs/container/identity-atlas) to see whether an upgrade is available. From e8ad5964ec6b28c15ced9c7de2a7bfaed89b24cf Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 19 Apr 2026 09:09:33 +0000 Subject: [PATCH 027/160] chore: bump version to 5.6.20260419.0909 --- CHANGES.md | 5 +++++ changes/feature-about-page-sbom.md | 2 -- setup/IdentityAtlas.psd1 | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) delete mode 100644 changes/feature-about-page-sbom.md diff --git a/CHANGES.md b/CHANGES.md index 822ba2676..27ccb89f9 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,10 @@ ## Changes in this PR +- Added an About page showing the MIT license text and Software Bill of Materials +- The version string in the footer is now a clickable link to the About page + +## Changes in this PR + - Added Service Principals to the Entra ID crawler. When the "Service Principals" object type is selected, the crawler now pulls all Entra service principals (enterprise apps, managed identities, AI agents) and writes them to the Principals table alongside user accounts — unblocking future Azure RM role-assignment imports that reference these identities. - New classification helper (`Get-FGServicePrincipalType`) tags each service principal as `ManagedIdentity`, `AIAgent`, or `ServicePrincipal` based on Graph's `servicePrincipalType`, well-known Microsoft AI platform tags (CopilotStudio, PowerVirtualAgents, AzureOpenAI, CognitiveServices), the Entra Agent ID markers (`AgenticInstance`, `AgenticApp`, `power-virtual-agents-*`), and display-name heuristics. Custom AI-name patterns can be supplied per crawler config via `aiNamePatterns`. - Extra SP metadata (`appId`, `servicePrincipalType`, `publisherName`, `homepage`, `tags`, `servicePrincipalNames`, `notes`) is stored in `extendedAttributes` so it shows up in the Users filter dropdown and detail pages. diff --git a/changes/feature-about-page-sbom.md b/changes/feature-about-page-sbom.md deleted file mode 100644 index b825bb489..000000000 --- a/changes/feature-about-page-sbom.md +++ /dev/null @@ -1,2 +0,0 @@ -- Added an About page showing the MIT license text and Software Bill of Materials -- The version string in the footer is now a clickable link to the About page diff --git a/setup/IdentityAtlas.psd1 b/setup/IdentityAtlas.psd1 index d4b6aea59..31b9e1b6d 100644 --- a/setup/IdentityAtlas.psd1 +++ b/setup/IdentityAtlas.psd1 @@ -12,7 +12,7 @@ RootModule = '.\IdentityAtlas.psm1' # Version number of this module. -ModuleVersion = '5.5.20260419.0804' +ModuleVersion = '5.6.20260419.0909' # Supported PSEditions # CompatiblePSEditions = @() From 64aee60af1700be33468fd14a33d92da68cd8281 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 19 Apr 2026 09:10:55 +0000 Subject: [PATCH 028/160] chore: bump version to 5.7.20260419.0910 --- CHANGES.md | 10 ++++++++++ changes/feature-docs-consistency-pass.md | 7 ------- setup/IdentityAtlas.psd1 | 2 +- 3 files changed, 11 insertions(+), 8 deletions(-) delete mode 100644 changes/feature-docs-consistency-pass.md diff --git a/CHANGES.md b/CHANGES.md index 27ccb89f9..2ef8e44f9 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,15 @@ ## Changes in this PR +- Added branching and versioning strategy reference page under docs/architecture +- Added `--pull always` flag to Quick Start commands in README, quickstart, docker-setup, and index docs so users always get the newest image +- Added `.env` setup step (copy from template) to all Quick Start sections +- Added tabbed Linux/macOS and Windows code blocks throughout docker-setup and local-dev docs +- Fixed version pinning example in quickstart to use release format (5.2.0.0) instead of edge timestamp format +- Fixed history.md to link to the branching strategy doc instead of referencing CLAUDE.md +- Aligned local-dev.md with docker-setup.md: added --build note for first run, tabbed stop/reset commands + +## Changes in this PR + - Added an About page showing the MIT license text and Software Bill of Materials - The version string in the footer is now a clickable link to the About page diff --git a/changes/feature-docs-consistency-pass.md b/changes/feature-docs-consistency-pass.md deleted file mode 100644 index efe4baba4..000000000 --- a/changes/feature-docs-consistency-pass.md +++ /dev/null @@ -1,7 +0,0 @@ -- Added branching and versioning strategy reference page under docs/architecture -- Added `--pull always` flag to Quick Start commands in README, quickstart, docker-setup, and index docs so users always get the newest image -- Added `.env` setup step (copy from template) to all Quick Start sections -- Added tabbed Linux/macOS and Windows code blocks throughout docker-setup and local-dev docs -- Fixed version pinning example in quickstart to use release format (5.2.0.0) instead of edge timestamp format -- Fixed history.md to link to the branching strategy doc instead of referencing CLAUDE.md -- Aligned local-dev.md with docker-setup.md: added --build note for first run, tabbed stop/reset commands diff --git a/setup/IdentityAtlas.psd1 b/setup/IdentityAtlas.psd1 index 31b9e1b6d..563109c46 100644 --- a/setup/IdentityAtlas.psd1 +++ b/setup/IdentityAtlas.psd1 @@ -12,7 +12,7 @@ RootModule = '.\IdentityAtlas.psm1' # Version number of this module. -ModuleVersion = '5.6.20260419.0909' +ModuleVersion = '5.7.20260419.0910' # Supported PSEditions # CompatiblePSEditions = @() From 76ba27fd3493a3bdd74b65dd9813d379c6c1c707 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 19 Apr 2026 09:12:17 +0000 Subject: [PATCH 029/160] chore: bump version to 5.8.20260419.0912 --- CHANGES.md | 7 +++++++ changes/feature-tag-based-releases.md | 4 ---- setup/IdentityAtlas.psd1 | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) delete mode 100644 changes/feature-tag-based-releases.md diff --git a/CHANGES.md b/CHANGES.md index 2ef8e44f9..3a4b5a44d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,12 @@ ## Changes in this PR +- Replaced long-lived release branches with git tags for release management — hotfixes now ship only the fix, without features already merged to main +- Added "Cut Release" workflow: tags vX.Y.Z on main HEAD, triggers :latest publish +- Added "Cut Hotfix" workflow: tags a hotfix branch commit as a new patch version, triggers :latest publish +- Removed release branch concept — no more "Compare & pull request" banner confusion after cutting a release + +## Changes in this PR + - Added branching and versioning strategy reference page under docs/architecture - Added `--pull always` flag to Quick Start commands in README, quickstart, docker-setup, and index docs so users always get the newest image - Added `.env` setup step (copy from template) to all Quick Start sections diff --git a/changes/feature-tag-based-releases.md b/changes/feature-tag-based-releases.md deleted file mode 100644 index 9b3796a57..000000000 --- a/changes/feature-tag-based-releases.md +++ /dev/null @@ -1,4 +0,0 @@ -- Replaced long-lived release branches with git tags for release management — hotfixes now ship only the fix, without features already merged to main -- Added "Cut Release" workflow: tags vX.Y.Z on main HEAD, triggers :latest publish -- Added "Cut Hotfix" workflow: tags a hotfix branch commit as a new patch version, triggers :latest publish -- Removed release branch concept — no more "Compare & pull request" banner confusion after cutting a release diff --git a/setup/IdentityAtlas.psd1 b/setup/IdentityAtlas.psd1 index 563109c46..1e8608550 100644 --- a/setup/IdentityAtlas.psd1 +++ b/setup/IdentityAtlas.psd1 @@ -12,7 +12,7 @@ RootModule = '.\IdentityAtlas.psm1' # Version number of this module. -ModuleVersion = '5.7.20260419.0910' +ModuleVersion = '5.8.20260419.0912' # Supported PSEditions # CompatiblePSEditions = @() From 6c1f76ba1400b9b06afaad7f31ec5fb18fcadf59 Mon Sep 17 00:00:00 2001 From: Taeke Kooiker Date: Sun, 19 Apr 2026 11:19:26 +0200 Subject: [PATCH 030/160] docs: add Release & Branching Strategy to mkdocs nav Co-Authored-By: Claude Sonnet 4.6 --- mkdocs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/mkdocs.yml b/mkdocs.yml index 9061e28ec..d3772c619 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -89,6 +89,7 @@ nav: - Plugin Architecture: risk-scoring/plugin-architecture.md - Architecture: - Docker Setup: architecture/docker-setup.md + - Release & Branching Strategy: architecture/branching-strategy.md - Ingest API: architecture/ingest-api.md - Audit History: architecture/audit-history.md - LLM & Risk Scoring Internals: architecture/llm-and-risk-scoring.md From bab0a30d8d590950c46522d3f45f6999b9a784f9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 19 Apr 2026 09:19:29 +0000 Subject: [PATCH 031/160] chore: bump version to 5.9.20260419.0919 --- setup/IdentityAtlas.psd1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/IdentityAtlas.psd1 b/setup/IdentityAtlas.psd1 index 1e8608550..395d0a68d 100644 --- a/setup/IdentityAtlas.psd1 +++ b/setup/IdentityAtlas.psd1 @@ -12,7 +12,7 @@ RootModule = '.\IdentityAtlas.psm1' # Version number of this module. -ModuleVersion = '5.8.20260419.0912' +ModuleVersion = '5.9.20260419.0919' # Supported PSEditions # CompatiblePSEditions = @() From f4d18a8bda14ec0b2559f20a758956d516347c5d Mon Sep 17 00:00:00 2001 From: Taeke Kooiker Date: Sun, 19 Apr 2026 11:33:11 +0200 Subject: [PATCH 032/160] feat: move About page from main nav into Admin sub-tab MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit About (license + SBOM) is now at Admin → About rather than a top-level nav tab. The version string in the footer still links to it, now navigating to #admin?sub=about. AboutPage is lazy-loaded inside AdminPage alongside the other sub-tabs. Co-Authored-By: Claude Sonnet 4.6 --- app/ui/src/App.jsx | 6 +----- app/ui/src/components/AdminPage.jsx | 8 ++++++++ 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/app/ui/src/App.jsx b/app/ui/src/App.jsx index 41a0289c0..e4ae1844e 100644 --- a/app/ui/src/App.jsx +++ b/app/ui/src/App.jsx @@ -22,7 +22,6 @@ const ContextDetailPage = lazy(() => import('./components/ContextDetailPage')); const IdentitiesPage = lazy(() => import('./components/IdentitiesPage')); const IdentityDetailPage = lazy(() => import('./components/IdentityDetailPage')); const AdminPage = lazy(() => import('./components/AdminPage')); -const AboutPage = lazy(() => import('./components/AboutPage')); // PerfPage and CrawlersPage are lazy-loaded inside AdminPage as sub-tabs. // const GovernancePage = lazy(() => import('./components/GovernancePage')); // temporarily disabled @@ -97,7 +96,6 @@ const ALL_NAV_TABS = [ { key: 'identities', label: 'Identities', feature: 'accountCorrelation', optional: true }, { key: 'org-chart', label: 'Org Chart', optional: true }, { key: 'admin', label: 'Admin' }, - { key: 'about', label: 'About' }, ]; export default function App() { @@ -498,8 +496,6 @@ export default function App() { ) : page === 'org-chart' ? ( - ) : page === 'about' ? ( - ) : page === 'performance' || page === 'crawlers' || page === 'admin' ? ( // Crawlers and Performance now live under Admin as sub-tabs. // Legacy #crawlers and #performance hashes redirect to the matching sub-tab. @@ -535,7 +531,7 @@ export default function App() { {/* Footer */}
}> + + + )}
); From 791811842b36875c84096a1428e98d04212e1faa Mon Sep 17 00:00:00 2001 From: Wim van den Heijkant Date: Sun, 19 Apr 2026 11:38:25 +0200 Subject: [PATCH 033/160] Import useCallback used by PowerQueryExportSection The new admin section hit a ReferenceError when opened because useCallback wasn't in the top-level React import. Same component, no logic change. Co-Authored-By: Claude Opus 4.7 (1M context) --- app/ui/src/components/AdminPage.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/ui/src/components/AdminPage.jsx b/app/ui/src/components/AdminPage.jsx index f796faf8f..7555cd1b8 100644 --- a/app/ui/src/components/AdminPage.jsx +++ b/app/ui/src/components/AdminPage.jsx @@ -1,4 +1,4 @@ -import { useState, useEffect, useRef, lazy, Suspense } from 'react'; +import { useState, useEffect, useRef, useCallback, lazy, Suspense } from 'react'; import { useAuth } from '../auth/AuthGate'; import ScheduleEditor from './ScheduleEditor'; From 953b54c600414c359f7a4b7c0cdc7a0c73122d45 Mon Sep 17 00:00:00 2001 From: "claude[bot]" <41898282+claude[bot]@users.noreply.github.com> Date: Sun, 19 Apr 2026 09:39:09 +0000 Subject: [PATCH 034/160] Fix #115: Add Windows PowerShell syntax for switching to edge image channel The Quick Start documentation now includes platform-specific instructions for switching to the edge channel, with proper PowerShell syntax for Windows users using $env:IMAGE_TAG instead of the Linux-only inline variable syntax. --- changes/fix-issue-115.md | 1 + docs/quickstart.md | 25 +++++++++++++++++++------ 2 files changed, 20 insertions(+), 6 deletions(-) create mode 100644 changes/fix-issue-115.md diff --git a/changes/fix-issue-115.md b/changes/fix-issue-115.md new file mode 100644 index 000000000..1ca1317d7 --- /dev/null +++ b/changes/fix-issue-115.md @@ -0,0 +1 @@ +- Fixed Quick Start documentation: Image channel switching code now includes Windows PowerShell syntax diff --git a/docs/quickstart.md b/docs/quickstart.md index 3d5631aef..efd4f6e6d 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -85,13 +85,26 @@ Identity Atlas publishes two channels: Both channels also publish an exact version tag (`:5.2.0.0`, `:5.2.1.0`) at the same time as `:latest`, so you can pin to a specific build. -```bash -# Default: pull the latest stable release -docker compose -f docker-compose.prod.yml up -d --pull always +=== "Linux / macOS" -# Edge: latest commit on main (may include unreleased features) -IMAGE_TAG=edge docker compose -f docker-compose.prod.yml up -d --pull always -``` + ```bash + # Default: pull the latest stable release + docker compose -f docker-compose.prod.yml up -d --pull always + + # Edge: latest commit on main (may include unreleased features) + IMAGE_TAG=edge docker compose -f docker-compose.prod.yml up -d --pull always + ``` + +=== "Windows (PowerShell)" + + ```powershell + # Default: pull the latest stable release + docker compose -f docker-compose.prod.yml up -d --pull always + + # Edge: latest commit on main (may include unreleased features) + $env:IMAGE_TAG = "edge" + docker compose -f docker-compose.prod.yml up -d --pull always + ``` --- From fb61d16416e6f55f442dee8f90495f963718b60e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 19 Apr 2026 10:50:45 +0000 Subject: [PATCH 035/160] chore: bump version to 5.10.20260419.1050 --- setup/IdentityAtlas.psd1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/IdentityAtlas.psd1 b/setup/IdentityAtlas.psd1 index 395d0a68d..8919cb6d8 100644 --- a/setup/IdentityAtlas.psd1 +++ b/setup/IdentityAtlas.psd1 @@ -12,7 +12,7 @@ RootModule = '.\IdentityAtlas.psm1' # Version number of this module. -ModuleVersion = '5.9.20260419.0919' +ModuleVersion = '5.10.20260419.1050' # Supported PSEditions # CompatiblePSEditions = @() From 46b568cd70d5ba7e32bacbadc22444e6c451637f Mon Sep 17 00:00:00 2001 From: Wim van den Heijkant Date: Sun, 19 Apr 2026 13:00:57 +0200 Subject: [PATCH 036/160] Fix pagination cap + auto-expand extendedAttributes in M code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two real bugs reported by a user running the MVP against their tenant: 1. Pagination stopped at 4,000 of 7,199 rows. The List.Generate next- state record literal used `[off] = [off] + PageSize`, where the inner `[off]` is parsed by M as a forward field-reference (the same record's own `off` field), not as the parameter access `_[off]`. The effective offset didn't advance past 4,000 and the loop terminated silently mid-way. Rewritten to use List.Numbers(0, PageCount, PageSize) to generate the offset sequence up-front, then List.Transform to fetch each page. No stateful records, no self-reference foot-gun. The same refactor runs in ARRAY_FETCH for /systems for consistency. 2. extendedAttributes landed as a "Record" column the user had to click open one by one — so they couldn't see userType, onPremisesSyncEnabled, signInActivity etc. as first-class columns. Both templates now auto-expand extendedAttributes. Keys are collected as the UNION across every row (not just the first), sparsely-populated keys survive. Expanded columns are prefixed `ext_` to avoid colliding with real columns of the same name. If the table has no extendedAttributes column, the expansion is a no-op. Two new vitest cases pin both behaviours so they can't silently regress. Co-Authored-By: Claude Opus 4.7 (1M context) --- app/api/src/export/excelWorkbook.test.js | 21 ++++++++ app/api/src/export/queryTemplates.js | 68 ++++++++++++++++++------ 2 files changed, 72 insertions(+), 17 deletions(-) diff --git a/app/api/src/export/excelWorkbook.test.js b/app/api/src/export/excelWorkbook.test.js index 3c7393e37..f2cb5bdd2 100644 --- a/app/api/src/export/excelWorkbook.test.js +++ b/app/api/src/export/excelWorkbook.test.js @@ -79,4 +79,25 @@ describe('generateWorkbook', () => { expect(cell).toContain(q.endpoint); } }); + + it('uses List.Numbers-based pagination (guards against the record-self-reference bug that capped the old List.Generate pattern at 4 pages)', () => { + const principals = wb.getWorksheet('Principals').getCell('A6').value; + expect(principals).toContain('List.Numbers'); + // The old buggy pattern had `[off] = [off] + PageSize` — a forward + // self-reference in a record literal — which halted iteration early. + // Guard against it creeping back. + expect(principals).not.toMatch(/off\s*=\s*\[off\]/); + }); + + it('auto-expands the extendedAttributes JSONB column', () => { + // Users of the workbook expect sub-keys (userType, onPremisesSyncEnabled, + // etc.) to appear as first-class columns, not "Record" cells they have + // to click open one by one. The load-bearing bit: ExpandRecordColumn is + // called against extendedAttributes with an ext_ prefix on the new + // column names to avoid colliding with real columns. + const principals = wb.getWorksheet('Principals').getCell('A6').value; + expect(principals).toContain('extendedAttributes'); + expect(principals).toContain('Record.FieldNames'); + expect(principals).toMatch(/ext_/); + }); }); diff --git a/app/api/src/export/queryTemplates.js b/app/api/src/export/queryTemplates.js index 819a518f4..95a55be06 100644 --- a/app/api/src/export/queryTemplates.js +++ b/app/api/src/export/queryTemplates.js @@ -16,11 +16,25 @@ const PAGE_SIZE = 1000; -// Single shared M function: paginate(endpoint, extraQuery) -// Returns the combined `data` array as a list of records. +// Shared paginated fetch template. Two responsibilities: // -// Heredoc-style template literal — we substitute PAGE_SIZE only. The user -// never edits this; they edit BaseUrl and AuthToken on the Settings sheet. +// 1. Walk the entire dataset across N pages of PAGE_SIZE records. We use +// List.Numbers to compute the page offsets up-front instead of the +// stateful List.Generate pattern — the latter has a record-self- +// reference foot-gun (`[off]` inside the next-state record literal +// gets parsed as forward field reference, not `_[off]`) that silently +// caps the loop at 4 iterations against the local stack. +// +// 2. Auto-expand the JSONB `extendedAttributes` column so users see the +// sub-keys (userType, onPremisesSyncEnabled, signInActivity, etc.) as +// first-class columns instead of "Record" cells they have to click +// one by one. Keys are collected as the union across every row so +// sparsely-populated keys (only some users have employeeId) still +// appear. Expanded columns are prefixed `ext_` to avoid collisions +// with real columns of the same name. +// +// The user never edits this; they edit BaseUrl / AuthToken on the Settings +// sheet and the queries pick up the new values on the next refresh. const PAGINATED_FETCH = ` let BaseUrl = Excel.CurrentWorkbook(){[Name="BaseUrl"]}[Content]{0}[Column1], @@ -35,18 +49,29 @@ let ])), First = FetchPage(0), Total = First[total], - Pages = if Total <= PageSize then {First} - else List.Generate( - () => [page = First, off = 0], - each [off] < Total, - each [page = FetchPage([off] + PageSize), off = [off] + PageSize], - each [page] - ), - Combined = List.Combine(List.Transform(Pages, each _[data])), - Table = Table.FromList(Combined, Splitter.SplitByNothing(), null, null, ExtraValues.Error), - Expanded = Table.ExpandRecordColumn(Table, "Column1", Record.FieldNames(Combined{0})) + PageCount = if Total = 0 then 0 else Number.RoundUp(Total / PageSize), + // Offsets: {0, PageSize, 2*PageSize, ...} for every page we need. + Offsets = List.Numbers(0, PageCount, PageSize), + // Skip the first offset because we already fetched it as First. + LaterPages = List.Transform(List.Skip(Offsets, 1), (off) => FetchPage(off)), + AllPageRecords = {First} & LaterPages, + AllRows = List.Combine(List.Transform(AllPageRecords, (p) => p[data])), + Table = Table.FromList(AllRows, Splitter.SplitByNothing(), null, null, ExtraValues.Error), + Expanded = if List.IsEmpty(AllRows) then Table + else Table.ExpandRecordColumn(Table, "Column1", Record.FieldNames(AllRows{0})), + // Auto-expand extendedAttributes. The keys vary per row — collect the + // union so we don't lose any. ext_ prefix avoids name collisions with + // real columns. Skipped if the table doesn't have an extendedAttributes + // column (most join-table endpoints). + HasExt = List.Contains(Table.ColumnNames(Expanded), "extendedAttributes"), + ExtKeys = if not HasExt then {} else List.Distinct( + List.Combine(List.Transform(Expanded[extendedAttributes], + (r) => if r = null then {} else Record.FieldNames(r)))), + ExtExpanded = if not HasExt or List.IsEmpty(ExtKeys) then Expanded + else Table.ExpandRecordColumn(Expanded, "extendedAttributes", + ExtKeys, List.Transform(ExtKeys, (k) => "ext_" & k)) in - Expanded + ExtExpanded `.trim(); function paginatedQuery(endpointPath) { @@ -68,9 +93,18 @@ let ])), Table = Table.FromList(Source, Splitter.SplitByNothing(), null, null, ExtraValues.Error), Expanded = if List.IsEmpty(Source) then Table - else Table.ExpandRecordColumn(Table, "Column1", Record.FieldNames(Source{0})) + else Table.ExpandRecordColumn(Table, "Column1", Record.FieldNames(Source{0})), + // Same extendedAttributes auto-expand as PAGINATED_FETCH — Systems has ext + // attrs too (tenant id, connector settings, etc.) + HasExt = List.Contains(Table.ColumnNames(Expanded), "extendedAttributes"), + ExtKeys = if not HasExt then {} else List.Distinct( + List.Combine(List.Transform(Expanded[extendedAttributes], + (r) => if r = null then {} else Record.FieldNames(r)))), + ExtExpanded = if not HasExt or List.IsEmpty(ExtKeys) then Expanded + else Table.ExpandRecordColumn(Expanded, "extendedAttributes", + ExtKeys, List.Transform(ExtKeys, (k) => "ext_" & k)) in - Expanded + ExtExpanded `.trim(); function arrayQuery(endpointPath) { From 0767980ac2a8030ece6384841c00c50ddfd02265 Mon Sep 17 00:00:00 2001 From: Wim van den Heijkant Date: Sun, 19 Apr 2026 13:09:49 +0200 Subject: [PATCH 037/160] Return extendedAttributes (and the rest) on /users and /resources MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both list endpoints were silently dropping ~half the columns of their underlying tables — including extendedAttributes. The UI didn't notice because it only renders displayName / UPN / dept / jobTitle. The Excel Power Query export, which auto-expands extendedAttributes into ext_* columns, was a no-op because the field never appeared in the JSON. /api/users now returns every Principals column: givenName, surname, employeeId, managerId, contextId, createdDateTime, extendedAttributes, riskScore, riskTier (in addition to the previous 10). /api/resources now returns every Resources column: mail, visibility, externalId, contextId, catalogId, isHidden, modifiedDateTime, riskScore, riskTier (in addition to the previous 8). The UI keeps working unchanged — extra fields fall through to JSON without rendering. extendedAttributes is parsed once on the server so both UI and Power Query receive a record/null instead of a string. Co-Authored-By: Claude Opus 4.7 (1M context) --- app/api/src/routes/resources.js | 6 ++++++ app/api/src/routes/tags.js | 18 ++++++++++++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/app/api/src/routes/resources.js b/app/api/src/routes/resources.js index cbdf2b20f..a11f00fcd 100644 --- a/app/api/src/routes/resources.js +++ b/app/api/src/routes/resources.js @@ -112,9 +112,15 @@ router.get('/resources', async (req, res) => { } where += filterWhere; + // Returns every Resources column so the same endpoint feeds the UI grid + // AND the Power Query Excel export (which auto-expands extendedAttributes + // into first-class ext_* columns). The UI ignores fields it doesn't need. const result = await request.query(` SELECT r.id, r."displayName", r."description", r."resourceType", r."systemId", r."enabled", r."createdDateTime", r."extendedAttributes", + r."mail", r."visibility", r."externalId", r."contextId", + r."catalogId", r."isHidden", r."modifiedDateTime", + r."riskScore", r."riskTier", (SELECT string_agg(t.id::text || ':' || t."name" || ':' || t."color", '|') FROM "GraphTagAssignments" ta INNER JOIN "GraphTags" t ON ta."tagId" = t.id AND t."entityType" IN ('resource', 'group') diff --git a/app/api/src/routes/tags.js b/app/api/src/routes/tags.js index ac759d49c..76681b9a5 100644 --- a/app/api/src/routes/tags.js +++ b/app/api/src/routes/tags.js @@ -441,10 +441,18 @@ router.get('/users', async (req, res) => { where += filterWhere; // Two-statement query: data + count, returned as recordsets[0] and [1]. + // Returns the FULL Principals row (every column on the table) so the same + // endpoint feeds both the UI table (which ignores extra columns) and the + // Excel Power Query export (which auto-expands extendedAttributes into + // first-class columns). The UI side is unaffected — extra fields fall + // through to the JSON without rendering. const result = await request.query(` SELECT u.id, u."displayName", u."email" AS "userPrincipalName", u."department", u."jobTitle", u."companyName", u."accountEnabled", u."principalType", u."systemId", u."externalId", + u."givenName", u."surname", u."employeeId", u."managerId", + u."contextId", u."createdDateTime", u."extendedAttributes", + u."riskScore", u."riskTier", (SELECT string_agg(t.id::text || ':' || t."name" || ':' || t."color", '|') FROM "GraphTagAssignments" ta INNER JOIN "GraphTags" t ON ta."tagId" = t.id AND t."entityType" = 'user' @@ -460,8 +468,14 @@ router.get('/users', async (req, res) => { `); const data = result.recordsets[0].map(r => { - const { tagString, ...rest } = r; - return { ...rest, tags: parseTags(tagString) }; + const { tagString, extendedAttributes, ...rest } = r; + // jsonb columns come back already-parsed from pg, but if it's a string + // (defensive — older shim path) parse it. Either way the UI gets a + // record/null and Power Query gets a record/null. + const parsedExt = extendedAttributes && typeof extendedAttributes === 'string' + ? (() => { try { return JSON.parse(extendedAttributes); } catch { return null; } })() + : extendedAttributes; + return { ...rest, extendedAttributes: parsedExt, tags: parseTags(tagString) }; }); res.json({ data, total: result.recordsets[1][0].total }); From 6b56ed2117304d08914f5a91895e7a2b9245ac51 Mon Sep 17 00:00:00 2001 From: Wim van den Heijkant Date: Sun, 19 Apr 2026 13:24:21 +0200 Subject: [PATCH 038/160] Raise /users + /resources page cap to 10k; harden M to track actual rows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit User reported still seeing only 4,000 of 7,911 rows after the previous List.Numbers fix. Root cause was a 500-row hard cap on /api/users: const limit = Math.min(parseInt(req.query.limit) || 100, 1, 500); The M code asked for `limit=1000` and trusted it, but the server returned 500. PageCount = ceil(7911/1000) = 8 → 8 pages * 500 = 4000. Same cap on /api/resources. Two-part fix: 1. Backend caps on /api/users and /api/resources raised from 500 to 10,000 to match the bulk-list endpoints. The UI defaults to 100 and never asks for more, so this only opens headroom for Power Query / BI exports. Per-page memory and response time stay bounded by the client's `limit` parameter. 2. The paginated M template no longer trusts the requested PageSize. It walks page-by-page, advancing state.off by `List.Count(page[data])` — whatever the server actually returned. Loop terminates when off >= Total or when a page returns zero rows. If anyone ever caps /api/users below 1000 again, the export keeps working (just slower). Vitest assertion added pinning the row-counting pattern (uses List.Count + advances by what came back) so the regression can't slip back in. Co-Authored-By: Claude Opus 4.7 (1M context) --- app/api/src/export/excelWorkbook.test.js | 17 +++++++++----- app/api/src/export/queryTemplates.js | 28 ++++++++++++++++++------ app/api/src/routes/resources.js | 4 +++- app/api/src/routes/tags.js | 5 ++++- 4 files changed, 39 insertions(+), 15 deletions(-) diff --git a/app/api/src/export/excelWorkbook.test.js b/app/api/src/export/excelWorkbook.test.js index f2cb5bdd2..24652aa4b 100644 --- a/app/api/src/export/excelWorkbook.test.js +++ b/app/api/src/export/excelWorkbook.test.js @@ -80,13 +80,18 @@ describe('generateWorkbook', () => { } }); - it('uses List.Numbers-based pagination (guards against the record-self-reference bug that capped the old List.Generate pattern at 4 pages)', () => { + it('paginates by actual-rows-received rather than arithmetic over Total (so it adapts if server caps below PageSize)', () => { + // Two failure modes this pins against: + // 1. The original List.Generate had `[off] = [off] + PageSize` — a + // forward self-reference that silently stopped at 4 pages. + // 2. A later attempt walked offsets arithmetically with + // Number.RoundUp(Total/PageSize); when /api/users capped rows at + // 500 instead of 1000 that stopped at 4000/7911 rows. + // Both are guarded by requiring the row-counting loop be in the M. const principals = wb.getWorksheet('Principals').getCell('A6').value; - expect(principals).toContain('List.Numbers'); - // The old buggy pattern had `[off] = [off] + PageSize` — a forward - // self-reference in a record literal — which halted iteration early. - // Guard against it creeping back. - expect(principals).not.toMatch(/off\s*=\s*\[off\]/); + expect(principals).toContain('List.Count'); // tracks actual rows + expect(principals).toContain('newOff'); // advance by what was returned + expect(principals).not.toMatch(/off\s*=\s*\[off\]/); // old record-self-ref }); it('auto-expands the extendedAttributes JSONB column', () => { diff --git a/app/api/src/export/queryTemplates.js b/app/api/src/export/queryTemplates.js index 95a55be06..df1c4b98b 100644 --- a/app/api/src/export/queryTemplates.js +++ b/app/api/src/export/queryTemplates.js @@ -49,13 +49,27 @@ let ])), First = FetchPage(0), Total = First[total], - PageCount = if Total = 0 then 0 else Number.RoundUp(Total / PageSize), - // Offsets: {0, PageSize, 2*PageSize, ...} for every page we need. - Offsets = List.Numbers(0, PageCount, PageSize), - // Skip the first offset because we already fetched it as First. - LaterPages = List.Transform(List.Skip(Offsets, 1), (off) => FetchPage(off)), - AllPageRecords = {First} & LaterPages, - AllRows = List.Combine(List.Transform(AllPageRecords, (p) => p[data])), + FirstRows = First[data], + // Walk pages by actual row count, not by arithmetic over Total. This is + // defensive: if the server ever caps below PageSize (the /users cap used + // to be 500 even when asked for 1000), an arithmetic walk silently stops + // at TotalBeforeCount records. Here we advance state[off] by whatever the + // last page actually returned, and stop when we've hit Total or when a + // page returns zero rows. + Pages = List.Generate( + () => [rows = FirstRows, off = List.Count(FirstRows), done = List.Count(FirstRows) >= Total or List.Count(FirstRows) = 0], + (state) => not state[done], + (state) => + let + nextPage = FetchPage(state[off]), + nextRows = nextPage[data], + nextCount = List.Count(nextRows), + newOff = state[off] + nextCount + in + [rows = nextRows, off = newOff, done = newOff >= Total or nextCount = 0], + (state) => state[rows] + ), + AllRows = List.Combine(Pages), Table = Table.FromList(AllRows, Splitter.SplitByNothing(), null, null, ExtraValues.Error), Expanded = if List.IsEmpty(AllRows) then Table else Table.ExpandRecordColumn(Table, "Column1", Record.FieldNames(AllRows{0})), diff --git a/app/api/src/routes/resources.js b/app/api/src/routes/resources.js index a11f00fcd..167603b50 100644 --- a/app/api/src/routes/resources.js +++ b/app/api/src/routes/resources.js @@ -44,7 +44,9 @@ router.get('/resources', async (req, res) => { const resourceType = (req.query.resourceType || '').trim(); const systemId = (req.query.systemId || '').trim(); const tagId = req.query.tagId ? parseInt(req.query.tagId) : null; - const limit = Math.min(Math.max(parseInt(req.query.limit) || 100, 1), 500); + // Cap matches the bulk-list endpoints; UI defaults to 100, Power Query + // walks in 1000-record pages. + const limit = Math.min(Math.max(parseInt(req.query.limit) || 100, 1), 10000); const offset = Math.max(parseInt(req.query.offset) || 0, 0); // Parse attribute filters diff --git a/app/api/src/routes/tags.js b/app/api/src/routes/tags.js index 76681b9a5..b79221aaa 100644 --- a/app/api/src/routes/tags.js +++ b/app/api/src/routes/tags.js @@ -398,7 +398,10 @@ router.get('/users', async (req, res) => { const search = (req.query.search || '').trim().slice(0, 200); const tagId = req.query.tagId ? parseInt(req.query.tagId) : null; - const limit = Math.min(Math.max(parseInt(req.query.limit) || 100, 1), 500); + // Cap at 10k to match the bulk-list endpoints. The UI defaults to 100 + // and never asks for more; the higher cap is there so Power Query / + // BI exports can page through the full dataset in fewer round trips. + const limit = Math.min(Math.max(parseInt(req.query.limit) || 100, 1), 10000); const offset = Math.max(parseInt(req.query.offset) || 0, 0); let attrFilters = {}; From 100d0b70a109d44f923169eb8b570b6fbc834e09 Mon Sep 17 00:00:00 2001 From: Wim van den Heijkant Date: Sun, 19 Apr 2026 15:02:27 +0200 Subject: [PATCH 039/160] =?UTF-8?q?Emit=20the=20tail=20partial=20page=20(7?= =?UTF-8?q?,000=20=E2=86=92=207,911=20on=20Principals)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Third pagination bug: the tail 911 rows went missing. The loop state carried a `done` boolean that flipped true when newOff >= Total — right on the same state that held the rows from the partial tail fetch. But List.Generate evaluates the condition BEFORE emitting, so a state whose condition says "stop" drops its own rows from the output. Restructured the state so rows and termination are independent: state = [rows = rows-to-emit, nextOff = offset of NEXT fetch] - selector always emits state[rows] - condition: List.Count(state[rows]) > 0 (does this state have data?) - next: if nextOff >= Total → return [rows = {}, nextOff = ...] else fetch one more page and wrap it The emptying-state trick lets the loop terminate cleanly: after the last "real" page is emitted, next() produces a state with 0 rows, the condition fails, the loop stops. No arithmetic relationship between PageSize / Total / row count is needed — the only thing the code trusts is the `total` field and the count of rows actually returned per page. Vitest extended with three assertions pinning the new design (rows-based condition, no `done` flag, no record self-reference). Co-Authored-By: Claude Opus 4.7 (1M context) --- app/api/src/export/excelWorkbook.test.js | 28 +++++++++++------ app/api/src/export/queryTemplates.js | 39 +++++++++++++++--------- 2 files changed, 42 insertions(+), 25 deletions(-) diff --git a/app/api/src/export/excelWorkbook.test.js b/app/api/src/export/excelWorkbook.test.js index 24652aa4b..7ab68b581 100644 --- a/app/api/src/export/excelWorkbook.test.js +++ b/app/api/src/export/excelWorkbook.test.js @@ -81,17 +81,25 @@ describe('generateWorkbook', () => { }); it('paginates by actual-rows-received rather than arithmetic over Total (so it adapts if server caps below PageSize)', () => { - // Two failure modes this pins against: - // 1. The original List.Generate had `[off] = [off] + PageSize` — a - // forward self-reference that silently stopped at 4 pages. - // 2. A later attempt walked offsets arithmetically with - // Number.RoundUp(Total/PageSize); when /api/users capped rows at - // 500 instead of 1000 that stopped at 4000/7911 rows. - // Both are guarded by requiring the row-counting loop be in the M. + // Three previous bugs this pins against: + // 1. List.Generate with `[off] = [off] + PageSize` — record + // self-reference stopped at 4 pages (4,000/7,911). + // 2. Arithmetic walk with Number.RoundUp(Total/PageSize) stopped + // when server capped below PageSize (4,000 again). + // 3. `done`-flag variant dropped the tail partial page: when + // newOff >= Total the condition flipped false on the same + // state that held the rows, so the last 911 rows were never + // emitted (7,000/7,911). + // The stable design: state carries "rows to emit" + "offset of next + // fetch"; selector emits unconditionally; condition asks "does this + // state have rows?"; next() short-circuits to an empty state when + // there's nothing more to fetch so the next condition stops the loop. const principals = wb.getWorksheet('Principals').getCell('A6').value; - expect(principals).toContain('List.Count'); // tracks actual rows - expect(principals).toContain('newOff'); // advance by what was returned - expect(principals).not.toMatch(/off\s*=\s*\[off\]/); // old record-self-ref + expect(principals).toContain('List.Count'); // counts actual rows + expect(principals).toContain('nextOff'); // offset of next fetch + expect(principals).toContain('List.Count(state[rows]) > 0'); // condition based on emitted rows + expect(principals).not.toMatch(/done\s*=/); // old boolean-flag pattern + expect(principals).not.toMatch(/off\s*=\s*\[off\]/); // old record-self-ref }); it('auto-expands the extendedAttributes JSONB column', () => { diff --git a/app/api/src/export/queryTemplates.js b/app/api/src/export/queryTemplates.js index df1c4b98b..e67ae8a1c 100644 --- a/app/api/src/export/queryTemplates.js +++ b/app/api/src/export/queryTemplates.js @@ -50,23 +50,32 @@ let First = FetchPage(0), Total = First[total], FirstRows = First[data], - // Walk pages by actual row count, not by arithmetic over Total. This is - // defensive: if the server ever caps below PageSize (the /users cap used - // to be 500 even when asked for 1000), an arithmetic walk silently stops - // at TotalBeforeCount records. Here we advance state[off] by whatever the - // last page actually returned, and stop when we've hit Total or when a - // page returns zero rows. + // Walk pages by actual row count — not by arithmetic over Total. Two + // failure modes this design guards against: + // 1. The server returns fewer rows than PageSize (historical cap, or a + // partial tail page). Arithmetic walks silently truncate. + // 2. The last page (partial) gets dropped by the condition check. The + // previous version had a `done` flag that flipped true right when + // the tail page had been fetched, and List.Generate skips a state + // whose condition evaluates false — so 911 rows on the last page + // vanished and the user saw 7,000 instead of 7,911. + // + // Contract of the state: [rows = rows-to-emit, nextOff = offset of NEXT + // fetch]. Selector emits state[rows] unconditionally; condition asks + // "does the state have rows to emit?"; next() fetches one more page + // (or short-circuits to an empty state when we've reached Total so the + // next condition evaluation stops the loop cleanly). Pages = List.Generate( - () => [rows = FirstRows, off = List.Count(FirstRows), done = List.Count(FirstRows) >= Total or List.Count(FirstRows) = 0], - (state) => not state[done], + () => [rows = FirstRows, nextOff = List.Count(FirstRows)], + (state) => List.Count(state[rows]) > 0, (state) => - let - nextPage = FetchPage(state[off]), - nextRows = nextPage[data], - nextCount = List.Count(nextRows), - newOff = state[off] + nextCount - in - [rows = nextRows, off = newOff, done = newOff >= Total or nextCount = 0], + if state[nextOff] >= Total then [rows = {}, nextOff = state[nextOff]] + else + let + page = FetchPage(state[nextOff]), + fetched = page[data] + in + [rows = fetched, nextOff = state[nextOff] + List.Count(fetched)], (state) => state[rows] ), AllRows = List.Combine(Pages), From 64ac867b00c4dcc54382b846ecc1d8b2ec3db4a0 Mon Sep 17 00:00:00 2001 From: Wim van den Heijkant Date: Sun, 19 Apr 2026 15:06:34 +0200 Subject: [PATCH 040/160] Escape backticks in M-comment JS template literal The previous commit put literal backticks around "done" inside a JS template literal (the PAGINATED_FETCH constant), terminating the template string early and breaking the module on startup. Escaping as \` keeps the JS string whole; the M output now contains \`done\` in the comment, which M treats as ordinary comment text and ignores. Co-Authored-By: Claude Opus 4.7 (1M context) --- app/api/src/export/queryTemplates.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/src/export/queryTemplates.js b/app/api/src/export/queryTemplates.js index e67ae8a1c..2b7ddf56d 100644 --- a/app/api/src/export/queryTemplates.js +++ b/app/api/src/export/queryTemplates.js @@ -55,7 +55,7 @@ let // 1. The server returns fewer rows than PageSize (historical cap, or a // partial tail page). Arithmetic walks silently truncate. // 2. The last page (partial) gets dropped by the condition check. The - // previous version had a `done` flag that flipped true right when + // previous version had a \`done\` flag that flipped true right when // the tail page had been fetched, and List.Generate skips a state // whose condition evaluates false — so 911 rows on the last page // vanished and the user saw 7,000 instead of 7,911. From ffce64509fbc80c62c8e2c2cdeec8c45a17d1a98 Mon Sep 17 00:00:00 2001 From: Wim van den Heijkant Date: Sun, 19 Apr 2026 15:44:29 +0200 Subject: [PATCH 041/160] Docs + auth-middleware test for the Power Query feature MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds docs/admin/excel-powerquery-export.md covering the end-to-end flow: what's in the download, how to use it, how to rotate/revoke tokens, how to retarget a workbook at another deployment, and the security posture of the read API tokens. The existing docs/admin/excel-template-authoring.md keeps the maintainer guide for the follow-up polished-template work. Adds auth.test.js — 10 unit tests pinning the security-critical scoping rules on the `fgr_` path of authMiddleware: - GET on non-admin endpoint, active token → next() called - GET on unknown/revoked/expired token → 401 - Any mutating method (POST/PUT/PATCH/DELETE) → 403 regardless of token validity - Any /api/admin/* path → 403 - Both scoping checks short-circuit before the DB lookup so a leaked token can't drive load on ReadApiKeys by hammering admin paths - Auth disabled (isAuthEnabled() = false) still bypasses as a no-op - Header hygiene: missing, malformed, or non-Bearer 401 Full vitest: 140/140 green (10 new). Co-Authored-By: Claude Opus 4.7 (1M context) --- app/api/src/middleware/auth.test.js | 163 ++++++++++++++++++++++++++ docs/admin/excel-powerquery-export.md | 137 ++++++++++++++++++++++ 2 files changed, 300 insertions(+) create mode 100644 app/api/src/middleware/auth.test.js create mode 100644 docs/admin/excel-powerquery-export.md diff --git a/app/api/src/middleware/auth.test.js b/app/api/src/middleware/auth.test.js new file mode 100644 index 000000000..608b27e92 --- /dev/null +++ b/app/api/src/middleware/auth.test.js @@ -0,0 +1,163 @@ +// Unit tests for the read-API-token path of authMiddleware. +// +// The JWT path is covered by the existing auth integration tests; here we +// pin the security-critical behaviours unique to the `fgr_…` credential: +// +// - Accepted ONLY on GET requests (POST/PUT/PATCH/DELETE → 403). +// - Accepted ONLY on non-admin paths (/api/admin/* → 403). +// - Accepted ONLY when the token resolves to an active, non-expired, +// non-revoked row in ReadApiKeys. +// - Falls through to next() on success, attaching req.readToken so +// downstream code can audit. +// +// Auth is enabled by mocking authConfig.isAuthEnabled() = true — otherwise +// the middleware short-circuits and these scoping checks never run. + +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +// Hoisted mocks. Vitest lifts vi.mock calls to the top of the module. +const findActive = vi.fn(); +vi.mock('../auth/readTokens.js', async () => { + const actual = await vi.importActual('../auth/readTokens.js'); + return { + ...actual, + findActiveByPlaintext: (...args) => findActive(...args), + }; +}); + +const isAuthEnabledMock = vi.fn(() => true); +vi.mock('../config/authConfig.js', () => ({ + isAuthEnabled: (...args) => isAuthEnabledMock(...args), + getJwksClient: () => null, + getTenantId: () => 'test-tenant', + getClientId: () => 'test-client', + getRequiredRoles: () => [], +})); + +const { authMiddleware } = await import('./auth.js'); + +// Minimal express-ish req/res/next stand-ins. res captures status + json body. +function makeReq({ path = '/api/users', method = 'GET', token }) { + return { + path, + method, + originalUrl: path, + headers: token ? { authorization: `Bearer ${token}` } : {}, + }; +} +function makeRes() { + const res = { + statusCode: null, + body: null, + status(code) { this.statusCode = code; return this; }, + json(obj) { this.body = obj; return this; }, + }; + return res; +} +async function run(req) { + const res = makeRes(); + let called = false; + const next = () => { called = true; }; + authMiddleware(req, res, next); + // authMiddleware's fgr_ path is async (promise resolves off the call + // stack) — give the microtask queue a tick to drain before asserting. + await new Promise(r => setImmediate(r)); + return { res, nextCalled: called }; +} + +beforeEach(() => { + findActive.mockReset(); + isAuthEnabledMock.mockReturnValue(true); +}); + +const GOOD_TOKEN = 'fgr_AbCdEfGhIjKlMnOpQrStUvWxYz0123456789_-abcdefg'; + +describe('authMiddleware — fgr_ token on GET non-admin endpoints', () => { + it('accepts and calls next() when the token is active', async () => { + findActive.mockResolvedValue({ id: 7, name: 'Analyst workbook', expiresAt: null, revoked: false }); + const { res, nextCalled } = await run(makeReq({ token: GOOD_TOKEN })); + expect(nextCalled).toBe(true); + expect(res.statusCode).toBeNull(); + }); + + it('rejects with 401 when the token is unknown', async () => { + findActive.mockResolvedValue(null); + const { res, nextCalled } = await run(makeReq({ token: GOOD_TOKEN })); + expect(nextCalled).toBe(false); + expect(res.statusCode).toBe(401); + expect(res.body.error).toMatch(/invalid/i); + }); + + it('attaches req.readToken on success so downstream code can audit', async () => { + findActive.mockResolvedValue({ id: 7, name: 'CI export', expiresAt: null, revoked: false }); + const req = makeReq({ token: GOOD_TOKEN }); + await run(req); + expect(req.readToken).toEqual({ id: 7, name: 'CI export' }); + }); +}); + +describe('authMiddleware — fgr_ token scoping', () => { + it('rejects any method other than GET with 403', async () => { + for (const method of ['POST', 'PUT', 'PATCH', 'DELETE']) { + findActive.mockResolvedValue({ id: 7, name: 'x', expiresAt: null, revoked: false }); + const { res, nextCalled } = await run(makeReq({ method, token: GOOD_TOKEN })); + expect(nextCalled).toBe(false); + expect(res.statusCode).toBe(403); + expect(res.body.error).toMatch(/GET/); + } + }); + + it('rejects any /api/admin/* path with 403 (even on GET)', async () => { + findActive.mockResolvedValue({ id: 7, name: 'x', expiresAt: null, revoked: false }); + for (const path of ['/api/admin/read-tokens', '/api/admin/crawlers', '/api/admin/export/curated']) { + const { res, nextCalled } = await run(makeReq({ path, token: GOOD_TOKEN })); + expect(nextCalled).toBe(false); + expect(res.statusCode).toBe(403); + expect(res.body.error).toMatch(/admin/i); + } + }); + + it('short-circuits: admin-path check happens before the DB lookup', async () => { + // Defence in depth: even with a valid token, never touch the DB on a + // path we're going to reject anyway. Keeps a leaked token that's + // pointed at admin endpoints from driving load on ReadApiKeys. + findActive.mockResolvedValue({ id: 7, name: 'x', expiresAt: null, revoked: false }); + await run(makeReq({ path: '/api/admin/anything', token: GOOD_TOKEN })); + expect(findActive).not.toHaveBeenCalled(); + }); + + it('short-circuits: method check happens before the DB lookup', async () => { + findActive.mockResolvedValue({ id: 7, name: 'x', expiresAt: null, revoked: false }); + await run(makeReq({ method: 'POST', token: GOOD_TOKEN })); + expect(findActive).not.toHaveBeenCalled(); + }); +}); + +describe('authMiddleware — auth disabled bypass', () => { + it('skips all checks entirely when isAuthEnabled() is false', async () => { + isAuthEnabledMock.mockReturnValue(false); + // No token at all — still should call next. + const { res, nextCalled } = await run(makeReq({ token: null })); + expect(nextCalled).toBe(true); + expect(res.statusCode).toBeNull(); + expect(findActive).not.toHaveBeenCalled(); + }); +}); + +describe('authMiddleware — header hygiene', () => { + it('returns 401 when no Authorization header is set and auth is on', async () => { + const { res, nextCalled } = await run(makeReq({ token: null })); + expect(nextCalled).toBe(false); + expect(res.statusCode).toBe(401); + }); + + it('returns 401 when the header does not start with Bearer', async () => { + const req = { + path: '/api/users', method: 'GET', originalUrl: '/api/users', + headers: { authorization: GOOD_TOKEN }, // missing "Bearer " + }; + const { res, nextCalled } = await run(req); + expect(nextCalled).toBe(false); + expect(res.statusCode).toBe(401); + }); +}); diff --git a/docs/admin/excel-powerquery-export.md b/docs/admin/excel-powerquery-export.md new file mode 100644 index 000000000..c4dd45b7a --- /dev/null +++ b/docs/admin/excel-powerquery-export.md @@ -0,0 +1,137 @@ +# Excel Power Query workbook export + +Identity Atlas can hand a data analyst a pre-configured Excel workbook that +pulls live data from the API via Power Query. A freshly-minted read-only +token is embedded in the workbook, so refreshing the data on any machine is +a single click — no API credentials to paste, no connections to configure. + +## Who this is for + +- Data analysts who want Identity Atlas data in Excel or Power BI without + learning the API +- Anyone building ad-hoc reports (access reviews, risk dashboards, compliance + reports) against live principal / assignment / resource data +- Teams that want to pipe Identity Atlas into an existing BI stack via the + API but without standing up interactive OAuth for their service accounts + +## What the download contains + +One `.xlsx` file with these sheets: + +| Sheet | Endpoint | Contents | +| ---------------------- | ----------------------------------- | -------------------------------------------------------- | +| README | — | Usage notes | +| Settings | — | `BaseUrl` + `AuthToken` pre-populated named-range cells | +| Systems | `GET /api/systems` | Connected systems (Entra ID, Omada, CSV-backed, …) | +| Principals | `GET /api/users` | Every principalType — users, service principals, MIs, AI agents | +| Resources | `GET /api/resources` | Groups, directory roles, app roles, business roles | +| Assignments | `GET /api/assignments` | Who has access to what (from `ResourceAssignments`) | +| Identities | `GET /api/identities` | Real-person identities aggregated from multiple accounts | +| IdentityMembers | `GET /api/identity-members` | Identity ↔ account links | +| ResourceRelationships | `GET /api/resource-relationships` | Parent↔child resource links (Contains, GrantsAccessTo) | + +The Principals and Resources tabs auto-expand the `extendedAttributes` JSONB +column into first-class `ext_*` columns (`ext_userType`, +`ext_onPremisesSyncEnabled`, `ext_signInActivity`, `ext_appId`, etc.). Keys +are collected across every row, so sparsely-populated attributes still show +up. + +## How to use it + +1. **Admin → Data → Excel Power Query Workbook**. +2. Click **Generate token & download workbook**. A token is created and + the workbook is streamed back (a few KB). +3. Open the file in Excel. +4. For each data tab (Principals, Resources, Assignments, …): + - Go to **Data → Get Data → From Other Sources → Blank Query**. + - In Power Query Editor: **Home → Advanced Editor**. + - Copy the M code from cell A6 of that sheet and paste it into the + editor. Click **Done**. + - Rename the query to match the sheet name (e.g. `Principals`). + - **Home → Close & Load To… → Existing worksheet → that sheet, cell A1**. +5. Back in Excel, **Data → Refresh All**. Every sheet fills with live data. + +> **One-click refresh coming soon.** The current workbook requires the +> paste-into-Advanced-Editor step per sheet. A follow-up PR ships a +> hand-built template where the queries auto-load on first open — at +> that point the flow becomes "download → open → Refresh All". + +## Rotating or retiring tokens + +Each download mints a **new** read token so you have a fresh credential. +The **Existing tokens** table on the same page lists every outstanding +token with its prefix, creation date, and last-used timestamp. Click +**Revoke** to invalidate any token immediately — workbooks using that +token stop refreshing on their next attempt. + +## How the token is used against the API + +The workbook's M code reads the token from the `AuthToken` named range +and sends it on every request: + +``` +Authorization: Bearer fgr_… +``` + +The token works on **every read endpoint** (`/api/users`, +`/api/resources`, `/api/systems`, `/api/assignments`, +`/api/identity-members`, `/api/resource-relationships`, +`/api/identities`, etc.). It does **not** work on mutating endpoints +(POST/PUT/PATCH/DELETE) nor on any `/api/admin/*` endpoint — the auth +middleware rejects those with HTTP 403. + +That scoping means you can hand the token to a BI stack, a script, or a +curl one-liner and the blast radius of a leak is read access only. It is +**not** a substitute for a user's Entra ID sign-in when you need to +manage data. + +## Updating a workbook to point at a different deployment + +The workbook works against whatever Identity Atlas host generated it. +To retarget an existing workbook (e.g. point a locally-authored workbook +at a production deployment): + +1. Open the workbook's **Settings** sheet. +2. Change cell **B2** (`BaseUrl`) to the new `/api` base — e.g. + `https://identityatlas.example.com/api`. +3. Paste in a new read token you generated on the target deployment in + cell **B3** (`AuthToken`). +4. **Refresh All**. Every query picks up the new values because they all + read from the named ranges. + +No M code edits needed — you only touch those two cells. + +## Building your own reports from this + +The workbook's tabs are just starting points. Once the raw data is loaded +as Excel tables, you can: + +- Pivot any tab against any other via Power Query's **Merge** feature + (e.g. Assignments ⨝ Principals on `principalId`) +- Build pivot tables directly on a loaded sheet +- Use the `ext_*` columns as pivot filters for fine-grained slicing + (e.g. "Service Principals with `ext_servicePrincipalType = ManagedIdentity`") +- Chain into Power BI — same M code, same token pattern, same endpoints + +## Security notes + +- **Tokens are secrets.** The `Settings` sheet is not encrypted. If you + share the workbook, anyone with the file can read from the API as + long as the token is active. Rotate the token after sharing. +- **Tokens are read-only.** They cannot mutate data or reach admin + endpoints — but they CAN list every principal, resource, and + assignment. Treat that with the same discretion as a database dump. +- **Revoke proactively.** When someone leaves the team, or when a + workbook is superseded, revoke the token from Admin → Data. The + token's `lastUsedAt` timestamp helps you tell active from stale. +- **Per-user tokens.** Click **Create token only…** to mint named + tokens that don't get bundled into a workbook, useful for scripts or + BI connectors. The Admin → Data list shows which person/integration + owns each one. + +## Related + +- [Authoring the polished XLSX template](./excel-template-authoring.md) — + for the maintainer wiring up the one-click-refresh template +- [API reference](../api/index.md) — the endpoints your Power Query + code hits From 052a3467682d4e6711abe3b99c44d9cfb7eb4011 Mon Sep 17 00:00:00 2001 From: Wim van den Heijkant Date: Sun, 19 Apr 2026 16:06:27 +0200 Subject: [PATCH 042/160] docs --- docs/admin/excel-template-authoring.md | 139 +++++++++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100644 docs/admin/excel-template-authoring.md diff --git a/docs/admin/excel-template-authoring.md b/docs/admin/excel-template-authoring.md new file mode 100644 index 000000000..4f7b4ac3a --- /dev/null +++ b/docs/admin/excel-template-authoring.md @@ -0,0 +1,139 @@ +# Authoring the Excel Power Query template + +This document is for the maintainer creating `tools/excel-queries/template.xlsx`. +End users never see this file — they only get a copy with their token stamped +in. The template is the **shape** of the workbook (sheets, named ranges, +queries) that the backend mutates per request. + +When this file exists in the repo, the download endpoint switches from the +"M-as-text, paste yourself" MVP to the polished "open → click Refresh" flow +automatically. No code change needed once the template is committed. + +## The two placeholders + +The backend swaps two strings in the saved workbook: + +| Placeholder | Settings sheet cell | Will be replaced with | +| ---------------- | ------------------- | ------------------------------------- | +| `{{BASE_URL}}` | B2 | The download host's `/api` base | +| `{{AUTH_TOKEN}}` | B3 | A freshly minted `fgr_…` read token | + +Use these exact strings — the backend matches them verbatim. Don't introduce +extra whitespace inside the curly braces. + +## Step by step + +### 1. Bootstrap the workbook with a real token (so you can validate) + +1. Open Identity Atlas → **Admin → Data → Excel Power Query Workbook**. +2. Click **Generate token & download workbook**. Save the file somewhere + throwaway — you'll only use this to crib the M code from each tab. +3. Note your local API URL (`http://localhost:3001/api`) and the token + shown in the Settings sheet of that downloaded file. + +### 2. Build the template structure + +In a fresh Excel workbook, create these sheets, in this order. Sheet names +must match exactly — they're the named-range / connection labels: + +1. **Settings** +2. **Systems** +3. **Principals** +4. **Resources** +5. **Assignments** +6. **Identities** +7. **IdentityMembers** +8. **ResourceRelationships** + +(Optional but nice: add a **README** sheet at the front with usage notes.) + +### 3. Wire up the Settings sheet + +On the **Settings** sheet: + +| Cell | Value | +| ---- | ------------- | +| A1 | Setting | +| B1 | Value | +| A2 | BaseUrl | +| B2 | _your real `http://localhost:3001/api` for now_ | +| A3 | AuthToken | +| B3 | _your real `fgr_…` token for now_ | + +Define two named ranges (Formulas → Name Manager → New): + +- **BaseUrl** → refers to `=Settings!$B$2` +- **AuthToken** → refers to `=Settings!$B$3` + +> The Power Query M code reads these via +> `Excel.CurrentWorkbook(){[Name="BaseUrl"]}[Content]{0}[Column1]`. If the +> names go missing, every query stops working at refresh time. + +### 4. Add one Power Query per data sheet + +For each of the 7 data sheets: + +1. Activate that sheet (e.g. **Principals**). +2. **Data → Get Data → From Other Sources → Blank Query**. +3. The Power Query Editor opens. **Home → Advanced Editor**. +4. Paste the M code for that sheet (see "M code per sheet" below). +5. Click **Done**. +6. In the left sidebar (Queries), rename the query to match the sheet name + exactly (e.g. `Principals` not `Query1`). +7. **Home → Close & Load To… → Existing worksheet → cell A1 of that sheet**. + This wires the query output to a table on the right sheet. + +Repeat for all 7 sheets. After each one, click **Refresh All** to confirm +data loads correctly. If a query fails, fix the M code and re-load. + +### 5. Replace cell values with placeholders, save, do NOT refresh again + +Once everything refreshes cleanly: + +1. On the Settings sheet, change cell **B2** to `{{BASE_URL}}` (literal text). +2. Change cell **B3** to `{{AUTH_TOKEN}}`. +3. **Do not click Refresh** — the queries would try to hit + `http://{{BASE_URL}}/users` and fail. Just save. +4. **File → Save As** → `tools/excel-queries/template.xlsx`. +5. Commit. + +The backend test suite has fixtures pinning the placeholder strings — if +they ever change, tests catch it. + +## M code per sheet + +Each block goes verbatim into the Advanced Editor. The names match the +constants in [`app/api/src/export/queryTemplates.js`](../../app/api/src/export/queryTemplates.js) +— if the templates change there, regenerate the workbook by downloading +the M-as-text MVP version (Admin → Data → Generate token & download +workbook), open it, and copy the updated M from each sheet's cell A6. + +(See the downloaded MVP file. Each sheet's A6 cell contains the exact M +code for that endpoint, ready to paste.) + +## Sanity-check the saved file + +Before committing: + +1. Close Excel. +2. Reopen `template.xlsx`. The Settings sheet should show the two + `{{...}}` placeholders verbatim. +3. **Don't click Refresh** — there's nothing to refresh against. +4. Right-click each query in the Queries pane → **Properties** → confirm + the query name matches the sheet name. + +## What the backend does at download time + +The download endpoint (`POST /api/admin/data-export/workbook`) does this +to each request: + +1. Mint a new `fgr_…` read token. +2. Open `tools/excel-queries/template.xlsx` as a zip. +3. Replace `{{BASE_URL}}` with the request's API base in every XML part. +4. Replace `{{AUTH_TOKEN}}` with the new token in every XML part. +5. Re-zip and stream the result. + +Excel stores cell values either inline or in `xl/sharedStrings.xml`. The +backend doesn't care which — it does string replace on the entire archive, +so as long as the placeholders are unique (which `{{BASE_URL}}` and +`{{AUTH_TOKEN}}` are, by construction) the swap is safe. From 414258e33ecf8c6463972dffc2c73951fa582eb3 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 19 Apr 2026 14:09:46 +0000 Subject: [PATCH 043/160] chore: bump version to 5.11.20260419.1409 --- CHANGES.md | 4 ++++ changes/fix-issue-115.md | 1 - setup/IdentityAtlas.psd1 | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) delete mode 100644 changes/fix-issue-115.md diff --git a/CHANGES.md b/CHANGES.md index 3a4b5a44d..9f4c9c6de 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,9 @@ ## Changes in this PR +- Fixed Quick Start documentation: Image channel switching code now includes Windows PowerShell syntax + +## Changes in this PR + - Replaced long-lived release branches with git tags for release management — hotfixes now ship only the fix, without features already merged to main - Added "Cut Release" workflow: tags vX.Y.Z on main HEAD, triggers :latest publish - Added "Cut Hotfix" workflow: tags a hotfix branch commit as a new patch version, triggers :latest publish diff --git a/changes/fix-issue-115.md b/changes/fix-issue-115.md deleted file mode 100644 index 1ca1317d7..000000000 --- a/changes/fix-issue-115.md +++ /dev/null @@ -1 +0,0 @@ -- Fixed Quick Start documentation: Image channel switching code now includes Windows PowerShell syntax diff --git a/setup/IdentityAtlas.psd1 b/setup/IdentityAtlas.psd1 index 8919cb6d8..9a8ec8dab 100644 --- a/setup/IdentityAtlas.psd1 +++ b/setup/IdentityAtlas.psd1 @@ -12,7 +12,7 @@ RootModule = '.\IdentityAtlas.psm1' # Version number of this module. -ModuleVersion = '5.10.20260419.1050' +ModuleVersion = '5.11.20260419.1409' # Supported PSEditions # CompatiblePSEditions = @() From 1aca5c1050a9fdce2f02674344df67f1a13e1171 Mon Sep 17 00:00:00 2001 From: Taeke Kooiker Date: Sun, 19 Apr 2026 16:20:13 +0200 Subject: [PATCH 044/160] fix: docker-publish workflow not triggering after PR merges to main The workflow_run branches filter matched the feature branch (from the pull_request event), not main, so the trigger was silently skipped. Also fixed head_branch resolving to the feature branch instead of main. Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/docker-publish.yml | 9 +++++---- changes/fix-docker-publish-trigger.md | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) create mode 100644 changes/fix-docker-publish-trigger.md diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 8283f97c2..1991611cd 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -19,8 +19,8 @@ on: workflow_run: workflows: ["Bump version on PR merge"] types: [completed] - branches: - - main + # No branches filter: the filter matches the feature branch that triggered + # bump-version (via pull_request), not main — so it would never fire. push: tags: - 'v*' @@ -57,8 +57,9 @@ jobs: # Tag push: github.ref_name = v5.2.0 REF="${{ github.ref_name }}" else - # workflow_run from bump-version on main - REF="${{ github.event.workflow_run.head_branch }}" + # workflow_run from bump-version: always build main (bump-version + # only runs on PR merge to main, so the result always lands on main) + REF="main" fi echo "ref=$REF" >> "$GITHUB_OUTPUT" diff --git a/changes/fix-docker-publish-trigger.md b/changes/fix-docker-publish-trigger.md new file mode 100644 index 000000000..cc854ed97 --- /dev/null +++ b/changes/fix-docker-publish-trigger.md @@ -0,0 +1 @@ +- Fixed Docker image publishing not triggering automatically after PR merges to main From ca07a725e98e9c8bf6bd68a28991c635fdba4df4 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 19 Apr 2026 14:37:33 +0000 Subject: [PATCH 045/160] chore: bump version to 5.12.20260419.1437 --- CHANGES.md | 4 ++++ changes/fix-docker-publish-trigger.md | 1 - setup/IdentityAtlas.psd1 | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) delete mode 100644 changes/fix-docker-publish-trigger.md diff --git a/CHANGES.md b/CHANGES.md index 9f4c9c6de..39877a410 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,9 @@ ## Changes in this PR +- Fixed Docker image publishing not triggering automatically after PR merges to main + +## Changes in this PR + - Fixed Quick Start documentation: Image channel switching code now includes Windows PowerShell syntax ## Changes in this PR diff --git a/changes/fix-docker-publish-trigger.md b/changes/fix-docker-publish-trigger.md deleted file mode 100644 index cc854ed97..000000000 --- a/changes/fix-docker-publish-trigger.md +++ /dev/null @@ -1 +0,0 @@ -- Fixed Docker image publishing not triggering automatically after PR merges to main diff --git a/setup/IdentityAtlas.psd1 b/setup/IdentityAtlas.psd1 index 9a8ec8dab..194f38c35 100644 --- a/setup/IdentityAtlas.psd1 +++ b/setup/IdentityAtlas.psd1 @@ -12,7 +12,7 @@ RootModule = '.\IdentityAtlas.psm1' # Version number of this module. -ModuleVersion = '5.11.20260419.1409' +ModuleVersion = '5.12.20260419.1437' # Supported PSEditions # CompatiblePSEditions = @() From 879b91926dba441c38a5af942d9e6a94151b6037 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 20 Apr 2026 06:26:50 +0000 Subject: [PATCH 046/160] chore: bump version to 5.13.20260420.0626 --- CHANGES.md | 7 +++++++ changes/feature-excel-powerquery-export.md | 4 ---- setup/IdentityAtlas.psd1 | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) delete mode 100644 changes/feature-excel-powerquery-export.md diff --git a/CHANGES.md b/CHANGES.md index 39877a410..e182c4b1b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,12 @@ ## Changes in this PR +- Added a one-click Excel Power Query workbook download under **Admin → Data → Excel Power Query Workbook**. Clicking "Generate token & download workbook" mints a read-only API key, embeds it in the workbook's Settings sheet alongside the API URL, and streams the file back. Data analysts can drop the file on any machine and refresh from any deployment without ever editing M code. +- Tabs included: Systems, Principals, Resources, Assignments, Identities, IdentityMembers, ResourceRelationships. Each tab includes pre-written paginated Power Query M code that the user pastes into Power Query Editor (Data → Get Data → Other Sources → Blank Query). A future iteration will swap in a hand-built template that loads queries automatically — the rest of the plumbing is already in place. +- New read-only API token type (`fgr_…`). Tokens are accepted on GET requests to non-admin endpoints only — they cannot mutate data or reach any admin endpoint, even if leaked. Stored as SHA-256 hashes; plaintext is shown to the operator exactly once at creation. Tokens can be revoked from the same admin section, which immediately stops every workbook holding them from refreshing. +- New bulk list endpoints for the join tables that previously had no flat listing: `/api/assignments`, `/api/identity-members`, `/api/resource-relationships`. All are paginated (`?limit=1000&offset=N`, max 10 000 per page) with optional `?systemId=N` filter, returning `{ data, total }` like every other list endpoint. + +## Changes in this PR + - Fixed Docker image publishing not triggering automatically after PR merges to main ## Changes in this PR diff --git a/changes/feature-excel-powerquery-export.md b/changes/feature-excel-powerquery-export.md deleted file mode 100644 index 47f642074..000000000 --- a/changes/feature-excel-powerquery-export.md +++ /dev/null @@ -1,4 +0,0 @@ -- Added a one-click Excel Power Query workbook download under **Admin → Data → Excel Power Query Workbook**. Clicking "Generate token & download workbook" mints a read-only API key, embeds it in the workbook's Settings sheet alongside the API URL, and streams the file back. Data analysts can drop the file on any machine and refresh from any deployment without ever editing M code. -- Tabs included: Systems, Principals, Resources, Assignments, Identities, IdentityMembers, ResourceRelationships. Each tab includes pre-written paginated Power Query M code that the user pastes into Power Query Editor (Data → Get Data → Other Sources → Blank Query). A future iteration will swap in a hand-built template that loads queries automatically — the rest of the plumbing is already in place. -- New read-only API token type (`fgr_…`). Tokens are accepted on GET requests to non-admin endpoints only — they cannot mutate data or reach any admin endpoint, even if leaked. Stored as SHA-256 hashes; plaintext is shown to the operator exactly once at creation. Tokens can be revoked from the same admin section, which immediately stops every workbook holding them from refreshing. -- New bulk list endpoints for the join tables that previously had no flat listing: `/api/assignments`, `/api/identity-members`, `/api/resource-relationships`. All are paginated (`?limit=1000&offset=N`, max 10 000 per page) with optional `?systemId=N` filter, returning `{ data, total }` like every other list endpoint. diff --git a/setup/IdentityAtlas.psd1 b/setup/IdentityAtlas.psd1 index 194f38c35..e9aa21405 100644 --- a/setup/IdentityAtlas.psd1 +++ b/setup/IdentityAtlas.psd1 @@ -12,7 +12,7 @@ RootModule = '.\IdentityAtlas.psm1' # Version number of this module. -ModuleVersion = '5.12.20260419.1437' +ModuleVersion = '5.13.20260420.0626' # Supported PSEditions # CompatiblePSEditions = @() From 8f3f8405d4c529b8e2e9d2ce6b6dc89228178f43 Mon Sep 17 00:00:00 2001 From: Wim van den Heijkant Date: Mon, 20 Apr 2026 09:02:16 +0200 Subject: [PATCH 047/160] Add calculated Link + OuPath fields to Entra-synced objects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every Entra object synced into Identity Atlas now gets two derived fields in extendedAttributes: Link Deep link into the Entra admin portal. Same blade URLs the Identity Atlas UI uses on its "Open in Entra ID" buttons on user / group / resource detail pages — picking either surface opens the same page. Driven by object type: User → UserProfileMenuBlade Group → GroupDetailsMenuBlade ServicePrincipal → ManagedAppMenuBlade (+ appId when available) Application → ApplicationMenuBlade (+ appId) _OuPath Forward-slash-separated OU path in root → leaf order, derived from any DN-shaped value on the object. Example: onPremisesDistinguishedName = "CN=204374,OU=Users,OU=Accounts,OU=Clients,DC=fujitsu,DC=ad,…" onPremisesDistinguishedName_OuPath = "Clients\Accounts\Users" CN / DC / UID components are dropped; LDAP's innermost-first ordering is reversed so the output reads top-to-bottom. Detection is opt-in by SHAPE (via Test-FGDistinguishedName) — not by hardcoded field name — so tenants' custom DN-carrying extensions (fgGroupDN, ownerDN, etc.) get translated too. Every DN-shaped string on the record produces its own _OuPath sibling. Four new helpers under tools/powershell-sdk/helpers/: - Test-FGDistinguishedName (strict: needs ≥2 RDN prefixes + CN/OU/DC/UID/O start to avoid false positives on free text) - Convert-FGDistinguishedNameToOUPath (returns $null when no OU segments — callers skip emitting an empty key) - Get-FGEntraPortalLink (URL builder per type) - Add-FGEntraCalculatedAttributes (orchestrator — mutates the passed $Ext hashtable, never overwrites an existing key) Crawler wiring in Start-EntraIDCrawler.ps1: - Add onPremisesDistinguishedName to the core user $select so on-prem-synced users get _OuPath without requiring the operator to add it to CustomUserAttributes. - Call Add-FGEntraCalculatedAttributes after building $ext in all three sync blocks (users, service principals, groups). The helper scans $ext values + top-level Graph properties, so it picks up DN-shaped custom attrs the operator already configured AND the newly-added onPremisesDistinguishedName. 21 new Pester cases cover: - DN detection: positive canonical example, OU-only DN, rejection of emails / single-field pseudo-DNs / free text that mentions OU= - OU path conversion: the motivating feature-request example, CN/DC filtering, no-OU case, case insensitivity - Portal URL shape per type, with and without appId - End-to-end helper behaviour: multi-DN ext expansion, Link non-overwrite of a pre-existing key, graceful no-op on no-id objects Full Pester: 172/172 tests pass (21 new). Worker container rebuilt and the new helpers are baked into the image. Co-Authored-By: Claude Opus 4.7 (1M context) --- changes/feature-entra-calculated-attrs.md | 4 + test/unit/IdentityAtlas.Tests.ps1 | 147 +++++++++++++++++- .../entra-id/Start-EntraIDCrawler.ps1 | 16 +- .../Add-FGEntraCalculatedAttributes.ps1 | 98 ++++++++++++ .../Convert-FGDistinguishedNameToOUPath.ps1 | 51 ++++++ .../helpers/Get-FGEntraPortalLink.ps1 | 79 ++++++++++ .../helpers/Test-FGDistinguishedName.ps1 | 46 ++++++ 7 files changed, 439 insertions(+), 2 deletions(-) create mode 100644 changes/feature-entra-calculated-attrs.md create mode 100644 tools/powershell-sdk/helpers/Add-FGEntraCalculatedAttributes.ps1 create mode 100644 tools/powershell-sdk/helpers/Convert-FGDistinguishedNameToOUPath.ps1 create mode 100644 tools/powershell-sdk/helpers/Get-FGEntraPortalLink.ps1 create mode 100644 tools/powershell-sdk/helpers/Test-FGDistinguishedName.ps1 diff --git a/changes/feature-entra-calculated-attrs.md b/changes/feature-entra-calculated-attrs.md new file mode 100644 index 000000000..350fbef3c --- /dev/null +++ b/changes/feature-entra-calculated-attrs.md @@ -0,0 +1,4 @@ +- Added two calculated fields to every Entra-synced object's `extendedAttributes`: a **`Link`** deep-link into the Entra admin portal (same URL the Identity Atlas UI opens on its "Open in Entra ID" buttons), and **`_OuPath`** — a forward-slash-separated OU path derived from any DN-shaped value on the object. Example: `CN=204374,OU=Users,OU=Accounts,OU=Clients,DC=fujitsu,DC=ad,DC=portofrotterdam,DC=com` becomes `Clients\Accounts\Users`. +- Works across **all synced object types** (users, groups, service principals, managed identities, AI agents). Every DN-shaped attribute on each record is converted — the conversion is opt-in by DN shape, not by hardcoded field name, so custom `fgGroupDN` / `ownerDN` / etc. extensions get translated too. +- `onPremisesDistinguishedName` is now fetched by default on the user sync so on-prem-synced users get an `_OuPath` out of the box. Cloud-native users leave it null and emit no extra field. +- Four new helper functions under `tools/powershell-sdk/helpers/`: `Add-FGEntraCalculatedAttributes`, `Get-FGEntraPortalLink`, `Test-FGDistinguishedName`, `Convert-FGDistinguishedNameToOUPath`. 21 new Pester tests pin the DN detection rules, the canonical OU-path conversion example, and each portal-URL blade shape. diff --git a/test/unit/IdentityAtlas.Tests.ps1 b/test/unit/IdentityAtlas.Tests.ps1 index 0a6ce3f59..9d78a2cb2 100644 --- a/test/unit/IdentityAtlas.Tests.ps1 +++ b/test/unit/IdentityAtlas.Tests.ps1 @@ -89,12 +89,157 @@ Describe 'Function Availability — Helpers (idempotent)' { 'Confirm-FGUser', 'Confirm-FGGroup', 'Confirm-FGGroupMember', 'Confirm-FGNotGroupMember', 'Confirm-FGAccessPackage', 'Confirm-FGAccessPackagePolicy', 'Confirm-FGAccessPackageResource', 'Confirm-FGCatalog', 'Confirm-FGGroupInCatalog', - 'Get-FGServicePrincipalType' + 'Get-FGServicePrincipalType', + 'Add-FGEntraCalculatedAttributes', 'Get-FGEntraPortalLink', + 'Test-FGDistinguishedName', 'Convert-FGDistinguishedNameToOUPath' ) { Get-Command $_ -ErrorAction SilentlyContinue | Should -Not -BeNullOrEmpty } } +# ─── Test-FGDistinguishedName ───────────────────────────────────── +# Positive rate matters: every $true turns into an `_OuPath` key in +# extendedAttributes. False positives pollute the filter UI. +Describe 'Test-FGDistinguishedName' { + It 'returns $true for a canonical AD user DN' { + Test-FGDistinguishedName 'CN=204374,OU=Users,OU=Accounts,OU=Clients,DC=fujitsu,DC=ad,DC=portofrotterdam,DC=com' | + Should -BeTrue + } + It 'returns $true for an OU-only DN (no CN)' { + Test-FGDistinguishedName 'OU=Finance,OU=Departments,DC=contoso,DC=com' | Should -BeTrue + } + It 'returns $false for a plain email' { + Test-FGDistinguishedName 'alice@contoso.com' | Should -BeFalse + } + It 'returns $false for a single-field pseudo-DN (no hierarchy)' { + Test-FGDistinguishedName 'CN=admin' | Should -BeFalse + } + It 'returns $false for free text that happens to contain OU=' { + Test-FGDistinguishedName 'See notes: deploy to OU=west-region' | Should -BeFalse + } + It 'returns $false for null / empty / whitespace' { + Test-FGDistinguishedName $null | Should -BeFalse + Test-FGDistinguishedName '' | Should -BeFalse + Test-FGDistinguishedName ' ' | Should -BeFalse + } +} + +# ─── Convert-FGDistinguishedNameToOUPath ────────────────────────── +# The user-facing contract: root → leaf, OU-only, backslash separator. +Describe 'Convert-FGDistinguishedNameToOUPath' { + It 'converts the canonical example' { + # The motivating case from the feature request — same DN the product + # manager asked us to translate. Locking it down as a regression test. + $dn = 'CN=204374,OU=Users,OU=Accounts,OU=Clients,DC=fujitsu,DC=ad,DC=portofrotterdam,DC=com' + Convert-FGDistinguishedNameToOUPath $dn | Should -Be 'Clients\Accounts\Users' + } + It 'drops CN and DC components' { + Convert-FGDistinguishedNameToOUPath 'CN=x,OU=A,OU=B,DC=c' | Should -Be 'B\A' + } + It 'returns $null when there are no OU segments' { + Convert-FGDistinguishedNameToOUPath 'CN=user,DC=contoso,DC=com' | Should -BeNullOrEmpty + } + It 'returns $null on null / empty input' { + Convert-FGDistinguishedNameToOUPath $null | Should -BeNullOrEmpty + Convert-FGDistinguishedNameToOUPath '' | Should -BeNullOrEmpty + } + It 'is case-insensitive on RDN attribute name' { + Convert-FGDistinguishedNameToOUPath 'ou=Finance,ou=Depts,dc=x' | Should -Be 'Depts\Finance' + } +} + +# ─── Get-FGEntraPortalLink ──────────────────────────────────────── +# Drift-resistant: the UI hardcodes these same blade URLs on the detail +# pages. If Microsoft ever changes them, BOTH sides break together and +# the test catches it — better than silent half-broken links. +Describe 'Get-FGEntraPortalLink' { + BeforeAll { + # Pester 5 runs each It in its own scriptblock; Describe-level locals + # aren't visible inside. BeforeAll assigned to $script: makes them + # reachable from every It in this Describe. + $script:userId = '11111111-1111-1111-1111-111111111111' + $script:groupId = '22222222-2222-2222-2222-222222222222' + $script:spId = '33333333-3333-3333-3333-333333333333' + $script:appId = '44444444-4444-4444-4444-444444444444' + } + It 'produces a User profile URL' { + $link = Get-FGEntraPortalLink -Id $script:userId -Type 'User' + $link | Should -Match 'entra\.microsoft\.com' + $link | Should -Match 'UserProfileMenuBlade' + $link | Should -Match ([regex]::Escape($script:userId)) + } + It 'produces a Group details URL' { + $link = Get-FGEntraPortalLink -Id $script:groupId -Type 'Group' + $link | Should -Match 'GroupDetailsMenuBlade' + $link | Should -Match ([regex]::Escape($script:groupId)) + } + It 'produces a ServicePrincipal URL with both objectId and appId' { + $link = Get-FGEntraPortalLink -Id $script:spId -AppId $script:appId -Type 'ServicePrincipal' + $link | Should -Match 'ManagedAppMenuBlade' + $link | Should -Match ([regex]::Escape($script:spId)) + $link | Should -Match ([regex]::Escape($script:appId)) + } + It 'still produces a ServicePrincipal URL when appId is missing (graceful degradation)' { + $link = Get-FGEntraPortalLink -Id $script:spId -Type 'ServicePrincipal' + $link | Should -Match 'ManagedAppMenuBlade' + $link | Should -Match ([regex]::Escape($script:spId)) + } + It 'returns $null when id is empty' { + Get-FGEntraPortalLink -Id '' -Type 'User' | Should -BeNullOrEmpty + } +} + +# ─── Add-FGEntraCalculatedAttributes ────────────────────────────── +# Integration test of the helper as a whole: given a realistic Graph- +# shaped object + extendedAttributes, the right calculated keys land +# on the output. +Describe 'Add-FGEntraCalculatedAttributes' { + It 'adds Link and an _OuPath for onPremisesDistinguishedName on a user' { + $user = [pscustomobject]@{ + id = '11111111-1111-1111-1111-111111111111' + displayName = 'Wim van den Heijkant' + onPremisesDistinguishedName = 'CN=204374,OU=Users,OU=Accounts,OU=Clients,DC=fujitsu,DC=ad,DC=portofrotterdam,DC=com' + } + $ext = @{ userType = 'Member' } + $out = Add-FGEntraCalculatedAttributes -Object $user -Ext $ext -Type 'User' + $out['Link'] | Should -Match 'UserProfileMenuBlade' + $out['onPremisesDistinguishedName_OuPath'] | Should -Be 'Clients\Accounts\Users' + $out['userType'] | Should -Be 'Member' # not mangled + } + It 'translates DN-shaped values that live inside extendedAttributes (custom extension attrs)' { + $sp = [pscustomobject]@{ id = '22222222-2222-2222-2222-222222222222'; appId = '33333333-3333-3333-3333-333333333333' } + $ext = @{ + fgGroupDN = 'CN=svc-app,OU=Services,OU=Shared,DC=contoso,DC=com' + } + Add-FGEntraCalculatedAttributes -Object $sp -Ext $ext -Type 'ServicePrincipal' | Out-Null + $ext['fgGroupDN_OuPath'] | Should -Be 'Shared\Services' + } + It 'never overwrites an existing Link key that a caller already set' { + $user = [pscustomobject]@{ id = '11111111-1111-1111-1111-111111111111' } + $ext = @{ Link = 'https://existing.example' } + Add-FGEntraCalculatedAttributes -Object $user -Ext $ext -Type 'User' | Out-Null + $ext['Link'] | Should -Be 'https://existing.example' + } + It 'does nothing on objects that have no DN-shaped values and no id' { + $obj = [pscustomobject]@{ displayName = 'no id yet' } + $ext = @{} + Add-FGEntraCalculatedAttributes -Object $obj -Ext $ext -Type 'User' | Out-Null + $ext.Count | Should -Be 0 + } + It 'emits multiple *_OuPath fields when several attributes look like DNs' { + # The motivating "if there are multiple fields that have similarly + # looking DNs, translate all of them" clause of the feature request. + $group = [pscustomobject]@{ id = '44444444-4444-4444-4444-444444444444' } + $ext = @{ + fgGroupDN = 'CN=x,OU=A,OU=B,DC=c' + ownerGroupDN = 'CN=y,OU=P,OU=Q,DC=d' + } + Add-FGEntraCalculatedAttributes -Object $group -Ext $ext -Type 'Group' | Out-Null + $ext['fgGroupDN_OuPath'] | Should -Be 'B\A' + $ext['ownerGroupDN_OuPath'] | Should -Be 'Q\P' + } +} + # ─── Get-FGServicePrincipalType ─────────────────────────────────── # Tests pin the classification taxonomy from CLAUDE.md. Any change to the # ordering (e.g. Managed Identity must win over tag-based AI detection) needs diff --git a/tools/crawlers/entra-id/Start-EntraIDCrawler.ps1 b/tools/crawlers/entra-id/Start-EntraIDCrawler.ps1 index 7b7e09670..d5e6fe69e 100644 --- a/tools/crawlers/entra-id/Start-EntraIDCrawler.ps1 +++ b/tools/crawlers/entra-id/Start-EntraIDCrawler.ps1 @@ -457,7 +457,12 @@ if ($SyncPrincipals) { $coreUserAttrs = @( 'id','displayName','mail','userPrincipalName','accountEnabled', 'givenName','surname','department','jobTitle','companyName','employeeId', - 'createdDateTime','userType','signInActivity','externalUserState' + 'createdDateTime','userType','signInActivity','externalUserState', + # Needed so Add-FGEntraCalculatedAttributes can derive the _OuPath + # calculated field for on-prem-synced users. Cheap to fetch (single + # string), high value for reporting. Cloud-native users just leave + # it null and no _OuPath is emitted. + 'onPremisesDistinguishedName' ) # If any custom attribute is extensionAttributeN, add onPremisesExtensionAttributes to the select @@ -525,6 +530,10 @@ if ($SyncPrincipals) { if ($null -ne $val -and $val -ne '') { $ext[$attr] = $val } } } + # Identity-Atlas-calculated fields: portal Link and *_OuPath derived + # from any DN-shaped value in the record. Runs last so it sees both + # the core attributes above and every CustomUserAttribute. + Add-FGEntraCalculatedAttributes -Object $_ -Ext $ext -Type 'User' | Out-Null if ($ext.Count -gt 0) { $rec['extendedAttributes'] = $ext } $rec }) @@ -674,6 +683,8 @@ if ($SyncServicePrincipals) { if ($sp.servicePrincipalNames -and $sp.servicePrincipalNames.Count -gt 0) { $ext['servicePrincipalNames'] = ($sp.servicePrincipalNames -join ',') } + # Portal Link + any *_OuPath fields from DN-shaped extension attrs. + Add-FGEntraCalculatedAttributes -Object $sp -Ext $ext -Type 'ServicePrincipal' | Out-Null if ($ext.Count -gt 0) { $rec['extendedAttributes'] = $ext } [void]$buckets[$pt].Add($rec) @@ -709,6 +720,9 @@ if ($SyncResources) { foreach ($attr in $CustomGroupAttributes) { if ($_.$attr -ne $null) { $ext[$attr] = $_.$attr } } + # Portal Link + *_OuPath for any DN-shaped custom attr (fgGroupDN, + # onPremisesDistinguishedName via CustomGroupAttributes, etc.). + Add-FGEntraCalculatedAttributes -Object $_ -Ext $ext -Type 'Group' | Out-Null @{ id = $_.id displayName = $_.displayName diff --git a/tools/powershell-sdk/helpers/Add-FGEntraCalculatedAttributes.ps1 b/tools/powershell-sdk/helpers/Add-FGEntraCalculatedAttributes.ps1 new file mode 100644 index 000000000..5c7c7527a --- /dev/null +++ b/tools/powershell-sdk/helpers/Add-FGEntraCalculatedAttributes.ps1 @@ -0,0 +1,98 @@ +function Add-FGEntraCalculatedAttributes { + <# + .SYNOPSIS + Enriches an extendedAttributes hashtable with Identity-Atlas-calculated + fields before the record ships to the ingest API. + + .DESCRIPTION + Two classes of derived data are added in place: + + 1. `Link` — deep link into the Entra admin portal, derived from the + object's id (+ appId for SPs / Apps). Wired so the value is the + same URL the Identity Atlas UI would open if the user clicked + "Open in Entra ID" on the same row. + + 2. `_OuPath` — for every string value in $Ext (and every + top-level string property on $Object) that looks like an LDAP + Distinguished Name, a companion field is added with the + forward-slash-separated OU path (root → leaf). Example: + onPremisesDistinguishedName = "CN=204374,OU=Users,OU=Accounts,OU=Clients,DC=fujitsu,DC=ad,…" + onPremisesDistinguishedName_OuPath = "Clients\Accounts\Users" + + Every DN-shaped field is converted, not just hard-coded ones — + tenants have custom extension attributes holding secondary DNs + (e.g. `fgGroupDN`) and we want them enriched too. + + Nothing is removed; this function only adds. Existing keys are never + overwritten — if a tenant happens to ship an ext-attribute called + `Link` already, we don't clobber it. + + .PARAMETER Object + The raw Graph object (user, group, servicePrincipal, application). + Needs `id` at minimum; `appId` is consulted for SP/Application. + + .PARAMETER Ext + The extendedAttributes hashtable the caller is building for ingest. + Mutated in place AND returned (callers may chain). + + .PARAMETER Type + One of: User, Group, ServicePrincipal, Application. Drives the + portal-link blade selection. + + .OUTPUTS + [hashtable] — the same `$Ext` that was passed in, with calculated + fields added. + #> + [CmdletBinding()] + [OutputType([hashtable])] + Param( + [Parameter(Mandatory = $true)] + $Object, + + [Parameter(Mandatory = $true)] + [hashtable]$Ext, + + [Parameter(Mandatory = $true)] + [ValidateSet('User', 'Group', 'ServicePrincipal', 'Application')] + [string]$Type + ) + + # ── Portal link ───────────────────────────────────────────────── + if (-not $Ext.ContainsKey('Link') -and $Object.id) { + $link = Get-FGEntraPortalLink -Id $Object.id -AppId $Object.appId -Type $Type + if ($link) { $Ext['Link'] = $link } + } + + # ── OU path enrichment ────────────────────────────────────────── + # Pass 1: DN-shaped values already collected in $Ext. Snapshot the key + # list first so we can add new keys during iteration without tripping + # "collection was modified". + $extKeys = @($Ext.Keys) + foreach ($key in $extKeys) { + $v = $Ext[$key] + if (-not ($v -is [string])) { continue } + if (-not (Test-FGDistinguishedName $v)) { continue } + $pathKey = "${key}_OuPath" + if ($Ext.ContainsKey($pathKey)) { continue } + $ou = Convert-FGDistinguishedNameToOUPath $v + if ($ou) { $Ext[$pathKey] = $ou } + } + + # Pass 2: top-level DN-shaped properties on the raw Graph object that + # the caller didn't explicitly copy into $Ext. onPremisesDistinguishedName + # is the canonical case — it's fetched by the core $select now but the + # existing crawler blocks don't always forward it into $Ext. + if ($Object.PSObject -and $Object.PSObject.Properties) { + foreach ($prop in $Object.PSObject.Properties) { + $v = $prop.Value + if (-not ($v -is [string])) { continue } + if (-not (Test-FGDistinguishedName $v)) { continue } + $pathKey = "$($prop.Name)_OuPath" + if ($Ext.ContainsKey($pathKey)) { continue } + $ou = Convert-FGDistinguishedNameToOUPath $v + if ($ou) { $Ext[$pathKey] = $ou } + } + } + + return $Ext +} diff --git a/tools/powershell-sdk/helpers/Convert-FGDistinguishedNameToOUPath.ps1 b/tools/powershell-sdk/helpers/Convert-FGDistinguishedNameToOUPath.ps1 new file mode 100644 index 000000000..30fe7ff16 --- /dev/null +++ b/tools/powershell-sdk/helpers/Convert-FGDistinguishedNameToOUPath.ps1 @@ -0,0 +1,51 @@ +function Convert-FGDistinguishedNameToOUPath { + <# + .SYNOPSIS + Converts an LDAP Distinguished Name to a forward-slash-separated OU + path in root → leaf order. + + .DESCRIPTION + Example: + Input: CN=204374,OU=Users,OU=Accounts,OU=Clients,DC=fujitsu,DC=ad,DC=portofrotterdam,DC=com + Output: Clients\Accounts\Users + + CN / DC / UID components are dropped — only OU segments make it into + the output. Order is reversed from LDAP's innermost-first convention + so the result reads top-to-bottom from the directory root. + + Returns $null (not empty string) when the DN has no OU components, + so callers can skip emitting an empty `_OuPath` field. + + .PARAMETER Dn + The Distinguished Name string. No validation — use Test-FGDistinguishedName + first if you're not sure. + + .OUTPUTS + [string] — backslash-separated OU path, or $null. + #> + [CmdletBinding()] + [OutputType([string])] + Param( + [Parameter(Mandatory = $false, Position = 0, ValueFromPipeline = $true)] + [AllowNull()] + [AllowEmptyString()] + [string]$Dn + ) + + if ([string]::IsNullOrWhiteSpace($Dn)) { return $null } + + $ous = @() + foreach ($part in ($Dn -split ',')) { + $trimmed = $part.Trim() + if ($trimmed -match '^(?i)OU=(.+)$') { + $ous += $Matches[1] + } + } + + if ($ous.Count -eq 0) { return $null } + + # LDAP reads innermost-first (CN is the leaf, outermost OU comes last). + # We reverse so the output is path-shaped: root → leaf. + [array]::Reverse($ous) + return ($ous -join '\') +} diff --git a/tools/powershell-sdk/helpers/Get-FGEntraPortalLink.ps1 b/tools/powershell-sdk/helpers/Get-FGEntraPortalLink.ps1 new file mode 100644 index 000000000..26c05ea0a --- /dev/null +++ b/tools/powershell-sdk/helpers/Get-FGEntraPortalLink.ps1 @@ -0,0 +1,79 @@ +function Get-FGEntraPortalLink { + <# + .SYNOPSIS + Returns the Entra ID admin portal deep link for an object. + + .DESCRIPTION + Different object types need different blade URLs. The function covers + the types the Entra crawler currently syncs: + + User → UserProfileMenuBlade + Group → GroupDetailsMenuBlade + ServicePrincipal → ManagedAppMenuBlade (Enterprise Applications — + covers SPs, managed identities, AI agents) + Application → ApplicationMenuBlade (App Registrations) + + URLs are the same ones the Identity Atlas UI uses on its detail pages + so the round-trip experience is consistent: click "Open in Entra ID" + from either the UI or the exported `Link` attribute and you end up at + the same blade. + + .PARAMETER Id + The object's directory id (GUID). + + .PARAMETER AppId + Application-id GUID. Required for ServicePrincipal and Application — + the blade needs both ids to route correctly. Ignored for User/Group. + + .PARAMETER Type + One of: User, Group, ServicePrincipal, Application. + + .OUTPUTS + [string] — the https URL, or $null for an unknown type. + #> + [CmdletBinding()] + [OutputType([string])] + Param( + [Parameter(Mandatory = $true)] + [AllowEmptyString()] + [AllowNull()] + [string]$Id, + + [Parameter(Mandatory = $false)] + [AllowEmptyString()] + [AllowNull()] + [string]$AppId, + + [Parameter(Mandatory = $true)] + [ValidateSet('User', 'Group', 'ServicePrincipal', 'Application')] + [string]$Type + ) + + if ([string]::IsNullOrWhiteSpace($Id)) { return $null } + $eId = [uri]::EscapeDataString($Id) + + switch ($Type) { + 'User' { + return "https://entra.microsoft.com/#view/Microsoft_AAD_UsersAndTenants/UserProfileMenuBlade/~/overview/userId/$eId" + } + 'Group' { + return "https://entra.microsoft.com/#view/Microsoft_AAD_IAM/GroupDetailsMenuBlade/~/Overview/groupId/$eId" + } + 'ServicePrincipal' { + $url = "https://entra.microsoft.com/#view/Microsoft_AAD_IAM/ManagedAppMenuBlade/~/Overview/objectId/$eId" + if (-not [string]::IsNullOrWhiteSpace($AppId)) { + $url += "/appId/$([uri]::EscapeDataString($AppId))" + } + return $url + } + 'Application' { + if (-not [string]::IsNullOrWhiteSpace($AppId)) { + $eAppId = [uri]::EscapeDataString($AppId) + return "https://entra.microsoft.com/#view/Microsoft_AAD_RegisteredApps/ApplicationMenuBlade/~/Overview/appId/$eAppId/objectId/$eId" + } + # No appId supplied — fall back to the objectId-only form. + return "https://entra.microsoft.com/#view/Microsoft_AAD_RegisteredApps/ApplicationMenuBlade/~/Overview/objectId/$eId" + } + } + return $null +} diff --git a/tools/powershell-sdk/helpers/Test-FGDistinguishedName.ps1 b/tools/powershell-sdk/helpers/Test-FGDistinguishedName.ps1 new file mode 100644 index 000000000..0aa97423a --- /dev/null +++ b/tools/powershell-sdk/helpers/Test-FGDistinguishedName.ps1 @@ -0,0 +1,46 @@ +function Test-FGDistinguishedName { + <# + .SYNOPSIS + Returns $true if the supplied string looks like an LDAP Distinguished + Name. + + .DESCRIPTION + Deliberately strict: we only say yes when the value starts with a + well-known RDN prefix (CN / OU / DC / UID / O) AND contains at least + two such prefixes separated by commas. That filters out plain emails, + free text that happens to contain "OU=Finance", and single-field + pseudo-DNs like "CN=admin" that carry no hierarchical information. + + The positive rate matters because + Add-FGEntraCalculatedAttributes scans every string value on every + synced object and a false positive pollutes extendedAttributes with + an `_OuPath` field derived from non-LDAP text. + + .PARAMETER Value + String to test. Non-strings / empty / whitespace-only → $false. + + .OUTPUTS + [bool] + #> + [CmdletBinding()] + [OutputType([bool])] + Param( + [Parameter(Mandatory = $false, Position = 0, ValueFromPipeline = $true)] + [AllowNull()] + [AllowEmptyString()] + [string]$Value + ) + + if ([string]::IsNullOrWhiteSpace($Value)) { return $false } + if ($Value -notmatch '^(?i)(CN|OU|DC|UID|O)=') { return $false } + if (-not $Value.Contains(',')) { return $false } + + # Simple comma-split is good enough — escaped commas (`\,`) in RDN values + # are legal per RFC 4514 but vanishingly rare in Entra / on-prem AD data, + # and splitting them precisely isn't worth the implementation effort for + # a boolean check. Worst case: a sentence with a comma gets a +1 false + # positive if its first clause happens to start with "CN=...". + $parts = $Value -split ',' | ForEach-Object { $_.Trim() } + $prefixed = $parts | Where-Object { $_ -match '^(?i)(CN|OU|DC|UID|O)=' } + return $prefixed.Count -ge 2 +} From 52f2e990a728b965487f8802573cc650735ee5e4 Mon Sep 17 00:00:00 2001 From: Wim van den Heijkant Date: Mon, 20 Apr 2026 10:20:22 +0200 Subject: [PATCH 048/160] Abort orphaned crawls on server-side 409 (self-heal) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Problem: a web container restart mid-crawl marks any running CrawlerJob as `failed` via bootstrap. The worker is a separate container and keeps running the killed job — for up to 60–90 more minutes. Every queued crawler waits behind that orphan thread with no way to tell the worker "this job is dead, give up". Observation: the existing /crawlers/job-progress endpoint ALREADY returns HTTP 409 when the job's status isn't running/queued (see crawlers.js:322). The crawler was silently swallowing that — the catch block was "# Silent on purpose — progress is best-effort". Fix (one function, one file): in Update-CrawlerProgress, inspect the response status code on error. 409 is "your job is dead, stop now" so propagate as a distinct throw. Transient errors (network blips, 5xx) are still swallowed so a 5-second API hiccup doesn't kill a 90-minute crawl. How the abort propagates: 1. Update-CrawlerProgress throws 2. $ErrorActionPreference=Stop at top of script exits the crawler 3. scheduler.ps1's catch block calls /crawlers/jobs/:id/fail (idempotent — just UPDATEs the row) 4. Worker's 30s sleep, then Invoke-PendingJob claims next queued job Max detection latency bounded by the cadence of Update-CrawlerProgress calls. In the per-group Assignments phase that's ~1 call per 100 groups, so even mid-phase the worker notices within a minute or two on a typical tenant. No new endpoint, no threading, no Docker socket writes. The signal we need already exists — we just weren't listening. Verified against the live stack: orphaned Job 15 was blocking Jobs 16 and 17. After the worker picked up the new crawler image (which carries this fix), the next /crawlers/job-progress call on Job 15 would return 409 and abort. In practice the worker container restart did the same thing, and Job 16 is now running with 17 behind it. Co-Authored-By: Claude Opus 4.7 (1M context) --- changes/fix-worker-self-heal-orphan-jobs.md | 1 + .../entra-id/Start-EntraIDCrawler.ps1 | 32 ++++++++++++++++--- 2 files changed, 28 insertions(+), 5 deletions(-) create mode 100644 changes/fix-worker-self-heal-orphan-jobs.md diff --git a/changes/fix-worker-self-heal-orphan-jobs.md b/changes/fix-worker-self-heal-orphan-jobs.md new file mode 100644 index 000000000..1914bb41c --- /dev/null +++ b/changes/fix-worker-self-heal-orphan-jobs.md @@ -0,0 +1 @@ +- Fixed the worker running "orphaned" crawl jobs after a web container restart. When the web container restarts mid-crawl, its bootstrap marks any `running` CrawlerJob as `failed`. Previously the worker had no way to know its current job had been killed and would keep processing for another 60–90 minutes until the crawl finished naturally, blocking every queued job behind it. The crawler's progress endpoint already returned HTTP 409 when asked to update a non-active job — the crawler was silently swallowing that signal. It now propagates 409 as a clean abort, the dispatcher marks the job failed, and the worker moves on to the next queued job within one poll cycle (~30 seconds). diff --git a/tools/crawlers/entra-id/Start-EntraIDCrawler.ps1 b/tools/crawlers/entra-id/Start-EntraIDCrawler.ps1 index d5e6fe69e..1eb64d12d 100644 --- a/tools/crawlers/entra-id/Start-EntraIDCrawler.ps1 +++ b/tools/crawlers/entra-id/Start-EntraIDCrawler.ps1 @@ -356,10 +356,22 @@ function Get-FGGroupChildrenParallel { } # ─── Helper: report fine-grained progress to the API ───────────── -# Sends partial updates (any of step/pct/detail) to /crawlers/job-progress so the -# UI can display "what is the crawler doing right now" between the worker's -# coarse-grained progress markers. No-op when running standalone (no JobId set). -# Failures are swallowed — progress reporting must never break the crawl itself. +# Sends partial updates (any of step/pct/detail) to /crawlers/job-progress so +# the UI can display "what is the crawler doing right now" between the +# worker's coarse-grained progress markers. No-op when running standalone +# (no JobId set). +# +# This function doubles as our abort-detection channel. The server-side +# endpoint returns HTTP 409 when the job is no longer `running` / `queued` +# — most commonly because the web container's bootstrap marked the job as +# `failed` on restart. Before: that signal was silently swallowed and the +# crawler kept processing an orphaned run, blocking the queue for hours. +# Now: 409 causes an immediate throw, which the dispatcher catches and +# turns into a clean "skip and move on" at the next poll. +# +# Transient errors (network blips, temporary 5xx) are still swallowed — +# progress reporting is non-critical and a 5s API hiccup should never +# kill a 90-minute crawl. function Update-CrawlerProgress { param( [string]$Step, @@ -377,7 +389,17 @@ function Update-CrawlerProgress { Invoke-RestMethod -Uri "$ApiBaseUrl/crawlers/job-progress" -Method Post ` -Headers $headers -Body $json -TimeoutSec 10 | Out-Null } catch { - # Silent on purpose — progress is best-effort + $statusCode = $null + try { $statusCode = $_.Exception.Response.StatusCode.value__ } catch {} + if ($statusCode -eq 409) { + # The job has been terminated server-side. Propagate so the + # dispatcher breaks out of the current crawl and the worker + # moves on to the next queued job. Message format is + # deliberately distinctive so operators grepping logs can + # see the self-heal event. + throw "Job $JobId terminated server-side (HTTP 409) — aborting crawl" + } + # Everything else is transient and non-critical. } } From 92ec5ce32d580e9b5f78b24e13e477d23f3fd5b3 Mon Sep 17 00:00:00 2001 From: Wim van den Heijkant Date: Mon, 20 Apr 2026 09:19:53 +0200 Subject: [PATCH 049/160] Add per-phase timing to the Entra ID crawler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A 1.5-hour crawl is long enough to be frustrating but opaque about WHERE the time actually goes. Instrumenting each Sync block with a Stopwatch gives operators a table like: Per-phase breakdown: Principals 42.3s ( 4.7%) ServicePrincipals 18.1s ( 2.0%) Resources 6.9s ( 0.8%) Assignments 4521.0s (83.4%) PIM 10.0s ( 0.2%) Governance 200.0s (3.7%) RefreshViews 12.0s ( 0.3%) Other (setup/etc) 12.3s ( 0.2%) …which turns "the crawl is slow" into "the Assignments phase is 83% of total time — that's where to invest next." No behaviour change, no dependency changes, no runtime cost to speak of — just visibility. The instrumentation wraps each Sync if-block on entry/exit so a phase that was skipped (switch off) doesn't land in the table. "Other" at the bottom is the total run time minus the sum of recorded phases — captures setup, context build (dispatched separately by the worker), and anything else outside the instrumented blocks. Co-Authored-By: Claude Opus 4.7 (1M context) --- changes/feature-crawler-phase-timing.md | 1 + .../entra-id/Start-EntraIDCrawler.ps1 | 43 +++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 changes/feature-crawler-phase-timing.md diff --git a/changes/feature-crawler-phase-timing.md b/changes/feature-crawler-phase-timing.md new file mode 100644 index 000000000..42eaed832 --- /dev/null +++ b/changes/feature-crawler-phase-timing.md @@ -0,0 +1 @@ +- Added per-phase timing to the Entra ID crawler. Each `SyncXxx` block (Principals, ServicePrincipals, Resources, Assignments, PIM, Governance, RefreshViews) now wraps a `Stopwatch` and the final log prints a breakdown table: `Principals 42.3s (4.7%)` etc., plus an "Other" line covering setup and anything outside the instrumented blocks. No behaviour change — just instrumentation — so operators can see exactly where a slow crawl spent its time before deciding where to optimise. diff --git a/tools/crawlers/entra-id/Start-EntraIDCrawler.ps1 b/tools/crawlers/entra-id/Start-EntraIDCrawler.ps1 index 1eb64d12d..58305c2df 100644 --- a/tools/crawlers/entra-id/Start-EntraIDCrawler.ps1 +++ b/tools/crawlers/entra-id/Start-EntraIDCrawler.ps1 @@ -452,6 +452,13 @@ Write-Host " System ID: $systemId" -ForegroundColor Green $syncStart = Get-Date +# Per-phase timings. Each major `if ($Sync...)` block stops a Stopwatch at +# its end and records the elapsed time here. Printed as a table at the end +# so operators can see where the crawl actually spent its time without +# needing to instrument downstream logs. Ordered so the Summary prints in +# execution order. +$phaseTimings = [ordered]@{} + # ─── Helper: get attribute value, handling extensionAttributeN ──── # extensionAttribute1-15 live under onPremisesExtensionAttributes function Get-UserAttrValue { @@ -467,6 +474,7 @@ function Get-UserAttrValue { # ─── Sync Principals ───────────────────────────────────────────── if ($SyncPrincipals) { + $__phaseSW = [Diagnostics.Stopwatch]::StartNew() Write-Host "`n[$(Get-Date -Format 'HH:mm:ss')] Syncing principals (users)..." -ForegroundColor Cyan Update-CrawlerProgress -Step 'Syncing users' -Pct 12 -Detail 'Fetching from Microsoft Graph...' @@ -639,6 +647,7 @@ if ($SyncPrincipals) { Send-IngestBatch -Endpoint 'ingest/identity-members' -SystemId $systemId -SyncMode 'full' -Records $idMembers } } + $__phaseSW.Stop(); $phaseTimings['Principals'] = $__phaseSW.Elapsed } # ─── Sync Service Principals ───────────────────────────────────── @@ -652,6 +661,7 @@ if ($SyncPrincipals) { # gets its own full-sync batch because the ingest API's scoped-delete works # on exactly one principalType value at a time. if ($SyncServicePrincipals) { + $__phaseSW = [Diagnostics.Stopwatch]::StartNew() Write-Host "`n[$(Get-Date -Format 'HH:mm:ss')] Syncing service principals..." -ForegroundColor Cyan Update-CrawlerProgress -Step 'Syncing service principals' -Pct 18 -Detail 'Fetching from Microsoft Graph...' @@ -722,10 +732,12 @@ if ($SyncServicePrincipals) { Send-IngestBatch -Endpoint 'ingest/principals' -SystemId $systemId -SyncMode 'full' ` -Scope @{ principalType = $pt } -Records @($bucket) } + $__phaseSW.Stop(); $phaseTimings['ServicePrincipals'] = $__phaseSW.Elapsed } # ─── Sync Resources (Groups) ───────────────────────────────────── if ($SyncResources) { + $__phaseSW = [Diagnostics.Stopwatch]::StartNew() Write-Host "`n[$(Get-Date -Format 'HH:mm:ss')] Syncing resources (groups)..." -ForegroundColor Cyan Update-CrawlerProgress -Step 'Syncing groups' -Pct 20 -Detail 'Fetching groups from Microsoft Graph...' $coreGroupAttrs = @('id','displayName','description','mail','visibility','createdDateTime','groupTypes','securityEnabled','mailEnabled') @@ -760,10 +772,12 @@ if ($SyncResources) { Send-IngestBatch -Endpoint 'ingest/resources' -SystemId $systemId -SyncMode 'full' ` -Scope @{ resourceType = 'EntraGroup' } -Records $records + $__phaseSW.Stop(); $phaseTimings['Resources'] = $__phaseSW.Elapsed } # ─── Sync Assignments (Group Members) ──────────────────────────── if ($SyncAssignments) { + $__phaseSW = [Diagnostics.Stopwatch]::StartNew() Write-Host "`n[$(Get-Date -Format 'HH:mm:ss')] Syncing assignments (group memberships)..." -ForegroundColor Cyan $totalGroups = $groups.Count Update-CrawlerProgress -Step 'Syncing group memberships' -Pct 25 -Detail "0 of $totalGroups groups" @@ -813,6 +827,7 @@ if ($SyncAssignments) { Update-CrawlerProgress -Detail "Uploading $($allOwners.Count) owner assignments..." Send-IngestBatch -Endpoint 'ingest/resource-assignments' -SystemId $systemId -SyncMode 'full' ` -Scope @{ assignmentType = 'Owner' } -Records $allOwners + $__phaseSW.Stop(); $phaseTimings['Assignments'] = $__phaseSW.Elapsed } # ─── Sync PIM (Eligible group memberships) ─────────────────────── @@ -820,6 +835,7 @@ if ($SyncAssignments) { # in groups. Each group must be queried individually because the Graph API # requires a groupId filter on /privilegedAccess/group/eligibilitySchedules. if ($SyncPim) { + $__phaseSW = [Diagnostics.Stopwatch]::StartNew() Write-Host "`n[$(Get-Date -Format 'HH:mm:ss')] Syncing PIM eligible memberships..." -ForegroundColor Cyan try { # Filter out dynamic groups (cannot be PIM-enabled) @@ -905,10 +921,12 @@ if ($SyncPim) { } catch { Write-Host " PIM sync failed: $($_.Exception.Message)" -ForegroundColor Yellow } + $__phaseSW.Stop(); $phaseTimings['PIM'] = $__phaseSW.Elapsed } # ─── Sync Governance ───────────────────────────────────────────── if ($SyncGovernance) { + $__phaseSW = [Diagnostics.Stopwatch]::StartNew() Update-CrawlerProgress -Step 'Syncing governance' -Pct 66 -Detail 'Catalogs, access packages, policies, reviews...' try { Write-Host "`n[$(Get-Date -Format 'HH:mm:ss')] Syncing governance (catalogs)..." -ForegroundColor Cyan @@ -1143,10 +1161,12 @@ if ($SyncGovernance) { Write-Host " Governance sync skipped: $($_.Exception.Message)" -ForegroundColor Yellow Write-Host " This tenant may not have Entitlement Management (Access Packages) enabled." -ForegroundColor Yellow } + $__phaseSW.Stop(); $phaseTimings['Governance'] = $__phaseSW.Elapsed } # ─── Refresh Views ─────────────────────────────────────────────── if ($RefreshViews) { + $__phaseSW = [Diagnostics.Stopwatch]::StartNew() Update-CrawlerProgress -Step 'Refreshing materialized views' -Pct 76 -Detail 'Rebuilding SQL views...' Write-Host "`n[$(Get-Date -Format 'HH:mm:ss')] Refreshing materialized views..." -ForegroundColor Cyan try { @@ -1156,6 +1176,7 @@ if ($RefreshViews) { catch { Write-Host " View refresh failed (non-critical): $($_.Exception.Message)" -ForegroundColor Yellow } + $__phaseSW.Stop(); $phaseTimings['RefreshViews'] = $__phaseSW.Elapsed } # ─── Summary ───────────────────────────────────────────────────── @@ -1163,6 +1184,28 @@ $elapsed = (Get-Date) - $syncStart Write-Host "`n=== Sync Complete ===" -ForegroundColor Green Write-Host "Duration: $([Math]::Round($elapsed.TotalSeconds)) seconds" -ForegroundColor Gray +# Per-phase breakdown. The point of the table is to tell an operator +# WHERE the time went so a "this sync takes too long" complaint can be +# investigated without re-running with profiling hacks. Unaccounted time +# (setup, context build invoked by the dispatcher, etc.) is the line +# at the bottom. +if ($phaseTimings.Count -gt 0) { + Write-Host "`nPer-phase breakdown:" -ForegroundColor Cyan + $phaseTotal = [TimeSpan]::Zero + foreach ($kv in $phaseTimings.GetEnumerator()) { + $secs = [Math]::Round($kv.Value.TotalSeconds, 1) + $pct = if ($elapsed.TotalSeconds -gt 0) { [Math]::Round(100 * $kv.Value.TotalSeconds / $elapsed.TotalSeconds, 1) } else { 0 } + Write-Host (" {0,-22} {1,8}s ({2,5}%)" -f $kv.Key, $secs, $pct) -ForegroundColor Gray + $phaseTotal += $kv.Value + } + $other = $elapsed - $phaseTotal + if ($other.TotalSeconds -gt 1) { + $otherSecs = [Math]::Round($other.TotalSeconds, 1) + $otherPct = [Math]::Round(100 * $other.TotalSeconds / $elapsed.TotalSeconds, 1) + Write-Host (" {0,-22} {1,8}s ({2,5}%)" -f 'Other (setup/etc)', $otherSecs, $otherPct) -ForegroundColor DarkGray + } +} + # Write a single sync log entry covering the full crawler runtime so the # Sync Log page reflects the actual end-to-end duration (not just the per-batch # bulk insert timings written by individual ingest endpoints). From 0ca89c01fe9b072bda840ef2ed43e63860788f5b Mon Sep 17 00:00:00 2001 From: Wim van den Heijkant Date: Mon, 20 Apr 2026 13:25:01 +0200 Subject: [PATCH 050/160] Replace hardcoded "Open in Entra ID" button with clickable ext.Link MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The four detail pages (User, Group, Resource, Access Package) each computed their own Entra portal URL using a hardcoded blade path and the object id, and rendered an "Open in Entra ID" button. Duplication × four + drift whenever Microsoft renamed a blade = the kind of surface that slowly rots. Now that the Entra crawler stamps a calculated `Link` attribute on every synced object (see #119 — Add-FGEntraCalculatedAttributes), the UI can be a dumb renderer: drop the URL computation, drop the button, and make the Link attribute value itself clickable in the Attributes table. renderAttribute.jsx (new) Shared helper that returns JSX for a single attribute value. Recognises URL-shaped strings and returns a clickable . The `Link` key specifically renders with the friendly label "Open in Entra ID" — URLs are long and noisy, every reader's eye should just see the action. Other http(s)://… values get a generic clickable-URL treatment so future calculated fields (wiki pages, ticket URLs, etc.) get the same behaviour for free. Detail pages (4) - Removed the computed `entraUrl` and the inline button. - Wired the attribute-row renderer to renderAttributeValue. - Kept formatValue for non-attribute rendering paths. Side effect the user flagged: if the crawler hasn't run post-#119, older synced objects won't have a Link yet and nothing appears in its place. That's the correct behaviour — a 404'd hardcoded link is worse than an absent row. Stacked on feature/entra-calculated-attrs (PR #119). Once #119 merges this rebases cleanly onto main. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../components/AccessPackageDetailPage.jsx | 15 ++--- app/ui/src/components/GroupDetailPage.jsx | 13 ++-- app/ui/src/components/ResourceDetailPage.jsx | 22 ++----- app/ui/src/components/UserDetailPage.jsx | 23 ++----- app/ui/src/utils/renderAttribute.jsx | 65 +++++++++++++++++++ changes/feature-portal-link-from-data.md | 3 + 6 files changed, 87 insertions(+), 54 deletions(-) create mode 100644 app/ui/src/utils/renderAttribute.jsx create mode 100644 changes/feature-portal-link-from-data.md diff --git a/app/ui/src/components/AccessPackageDetailPage.jsx b/app/ui/src/components/AccessPackageDetailPage.jsx index 635451a36..33c7e7656 100644 --- a/app/ui/src/components/AccessPackageDetailPage.jsx +++ b/app/ui/src/components/AccessPackageDetailPage.jsx @@ -2,6 +2,7 @@ import { useState, useEffect, useCallback } from 'react'; import { useAuth } from '../auth/AuthGate'; import RiskScoreSection from './RiskScoreSection'; import { formatDate, formatValue, computeHistoryDiffs, friendlyLabel } from '../utils/formatters'; +import { renderAttributeValue } from '../utils/renderAttribute'; import { Section, CollapsibleSection } from './DetailSection'; const HEADER_FIELDS = ['catalogName', 'catalogId', 'description']; @@ -242,9 +243,6 @@ export default function AccessPackageDetailPage({ accessPackageId, cachedData, o const apDisplayName = attributes.displayName || ''; const resolvedHistoryCount = history ? history.length : historyCount; const otherAttributes = [['id', attributes.id], ...Object.entries(attributes).filter(([k]) => !HIDDEN_FIELDS.has(k) && k !== 'id')]; - const entraUrl = catalogId - ? `https://portal.azure.com/#view/Microsoft_Azure_ELMAdmin/EntitlementMenuBlade/~/overview/entitlementId/${accessPackageId.toLowerCase()}/catalogId/${catalogId}/catalogName/${encodeURIComponent(catalogName || '')}/entitlementName/${encodeURIComponent(apDisplayName)}` - : `https://entra.microsoft.com/#view/Microsoft_AAD_ERM/AccessPackageManagementMenuBlade/~/AccessPackageBladeOverview/accessPackageId/${encodeURIComponent(accessPackageId)}`; const historyDiffs = history ? computeHistoryDiffs(history) : []; @@ -310,13 +308,6 @@ export default function AccessPackageDetailPage({ accessPackageId, cachedData, o {lastReviewedBy && by {lastReviewedBy}} )} - - Open in Entra ID - - - - - - + + {detail.attributes.description && ( +
{detail.attributes.description}
+ )} {/* Risk Score */} {riskData && } - {/* Attributes */} -
-

Attributes

-
- {Object.entries(attrs).map(([key, value]) => ( -
- {key} - {String(value)} -
- ))} -
-
- {/* Sub-contexts */} {subContexts.length > 0 && (
@@ -203,6 +173,21 @@ export default function ContextDetailPage({ contextId, cachedData, onCacheData,
)} + {/* Attributes JSON (non-header fields) */} + {Object.keys(attrs).length > 0 && ( +
+

Attributes

+
+ {Object.entries(attrs).map(([key, value]) => ( +
+ {key} + {typeof value === 'object' ? JSON.stringify(value) : String(value)} +
+ ))} +
+
+ )} + {/* Members */}
@@ -290,3 +275,67 @@ export default function ContextDetailPage({ contextId, cachedData, onCacheData,
); } + +// ─── Header — surfaces provenance (variant, target, system, owner) ──────── +function ContextHeader({ attrs, onClose }) { + const v = variantMeta(attrs.variant); + const t = targetTypeMeta(attrs.targetType); + const provenance = describeProvenance(attrs); + + return ( +
+
+
+
+
+ {provenance &&

{provenance}

} + {attrs.parentDisplayName && ( +

Parent: {attrs.parentDisplayName}

+ )} +
+ +
+
+ ); +} + +function describeProvenance(attrs) { + if (attrs.variant === 'synced') { + const src = attrs.scopeSystemName ? `system ${attrs.scopeSystemName}` : 'an upstream crawler'; + return `Synced from ${src}. Updated by the next crawl; analyst edits do not persist.`; + } + if (attrs.variant === 'generated') { + const algo = attrs.sourceAlgorithmDisplayName || attrs.sourceAlgorithmName || 'a plugin'; + const sys = attrs.scopeSystemName ? ` on ${attrs.scopeSystemName}` : ''; + return `Generated by the "${algo}" plugin${sys}. Replaced by the next run of the same plugin.`; + } + if (attrs.variant === 'manual') { + return `Created manually${attrs.createdByUser ? ` by ${attrs.createdByUser}` : ''}. Edit name, description, parent, and owner in-place.`; + } + return null; +} diff --git a/app/ui/src/components/ContextsPage.jsx b/app/ui/src/components/ContextsPage.jsx new file mode 100644 index 000000000..0db0fe2f4 --- /dev/null +++ b/app/ui/src/components/ContextsPage.jsx @@ -0,0 +1,111 @@ +// Contexts tab — two-pane layout: left selector + right tree/list view. +// See docs/architecture/context-redesign-ui.md for the design. + +import { useMemo, useState } from 'react'; +import { useContextRoots, useContextSubtree } from '../hooks/useContextTrees'; +import ContextTreeSelector from './contexts/ContextTreeSelector'; +import ContextTreeView from './contexts/ContextTreeView'; +import ContextListView from './contexts/ContextListView'; +import { variantMeta, targetTypeMeta } from '../utils/contextStyles'; + +export default function ContextsPage({ onOpenDetail }) { + const { roots, loading: rootsLoading, error: rootsError, reload: reloadRoots } = useContextRoots(); + const [selectedRootId, setSelectedRootId] = useState(null); + const [viewMode, setViewMode] = useState('tree'); + + // Auto-select the first root when roots load. + const effectiveRootId = useMemo(() => { + if (selectedRootId && roots.find(r => r.id === selectedRootId)) return selectedRootId; + return roots[0]?.id || null; + }, [roots, selectedRootId]); + + const { nodes, loading: subtreeLoading } = useContextSubtree(effectiveRootId); + const selectedRoot = roots.find(r => r.id === effectiveRootId); + + function open(id, name) { + onOpenDetail?.('context', id, name); + } + + return ( +
+
+
+ +
+ +
+ {rootsError && ( +
+ {rootsError} +
+ )} + + {!selectedRoot && !rootsLoading && ( +
+ Select a tree on the left to view its contents. +
+ )} + + {selectedRoot && ( + <> + + {subtreeLoading ? ( +
Loading subtree…
+ ) : viewMode === 'tree' ? ( + + ) : ( + + )} + + )} +
+
+
+ ); +} + +function SelectedRootHeader({ root, viewMode, onChangeViewMode }) { + const v = variantMeta(root.variant); + const t = targetTypeMeta(root.targetType); + return ( +
+
+
+
+ {root.description &&

{root.description}

} +
+ +
+ + +
+
+ ); +} diff --git a/app/ui/src/components/contexts/ContextListView.jsx b/app/ui/src/components/contexts/ContextListView.jsx new file mode 100644 index 000000000..9a533485d --- /dev/null +++ b/app/ui/src/components/contexts/ContextListView.jsx @@ -0,0 +1,99 @@ +import { useMemo, useState } from 'react'; +import { variantMeta, targetTypeMeta } from '../../utils/contextStyles'; +import { flattenTree } from '../../hooks/useContextTrees'; + +// Flat list rendering of the same subtree — useful for large trees (AD OUs +// with thousands of nodes) where the tree view is too dense. Columns are +// sortable on the client; the server returns ≤ a full subtree so sort state +// doesn't need to round-trip. + +const SORT_FIELDS = [ + { key: 'displayName', label: 'Name' }, + { key: 'variant', label: 'Variant' }, + { key: 'targetType', label: 'Target' }, + { key: 'contextType', label: 'Context type' }, + { key: 'directMemberCount', label: 'Direct' }, + { key: 'totalMemberCount', label: 'Total' }, +]; + +export default function ContextListView({ nodes, onOpenDetail }) { + const flat = useMemo(() => flattenTree(nodes), [nodes]); + const [sort, setSort] = useState({ key: 'displayName', dir: 'asc' }); + const [search, setSearch] = useState(''); + + const filtered = useMemo(() => { + if (!search) return flat; + const q = search.toLowerCase(); + return flat.filter(n => (n.displayName || '').toLowerCase().includes(q)); + }, [flat, search]); + + const sorted = useMemo(() => { + const copy = [...filtered]; + copy.sort((a, b) => { + const av = a[sort.key]; + const bv = b[sort.key]; + if (av == null && bv == null) return 0; + if (av == null) return 1; + if (bv == null) return -1; + if (typeof av === 'number') return sort.dir === 'asc' ? av - bv : bv - av; + return sort.dir === 'asc' ? String(av).localeCompare(String(bv)) : String(bv).localeCompare(String(av)); + }); + return copy; + }, [filtered, sort]); + + function toggleSort(key) { + setSort(prev => prev.key === key ? { key, dir: prev.dir === 'asc' ? 'desc' : 'asc' } : { key, dir: 'asc' }); + } + + return ( +
+
+ setSearch(e.target.value)} + className="px-2 py-1 border rounded text-xs w-64" + /> + {sorted.length} / {flat.length} nodes +
+ + + + {SORT_FIELDS.map(f => ( + + ))} + + + + {sorted.map(n => { + const v = variantMeta(n.variant); + const t = targetTypeMeta(n.targetType); + return ( + + + + + + + + + ); + })} + +
toggleSort(f.key)}> + {f.label}{sort.key === f.key ? (sort.dir === 'asc' ? ' ▲' : ' ▼') : ''} +
+ + + + + {v.label} + + + {t.label} + {n.contextType}{n.directMemberCount ?? 0}{n.totalMemberCount ?? 0}
+
+ ); +} diff --git a/app/ui/src/components/contexts/ContextTreeSelector.jsx b/app/ui/src/components/contexts/ContextTreeSelector.jsx new file mode 100644 index 000000000..b8fc63b06 --- /dev/null +++ b/app/ui/src/components/contexts/ContextTreeSelector.jsx @@ -0,0 +1,130 @@ +import { useMemo, useState } from 'react'; +import { variantMeta, targetTypeMeta } from '../../utils/contextStyles'; + +// Left pane of the Contexts tab. Lists every root context. Grouped by +// contextType so "all OrgUnit roots" cluster, "all ResourceCluster roots" +// cluster, etc. Within a group, each entry shows variant colour + target +// badge + scope-system chip (when applicable). +// +// Filter bar on top: target type, variant, system. Useful when there are +// dozens of trees. + +export default function ContextTreeSelector({ roots, selectedRootId, onSelectRoot, onNewTree, loading }) { + const [filterTarget, setFilterTarget] = useState(''); + const [filterVariant, setFilterVariant] = useState(''); + const [filterSystem, setFilterSystem] = useState(''); + + const systems = useMemo(() => { + const seen = new Map(); + for (const r of roots) { + if (r.scopeSystemId && r.scopeSystemName && !seen.has(r.scopeSystemId)) { + seen.set(r.scopeSystemId, r.scopeSystemName); + } + } + return [...seen.entries()].sort((a, b) => a[1].localeCompare(b[1])); + }, [roots]); + + const filtered = useMemo(() => { + return roots.filter(r => + (!filterTarget || r.targetType === filterTarget) && + (!filterVariant || r.variant === filterVariant) && + (!filterSystem || String(r.scopeSystemId) === filterSystem) + ); + }, [roots, filterTarget, filterVariant, filterSystem]); + + const groups = useMemo(() => { + const map = new Map(); + for (const r of filtered) { + const key = `${r.contextType} (${r.targetType})`; + if (!map.has(key)) map.set(key, []); + map.get(key).push(r); + } + return [...map.entries()].sort((a, b) => a[0].localeCompare(b[0])); + }, [filtered]); + + return ( +
+
+
Trees
+ {onNewTree && ( + + )} +
+ +
+
+ + +
+ {systems.length > 0 && ( + + )} +
+ +
+ {loading &&
Loading…
} + {!loading && filtered.length === 0 && ( +
+ No trees match the current filters. Contexts arrive from a crawler (synced), from a plugin run (generated), or from the "+ New" button (manual). +
+ )} + {groups.map(([group, items]) => ( +
+
+ {group} · {items.length} +
+
    + {items.map(n => { + const v = variantMeta(n.variant); + const t = targetTypeMeta(n.targetType); + const selected = n.id === selectedRootId; + return ( +
  • + +
  • + ); + })} +
+
+ ))} +
+
+ ); +} diff --git a/app/ui/src/components/contexts/ContextTreeView.jsx b/app/ui/src/components/contexts/ContextTreeView.jsx new file mode 100644 index 000000000..24cdc32ef --- /dev/null +++ b/app/ui/src/components/contexts/ContextTreeView.jsx @@ -0,0 +1,63 @@ +import { useState } from 'react'; +import { variantMeta, targetTypeMeta } from '../../utils/contextStyles'; + +// Recursive tree renderer. Every node is a button (keyboard-accessible). +// aria-expanded set on parents. onOpenDetail opens the Context Detail tab. + +export default function ContextTreeView({ nodes, onOpenDetail }) { + return ( +
+
    + {nodes.map(n => )} +
+
+ ); +} + +function TreeNode({ node, depth, onOpenDetail }) { + const [expanded, setExpanded] = useState(depth < 2); + const hasChildren = node.children && node.children.length > 0; + const v = variantMeta(node.variant); + const t = targetTypeMeta(node.targetType); + + return ( +
  • +
    + {hasChildren ? ( + + ) : ( + + )} + +
    + {hasChildren && expanded && ( +
      + {node.children.map(c => ( + + ))} +
    + )} +
  • + ); +} diff --git a/app/ui/src/hooks/useContextTrees.js b/app/ui/src/hooks/useContextTrees.js new file mode 100644 index 000000000..7510c3df8 --- /dev/null +++ b/app/ui/src/hooks/useContextTrees.js @@ -0,0 +1,69 @@ +// Fetches and caches the list of Context roots + the currently-selected +// root's subtree. A single hook for the Contexts tab — the tree selector +// and the tree/list view both consume its output. + +import { useCallback, useEffect, useState } from 'react'; +import { useAuth } from '../auth/AuthGate'; + +export function useContextRoots() { + const { authFetch } = useAuth(); + const [roots, setRoots] = useState([]); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + + const reload = useCallback(async () => { + setLoading(true); setError(null); + try { + const r = await authFetch('/api/contexts'); + if (!r.ok) throw new Error(`HTTP ${r.status}`); + const body = await r.json(); + setRoots(body.data || []); + } catch (err) { + console.error('Failed to load context roots:', err); + setError(err.message || 'Failed to load contexts'); + } finally { + setLoading(false); + } + }, [authFetch]); + + useEffect(() => { reload(); }, [reload]); + + return { roots, loading, error, reload }; +} + +export function useContextSubtree(rootId) { + const { authFetch } = useAuth(); + const [nodes, setNodes] = useState([]); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + + const reload = useCallback(async () => { + if (!rootId) { setNodes([]); return; } + setLoading(true); setError(null); + try { + const r = await authFetch(`/api/contexts/tree?root=${encodeURIComponent(rootId)}`); + if (!r.ok) throw new Error(`HTTP ${r.status}`); + const body = await r.json(); + setNodes(Array.isArray(body) ? body : []); + } catch (err) { + console.error('Failed to load subtree:', err); + setError(err.message || 'Failed to load subtree'); + } finally { + setLoading(false); + } + }, [authFetch, rootId]); + + useEffect(() => { reload(); }, [reload]); + + return { nodes, loading, error, reload }; +} + +// Flattens a nested tree (children-of-children) into an indent-aware list. +// Used by the list view. +export function flattenTree(nodes, depth = 0, out = []) { + for (const n of nodes || []) { + out.push({ ...n, _depth: depth }); + if (n.children && n.children.length) flattenTree(n.children, depth + 1, out); + } + return out; +} diff --git a/app/ui/src/utils/contextStyles.js b/app/ui/src/utils/contextStyles.js new file mode 100644 index 000000000..3b1f733eb --- /dev/null +++ b/app/ui/src/utils/contextStyles.js @@ -0,0 +1,24 @@ +// Visual language for the Contexts tab. +// Two orthogonal dimensions — variant (who produced this context) and +// targetType (what it contains) — each with a distinct visual treatment. + +export const VARIANT_META = { + synced: { label: 'Synced', borderClass: 'border-blue-500', dotClass: 'bg-blue-500', textClass: 'text-blue-700' }, + generated: { label: 'Generated', borderClass: 'border-emerald-500', dotClass: 'bg-emerald-500', textClass: 'text-emerald-700' }, + manual: { label: 'Manual', borderClass: 'border-amber-600', dotClass: 'bg-amber-600', textClass: 'text-amber-700' }, +}; + +export const TARGET_TYPE_META = { + Identity: { label: 'Identity', badgeClass: 'bg-purple-100 text-purple-700 border-purple-200' }, + Resource: { label: 'Resource', badgeClass: 'bg-orange-100 text-orange-700 border-orange-200' }, + Principal: { label: 'Principal', badgeClass: 'bg-gray-100 text-gray-700 border-gray-200' }, + System: { label: 'System', badgeClass: 'bg-yellow-100 text-yellow-800 border-yellow-200' }, +}; + +export function variantMeta(variant) { + return VARIANT_META[variant] || { label: variant || 'Unknown', borderClass: 'border-gray-300', dotClass: 'bg-gray-300', textClass: 'text-gray-600' }; +} + +export function targetTypeMeta(t) { + return TARGET_TYPE_META[t] || { label: t || 'Unknown', badgeClass: 'bg-gray-100 text-gray-700 border-gray-200' }; +} diff --git a/changes/feature-context-redesign.md b/changes/feature-context-redesign.md index 23a5a5d9c..3789e0127 100644 --- a/changes/feature-context-redesign.md +++ b/changes/feature-context-redesign.md @@ -2,6 +2,7 @@ - CSV import: `Contexts.csv` gained `TargetType` and `OwnerUserId` columns; added `ContextMembers.csv` for explicit membership rows. Entra crawler no longer has a "Context" object type — context generation (org-chart, department tree) moves to plugin runs. - Added a context-algorithm plugin framework (registry, runner, dry-run, run history) with two initial plugins — `manager-hierarchy` and `department-tree` — that replace the old `/ingest/refresh-contexts` flow. Plugins are in-tree Node modules under `app/api/src/contexts/plugins/`; registered plugins are seeded into `ContextAlgorithms` at container startup. - Added `/api/context-plugins` endpoints: list plugins, dry-run, run (async, returns runId), list runs, and per-run status. +- UI: new **Contexts** tab with a grouped tree selector (by context type) on the left and tree / list views on the right. Visual signals distinguish variant (border + dot colour) from target type (pill) and show a scope-system chip when set. Context detail page rewritten to surface variant, target, scope system, owner, and provenance. - Added `ContextAlgorithms` and `ContextAlgorithmRuns` tables for plugin-driven context generation. - Rewrote `/api/contexts` routes: list / tree / detail / members plus full CRUD for manual contexts and their members. - Ingest API: added `/api/ingest/context-members`; removed the obsolete `/api/admin/refresh-contexts` and `/api/ingest/refresh-contexts` endpoints (replaced by the `department-tree` plugin once it ships). From 40ee054492e864e80392cae2c7c097005b0fb3a9 Mon Sep 17 00:00:00 2001 From: Wim van den Heijkant Date: Tue, 21 Apr 2026 15:02:00 +0200 Subject: [PATCH 062/160] =?UTF-8?q?Phase=205=20(WIP)=20=E2=80=94=20CreateM?= =?UTF-8?q?anualTreeModal=20component?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Standalone modal for creating a root-level manual context: target type, context type, display name, optional description, optional scope system. Calls POST /api/contexts. Not yet wired into the Contexts tab — the "+ New" dispatcher and the plugin-run modal still need to land before this is usable from the UI. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../contexts/CreateManualTreeModal.jsx | 145 ++++++++++++++++++ 1 file changed, 145 insertions(+) create mode 100644 app/ui/src/components/contexts/CreateManualTreeModal.jsx diff --git a/app/ui/src/components/contexts/CreateManualTreeModal.jsx b/app/ui/src/components/contexts/CreateManualTreeModal.jsx new file mode 100644 index 000000000..0046cb1ff --- /dev/null +++ b/app/ui/src/components/contexts/CreateManualTreeModal.jsx @@ -0,0 +1,145 @@ +import { useState, useEffect } from 'react'; +import { useAuth } from '../../auth/AuthGate'; + +// Minimal wizard for creating a manual root context. Target type + context +// type + name + optional description + optional scope system. + +export default function CreateManualTreeModal({ open, onClose, onCreated }) { + const { authFetch } = useAuth(); + const [targetType, setTargetType] = useState('Identity'); + const [contextType, setContextType] = useState(''); + const [displayName, setDisplayName] = useState(''); + const [description, setDescription] = useState(''); + const [scopeSystemId, setScopeSystemId] = useState(''); + const [systems, setSystems] = useState([]); + const [submitting, setSubmitting] = useState(false); + const [error, setError] = useState(null); + + useEffect(() => { + if (!open) return; + (async () => { + try { + const r = await authFetch('/api/systems'); + if (r.ok) { + const body = await r.json(); + setSystems(body.data || body || []); + } + } catch { /* non-critical */ } + })(); + }, [open, authFetch]); + + useEffect(() => { + if (!open) { + setTargetType('Identity'); setContextType(''); setDisplayName(''); + setDescription(''); setScopeSystemId(''); setError(null); setSubmitting(false); + } + }, [open]); + + if (!open) return null; + + const canSubmit = !!displayName.trim() && !!contextType.trim() && !submitting; + + async function submit() { + setSubmitting(true); setError(null); + try { + const body = { + targetType, + contextType: contextType.trim(), + displayName: displayName.trim(), + description: description.trim() || null, + scopeSystemId: scopeSystemId ? parseInt(scopeSystemId, 10) : null, + }; + const r = await authFetch('/api/contexts', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }); + if (!r.ok) { + const err = await r.json().catch(() => ({})); + throw new Error(err.error || `HTTP ${r.status}`); + } + const created = await r.json(); + onCreated?.(created); + onClose(); + } catch (err) { + setError(err.message || 'Failed to create context'); + } finally { + setSubmitting(false); + } + } + + return ( + +
    + + + + + setContextType(e.target.value)} + placeholder="Application" + className="w-full border rounded px-2 py-1 text-sm" + /> + + + setDisplayName(e.target.value)} + placeholder="Procurement app" + className="w-full border rounded px-2 py-1 text-sm" + /> + + +