From 86570be72bad9434ceae27a8a13087c2f0addf2d Mon Sep 17 00:00:00 2001 From: Karen Lai <7976322+karkarl@users.noreply.github.com> Date: Thu, 2 Jul 2026 16:10:30 -0700 Subject: [PATCH] feat: add Accessibility Insights CI tests using Axe.Windows Implements automated accessibility scanning modeled after the WinUI-Gallery pattern. Each app page is instantiated in the UIThreadFixture's hidden window and scanned via Axe.Windows.Automation against the live UIA tree for WCAG violations. Components added: - AxeHelper.cs: Wraps Axe.Windows scanner with global exclusions for known WinUI framework bugs (same set as WinUI-Gallery) and per-page exclusion support. Error output includes ControlType, Name, and AutomationId for actionable remediation. - AccessibilityScanTests.cs: Data-driven xUnit Theory that scans all 17 app pages. New pages are picked up by adding to PageTestData(). ChatPage is excluded (WebView2/CefSharp UIA tree crashes Axe). CI integration (.github/workflows/ci.yml): - Existing UI tests now exclude Category=Accessibility to avoid blocking on known violations while the team remediates (PR #921). - Dedicated 'Run Accessibility Tests (Axe.Windows)' step with continue-on-error: true surfaces violations as a visible warning in the GitHub Actions summary without blocking merges. - Step summary reports pass/fail with link to PR #921 for fixes. The tests will initially fail until PR #921 (accessibility fixes) lands. Remove continue-on-error once all violations are resolved. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/ci.yml | 37 ++++- .../AccessibilityScanTests.cs | 129 ++++++++++++++++++ tests/OpenClaw.Tray.UITests/AxeHelper.cs | 105 ++++++++++++++ .../OpenClaw.Tray.UITests.csproj | 8 +- 4 files changed, 276 insertions(+), 3 deletions(-) create mode 100644 tests/OpenClaw.Tray.UITests/AccessibilityScanTests.cs create mode 100644 tests/OpenClaw.Tray.UITests/AxeHelper.cs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 90ad6932a..1a53edd08 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -275,7 +275,42 @@ jobs: -r win-x64 --verbosity normal --results-directory TestResults\TrayUI - --logger trx;LogFileName=OpenClaw.Tray.UITests.trx" + --logger trx;LogFileName=OpenClaw.Tray.UITests.trx + --filter Category!=Accessibility" + + # Accessibility Insights CI pass: runs Axe.Windows scans against each app page. + # Uses continue-on-error so violations surface as a visible warning without + # blocking the build while the team remediates. Remove continue-on-error once + # all violations are resolved (see PR #921 for the initial fixes). + - name: Run Accessibility Tests (Axe.Windows) + id: a11y + continue-on-error: true + run: > + dotnet test tests/OpenClaw.Tray.UITests + --no-build + -c Debug + -r win-x64 + --verbosity normal + --results-directory TestResults\Accessibility + --logger "trx;LogFileName=Accessibility.trx" + --filter Category=Accessibility + + - name: Accessibility test summary + if: always() && steps.a11y.outcome != 'skipped' + shell: pwsh + run: | + if ("${{ steps.a11y.outcome }}" -eq "failure") { + "### :warning: Accessibility violations detected" >> $env:GITHUB_STEP_SUMMARY + "" >> $env:GITHUB_STEP_SUMMARY + "Axe.Windows scans found WCAG violations in one or more pages." >> $env:GITHUB_STEP_SUMMARY + "See the `Accessibility.trx` artifact for details." >> $env:GITHUB_STEP_SUMMARY + "" >> $env:GITHUB_STEP_SUMMARY + "Fixes for known violations: [PR #921](https://github.com/${{ github.repository }}/pull/921)" >> $env:GITHUB_STEP_SUMMARY + } else { + "### :white_check_mark: Accessibility scans passed" >> $env:GITHUB_STEP_SUMMARY + "" >> $env:GITHUB_STEP_SUMMARY + "All pages passed Axe.Windows accessibility validation." >> $env:GITHUB_STEP_SUMMARY + } - name: Upload Test Results if: always() diff --git a/tests/OpenClaw.Tray.UITests/AccessibilityScanTests.cs b/tests/OpenClaw.Tray.UITests/AccessibilityScanTests.cs new file mode 100644 index 000000000..49e86f4d1 --- /dev/null +++ b/tests/OpenClaw.Tray.UITests/AccessibilityScanTests.cs @@ -0,0 +1,129 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Axe.Windows.Core.Enums; +using Microsoft.UI.Xaml; +using Microsoft.UI.Xaml.Controls; +using Xunit; + +namespace OpenClaw.Tray.UITests; + +/// +/// Data-driven accessibility tests that scan each page in the OpenClaw app for +/// WCAG violations using Axe.Windows. Modeled after WinUI-Gallery's +/// AxeScanAllTests pattern. +/// +/// Each page is instantiated in the 's hidden window +/// and scanned via the UIA tree. New pages added to the app should be registered +/// in to be automatically included in CI scans. +/// +/// See PR #921 for fixes to the low-hanging accessibility violations discovered +/// by these tests. +/// +[Collection(UICollection.Name)] +public sealed class AccessibilityScanTests +{ + private readonly UIThreadFixture _ui; + + public AccessibilityScanTests(UIThreadFixture ui) + { + _ui = ui; + } + + /// + /// Pages that are completely excluded from scanning because they crash Axe + /// or require infrastructure unavailable in CI (e.g. WebView2, live connections). + /// + private static readonly HashSet ExcludedPages = + [ + // ChatPage hosts CefSharp/WebView2 which causes Axe to traverse the + // Chromium UIA tree and throw NullReferenceException (same root cause as + // WinUI-Gallery's WebView2 exclusion: axe-windows/issues/662). + "ChatPage", + ]; + + /// + /// Per-page rule exclusions for known issues specific to certain pages. + /// Add entries here when a page has violations caused by WinUI framework + /// limitations or third-party controls that cannot be fixed in app code. + /// Document the reason with a link to the upstream issue. + /// + private static readonly Dictionary PageRuleExclusions = new() + { + // ConnectionPage: gateway row cards use dynamic binding for accessible name; + // when no gateways are configured the template has empty Name. + // Fixed in PR #921; keep exclusion in case tests run without that PR. + ["ConnectionPage"] = [RuleId.NameNotNull], + }; + + /// + /// Enumerates all app pages for data-driven testing. Each entry is + /// [pageName, pageType]. New pages are automatically picked up when added here. + /// + public static IEnumerable PageTestData() + { + var pages = new (string Name, Type Type)[] + { + ("AgentEventsPage", typeof(OpenClawTray.Pages.AgentEventsPage)), + ("BindingsPage", typeof(OpenClawTray.Pages.BindingsPage)), + ("ChannelsPage", typeof(OpenClawTray.Pages.ChannelsPage)), + ("ConfigPage", typeof(OpenClawTray.Pages.ConfigPage)), + ("ConnectionPage", typeof(OpenClawTray.Pages.ConnectionPage)), + ("CronPage", typeof(OpenClawTray.Pages.CronPage)), + ("DebugPage", typeof(OpenClawTray.Pages.DebugPage)), + ("InstancesPage", typeof(OpenClawTray.Pages.InstancesPage)), + ("NotificationsPage", typeof(OpenClawTray.Pages.NotificationsPage)), + ("PermissionsPage", typeof(OpenClawTray.Pages.PermissionsPage)), + ("SandboxPage", typeof(OpenClawTray.Pages.SandboxPage)), + ("SessionsPage", typeof(OpenClawTray.Pages.SessionsPage)), + ("SettingsPage", typeof(OpenClawTray.Pages.SettingsPage)), + ("SkillsPage", typeof(OpenClawTray.Pages.SkillsPage)), + ("UsagePage", typeof(OpenClawTray.Pages.UsagePage)), + ("VoiceSettingsPage", typeof(OpenClawTray.Pages.VoiceSettingsPage)), + ("WorkspacePage", typeof(OpenClawTray.Pages.WorkspacePage)), + }; + + foreach (var (name, type) in pages) + { + if (!ExcludedPages.Contains(name)) + yield return [name, type]; + } + } + + /// + /// Instantiates each page in the test window and scans for accessibility + /// violations using Axe.Windows. Failures include the rule ID, element + /// control type, name, and automation ID for actionability. + /// + [Theory] + [Trait("Category", "Accessibility")] + [MemberData(nameof(PageTestData))] + public async Task Page_PassesAccessibilityScan(string pageName, Type pageType) + { + await _ui.ResetContainerAsync(); + + await _ui.RunOnUIAsync(() => + { + // Initialize Axe scanner on first use (attaches to current process) + AxeHelper.Initialize(); + + // Instantiate the page and host it in the test container + var page = (UIElement)Activator.CreateInstance(pageType)!; + + _ui.Container.Children.Clear(); + _ui.Container.Children.Add(page); + + // Force layout so the UIA tree is fully populated + _ui.Container.UpdateLayout(); + }); + + // Allow any async UI initialization (bindings, loaded events) to settle + await Task.Delay(500); + + await _ui.RunOnUIAsync(() => + { + PageRuleExclusions.TryGetValue(pageName, out var ruleExclusions); + AxeHelper.AssertNoAccessibilityErrors(ruleExclusions, pageName); + }); + } +} diff --git a/tests/OpenClaw.Tray.UITests/AxeHelper.cs b/tests/OpenClaw.Tray.UITests/AxeHelper.cs new file mode 100644 index 000000000..6a3a42fb5 --- /dev/null +++ b/tests/OpenClaw.Tray.UITests/AxeHelper.cs @@ -0,0 +1,105 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Axe.Windows.Automation; +using Axe.Windows.Core.Enums; +using Xunit; + +namespace OpenClaw.Tray.UITests; + +/// +/// Wraps Axe.Windows scanner to validate the live UI Automation tree for +/// accessibility violations. Modeled after the WinUI-Gallery AxeHelper pattern. +/// +/// The scanner attaches to the current process (which hosts the WinUI3 test window +/// via ) and inspects the UIA tree for WCAG violations. +/// +public static class AxeHelper +{ + private static IScanner? _scanner; + private static readonly object _lock = new(); + + /// + /// Rules excluded globally due to known WinUI framework bugs that are not + /// fixable in application code. These mirror the WinUI-Gallery exclusions. + /// + private static readonly HashSet GloballyExcludedRules = + [ + // WinUI framework generates non-informative names for some built-in controls + RuleId.NameIsInformative, + // Framework includes control type in auto-generated accessible names + RuleId.NameExcludesControlType, + // Same as above, localized variant + RuleId.NameExcludesLocalizedControlType, + // WinUI framework repeats sibling names in some control patterns + RuleId.SiblingUniqueAndFocusable, + ]; + + /// + /// Initialize the Axe.Windows scanner for the current process. + /// Thread-safe; subsequent calls are no-ops. + /// + public static void Initialize() + { + if (_scanner != null) return; + + lock (_lock) + { + if (_scanner != null) return; + + var processId = Environment.ProcessId; + var config = Config.Builder.ForProcessId(processId).Build(); + _scanner = ScannerFactory.CreateScanner(config); + } + } + + /// + /// Scan the current window's UIA tree and assert no accessibility violations exist. + /// + /// + /// Optional per-page rule exclusions for known issues specific to a page. + /// + /// + /// Optional context string (e.g. page name) included in failure messages. + /// + public static void AssertNoAccessibilityErrors( + IEnumerable? pageRuleExclusions = null, + string? context = null) + { + if (_scanner == null) + throw new InvalidOperationException( + "AxeHelper.Initialize() must be called before scanning."); + + var excludedRules = new HashSet(GloballyExcludedRules); + if (pageRuleExclusions != null) + excludedRules.UnionWith(pageRuleExclusions); + + var scanOutput = _scanner.Scan(null); + + var errors = scanOutput.WindowScanOutputs + .SelectMany(output => output.Errors) + .Where(error => !excludedRules.Contains(error.Rule.ID)) + .ToList(); + + if (errors.Count == 0) return; + + var errorMessages = errors.Select(error => + { + var controlType = error.Element.Properties.TryGetValue("ControlType", out var ct) + ? ct : "Unknown"; + var name = error.Element.Properties.TryGetValue("Name", out var n) + ? n : "(no name)"; + var automationId = error.Element.Properties.TryGetValue("AutomationId", out var aid) + ? aid : "(no id)"; + return $" [{error.Rule.ID}] Element '{controlType}' " + + $"(Name='{name}', AutomationId='{automationId}') " + + $"violated rule: {error.Rule.Description}"; + }); + + var header = string.IsNullOrEmpty(context) + ? $"Accessibility scan found {errors.Count} violation(s):" + : $"Accessibility scan of '{context}' found {errors.Count} violation(s):"; + + Assert.Fail($"{header}\r\n{string.Join("\r\n", errorMessages)}"); + } +} diff --git a/tests/OpenClaw.Tray.UITests/OpenClaw.Tray.UITests.csproj b/tests/OpenClaw.Tray.UITests/OpenClaw.Tray.UITests.csproj index cec2a6b9d..aade72467 100644 --- a/tests/OpenClaw.Tray.UITests/OpenClaw.Tray.UITests.csproj +++ b/tests/OpenClaw.Tray.UITests/OpenClaw.Tray.UITests.csproj @@ -30,11 +30,15 @@ - $(WarningsNotAsErrors);NU1904 + sets TreatWarningsAsErrors=true. + NU1903 also fires for System.IO.Packaging 8.0.0 from Axe.Windows + (GHSA-f32c-w444-8ppv, GHSA-qj66-m88j-hmgj); the scanner runs in-process + during tests only — not shipped — so the advisory is not actionable. --> + $(WarningsNotAsErrors);NU1904;NU1903 +