diff --git a/common/changes/@rushstack/rush-reporter/copilot-reporter-r8a-ai-gates_2026-08-28-08-45.json b/common/changes/@rushstack/rush-reporter/copilot-reporter-r8a-ai-gates_2026-08-28-08-45.json new file mode 100644 index 0000000000..aba0aed31d --- /dev/null +++ b/common/changes/@rushstack/rush-reporter/copilot-reporter-r8a-ai-gates_2026-08-28-08-45.json @@ -0,0 +1,11 @@ +{ + "changes": [ + { + "packageName": "@rushstack/rush-reporter", + "comment": "Add adversarial deterministic AI reporter qualification gates and privacy-safe actionable context.", + "type": "minor" + } + ], + "packageName": "@rushstack/rush-reporter", + "email": "TheLarkInn@users.noreply.github.com" +} diff --git a/common/changes/@rushstack/rush-reporter/r8-native-windows-qualification_2026-09-09.json b/common/changes/@rushstack/rush-reporter/r8-native-windows-qualification_2026-09-09.json new file mode 100644 index 0000000000..18702fe785 --- /dev/null +++ b/common/changes/@rushstack/rush-reporter/r8-native-windows-qualification_2026-09-09.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@rushstack/rush-reporter", + "comment": "Coalesce buffered start acknowledgements superseded by final results without dropping final context or log references, keeping native Windows AI output within the unchanged raw-byte qualification budget.", + "type": "patch" + } + ], + "packageName": "@rushstack/rush-reporter" +} diff --git a/common/changes/@rushstack/rush-reporter/review-r8-qualification_2026-09-09-13-00.json b/common/changes/@rushstack/rush-reporter/review-r8-qualification_2026-09-09-13-00.json new file mode 100644 index 0000000000..c809729dd4 --- /dev/null +++ b/common/changes/@rushstack/rush-reporter/review-r8-qualification_2026-09-09-13-00.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@rushstack/rush-reporter", + "comment": "Measure actual AI output bytes, enforce the invocation-wide NDJSON budget, and qualify retained fallback context and exact remediation without inflating baselines.", + "type": "patch" + } + ], + "packageName": "@rushstack/rush-reporter" +} diff --git a/common/reviews/api/rush-reporter.api.md b/common/reviews/api/rush-reporter.api.md index a11c6d1b87..575e3992d2 100644 --- a/common/reviews/api/rush-reporter.api.md +++ b/common/reviews/api/rush-reporter.api.md @@ -7,6 +7,12 @@ import type { Readable } from 'node:stream'; import type { Writable } from 'node:stream'; +// @beta +export const AI_REPORTER_QUALIFICATION_SCHEMA_VERSION: '1.0'; + +// @beta +export const AI_REPORTER_QUALIFICATION_THRESHOLDS: IAiReporterQualificationThresholds; + // @beta export class AiReporter implements IReporter { constructor(options: IAiReporterOptions); @@ -139,6 +145,9 @@ export function detectAgent(env: Record, configuredV // @beta export function encodeNdjsonRecord(value: unknown, options?: INdjsonOptions): string; +// @beta +export function evaluateAiReporterQualification(cases: readonly IAiReporterQualificationCaseResult[], thresholds?: IAiReporterQualificationThresholds): IAiReporterQualificationResult; + // @beta export function evaluatePluginApplyGate(manifests: readonly IRushPluginManifest[], options: IPluginApplyGateOptions): IPluginApplyDecision[]; @@ -170,6 +179,9 @@ export class FileReporter implements IReporter { // @beta export function filterEventsForLogLevel(logLevel: ReporterLogLevel, events: readonly IReporterEventEnvelope[]): IReporterEventEnvelope[]; +// @beta +export function formatAiReporterQualificationFailures(result: IAiReporterQualificationResult): string; + // @beta export function getBlockedPlugins(decisions: readonly IPluginApplyDecision[]): IPluginApplyDecision[]; @@ -182,6 +194,9 @@ export function getLogLevelRank(level: ReporterLogLevel): number; // @beta export function getPrivacyClassificationRank(classification: ReporterPrivacyClassification): number; +// @beta +export function getQualifiedAiReporterDecision(env: Record, configuredAgentEnvironmentVariables: readonly string[], qualification: IAiReporterQualificationResult | undefined, privacyPrerequisiteAccepted?: boolean): IQualifiedAiReporterDecision; + // @beta export function getReporterMigrationPhase(id: ReporterMigrationPhaseId): IReporterMigrationPhase; @@ -219,11 +234,18 @@ export interface IAiDiagnostic { // (undocumented) readonly code: string; // (undocumented) + readonly context?: Readonly>; + // (undocumented) + readonly detailKey?: string; + // (undocumented) + readonly diagnosticId?: string; + // (undocumented) readonly remediation?: readonly IRushRemediationAction[]; // (undocumented) readonly severity: string; // (undocumented) readonly summary?: string; + readonly summaryKey?: string; } // @beta @@ -280,6 +302,98 @@ export interface IAiReporterOptions { readonly write: (text: string) => void; } +// @beta +export interface IAiReporterQualificationCaseResult { + // (undocumented) + readonly actionable: boolean; + // (undocumented) + readonly aiOutputBytes: number; + // (undocumented) + readonly deterministic: boolean; + // (undocumented) + readonly expectedResult: 'succeeded' | 'failed'; + // (undocumented) + readonly failures: readonly string[]; + // (undocumented) + readonly fullLogValid: boolean; + // (undocumented) + readonly legacyOutputBytes: number; + // (undocumented) + readonly name: string; + // (undocumented) + readonly normalizedAiOutputSha256: string; + // (undocumented) + readonly plaintextOutputBytes: number; + // (undocumented) + readonly privacySafe: boolean; + // (undocumented) + readonly scenario: string; + // (undocumented) + readonly stdoutContractValid: boolean; + // (undocumented) + readonly warningContractValid: boolean; +} + +// @beta +export interface IAiReporterQualificationGateResult { + // (undocumented) + readonly actual: number; + // (undocumented) + readonly failedCases: readonly string[]; + // (undocumented) + readonly id: string; + // (undocumented) + readonly passed: boolean; + // (undocumented) + readonly threshold: string; +} + +// @beta +export interface IAiReporterQualificationResult { + // (undocumented) + readonly cases: readonly IAiReporterQualificationCaseResult[]; + // (undocumented) + readonly gates: readonly IAiReporterQualificationGateResult[]; + // (undocumented) + readonly passed: boolean; + // (undocumented) + readonly schemaVersion: typeof AI_REPORTER_QUALIFICATION_SCHEMA_VERSION; + // (undocumented) + readonly thresholds: IAiReporterQualificationThresholds; +} + +// @beta +export interface IAiReporterQualificationThresholds { + // (undocumented) + readonly deterministicRunCount: number; + // (undocumented) + readonly maximumAggregateAiToLegacyPercent: number; + // (undocumented) + readonly maximumAggregateAiToPlaintextPercent: number; + // (undocumented) + readonly maximumCompactCaseAiOutputBytes: number; + // (undocumented) + readonly maximumOutputBytesPerCase: number; + // (undocumented) + readonly maximumPerCaseAiToBaselinePercent: number; + // (undocumented) + readonly minimumActionableFailurePercent: number; + // (undocumented) + readonly minimumComparableBaselineBytes: number; + // (undocumented) + readonly minimumControlCases: number; + // (undocumented) + readonly minimumFailureCases: number; + // (undocumented) + readonly minimumFullLogPassPercent: number; + // (undocumented) + readonly minimumPrivacyPassPercent: number; + // (undocumented) + readonly minimumStdoutContractPassPercent: number; + // (undocumented) + readonly minimumWarningContractPassPercent: number; +} + // @beta export interface IAutomaticReporterPlan { readonly emergencyDestination: 'stderr'; @@ -702,6 +816,18 @@ export interface IProblemMatcherResult { readonly unmatchedLineCount: number; } +// @beta +export interface IQualifiedAiReporterDecision { + // (undocumented) + readonly agentDetected: boolean; + // (undocumented) + readonly eligible: boolean; + // (undocumented) + readonly reason: 'RUSH_REPORTER=legacy' | 'agent not detected' | 'qualification unavailable' | 'qualification failed' | 'privacy prerequisite unavailable' | 'qualified'; + // (undocumented) + readonly reporter?: 'ai'; +} + // @beta export interface IRenderLiveRegionOptions { readonly color: IColorizer; @@ -1468,6 +1594,9 @@ export function resolveReporterCompatibility(frontend: IReporterFrontendDescript // @beta export function resolveReporterSelection(input: IReporterSelectionInput): IReporterSelection; +// @beta +export function runAiReporterQualificationCorpusAsync(): Promise; + // @beta export function runProblemMatchers(events: readonly IReporterEventEnvelope[], matchers: readonly IProblemMatcher[], options?: IRunProblemMatchersOptions): IProblemMatcherResult; diff --git a/libraries/reporter/README.md b/libraries/reporter/README.md index b326abb9f1..a21527f076 100644 --- a/libraries/reporter/README.md +++ b/libraries/reporter/README.md @@ -4,6 +4,55 @@ Canonical event protocol, reporter manager, and built-in reporters for Rush. This package is released as a public beta. Exported contracts may change before the stable release. +## AI reporter qualification + +The network-free qualification corpus runs representative bootstrap/version, configuration, input, +dependency-tool, operation, cache, network/auth, plugin, cancellation, and internal failures plus +successful and warning-only controls through the AI, detailed plaintext, legacy, and full-log reporters. +Scenario-specific external output is included only where the real failure or control would produce it. + +| Gate | Blocking threshold | +| --- | --- | +| Failure/control coverage | At least 10 failure cases and 2 successful controls | +| Actionability | 100% of failures retain stable code, category, context, and remediation | +| Output size | At most 64 KiB per case; compact cases at most 2 KiB; AI no larger than comparable per-case baselines; aggregate AI bytes at most 50% of legacy and plaintext | +| Determinism | Byte-identical normalized AI output across 3 runs | +| Privacy | 100% secret redaction and no private producer identity leakage | +| Full log | 100% absolute, existing, owner-only where supported, complete, and failure-correlated | +| Stdout/warnings | 100% payload-only NDJSON and warning suppression/detail compliance | + +Run `rushx build && node scripts/runAiReporterQualification.js` from this project to print the +machine-readable result. Byte gates measure the actual emitted UTF-8 strings, including absolute paths and +NDJSON delimiters. Paths are normalized only for deterministic comparison/hashing and are not stored. +Separate near-limit and sustained-watch probes enforce the invocation budget without adding artificial +baseline volume to the comparison corpus. Passing +these gates only produces a reusable qualification decision; it does not enable environment-based automatic +reporter selection. That decision also requires the separate telemetry privacy prerequisite to be accepted. +The pre-major Rush frontend remains explicit/repository-opt-in, and `RUSH_REPORTER=legacy` remains +authoritative. +The Jest setup hook has a bounded 15-second allowance for the three file-backed corpus passes, matching +the integration test setup policy. This allowance does not change any quality gate or production deadline. + +AI output reserves final-record space, including its supplied log reference, before emitting progress. +Progress is buffered within the invocation byte limit until the primary log reservation is known, or until +close if no log is supplied. Excess progress/details set `truncated`; the final result remains valid JSON. +An unrendered start acknowledgement is coalesced into a known final result. Ongoing commands still expose +buffered status at the next non-terminal event or explicit flush; watch history and every final field, +including the supplied log reference, are retained. No path shortening or measurement normalization is used. +The final scope carries the command name, and standard `diagnostic..summary` keys are implicit rather +than repeated alongside the same code. Custom summary keys are preserved. + +AI fallback message text is emitted only for public envelopes. Its context retains the known command, and +the usage-review action invokes that command's help (or `rush --help` when the command is unavailable). +Qualification checks exact expected context, remediation commands/URLs, descriptions, and execution safety; +summary-only failures and unrelated actions do not qualify. Non-public fallback errors remain countable +and refer to the protected full-detail log. JSON oversized-record markers preserve the original privacy +classification and omit non-public source and scope metadata. + +Secret envelopes retain only protocol, event identity, ordering, timing, type, privacy, and fully redacted +source and payload fields. Contextual parent, command, operation, project, phase, and scope metadata is +removed. + ## Links - [CHANGELOG.md](https://github.com/microsoft/rushstack/blob/main/libraries/reporter/CHANGELOG.md) - Find out diff --git a/libraries/reporter/scripts/runAiReporterQualification.js b/libraries/reporter/scripts/runAiReporterQualification.js new file mode 100644 index 0000000000..53e714d6fa --- /dev/null +++ b/libraries/reporter/scripts/runAiReporterQualification.js @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +const { + formatAiReporterQualificationFailures, + runAiReporterQualificationCorpusAsync +} = require('../lib-commonjs'); + +runAiReporterQualificationCorpusAsync() + .then((result) => { + process.stdout.write(`${JSON.stringify(result, undefined, 2)}\n`); + if (!result.passed) { + process.stderr.write(`${formatAiReporterQualificationFailures(result)}\n`); + process.exitCode = 1; + } + }) + .catch((error) => { + process.stderr.write(`${error.stack || error.message}\n`); + process.exitCode = 1; + }); diff --git a/libraries/reporter/src/index.ts b/libraries/reporter/src/index.ts index 2167262849..4b2509960f 100644 --- a/libraries/reporter/src/index.ts +++ b/libraries/reporter/src/index.ts @@ -337,6 +337,22 @@ export { isWithinMemoryBudget } from './perf/PerformanceBudgets'; +export type { + IAiReporterQualificationThresholds, + IAiReporterQualificationCaseResult, + IAiReporterQualificationGateResult, + IAiReporterQualificationResult, + IQualifiedAiReporterDecision +} from './qualification/AiReporterQualification'; +export { + AI_REPORTER_QUALIFICATION_SCHEMA_VERSION, + AI_REPORTER_QUALIFICATION_THRESHOLDS, + evaluateAiReporterQualification, + formatAiReporterQualificationFailures, + getQualifiedAiReporterDecision +} from './qualification/AiReporterQualification'; +export { runAiReporterQualificationCorpusAsync } from './qualification/AiReporterQualificationCorpus'; + export type { ReporterMigrationPhaseId, IReporterMigrationPhase } from './migration/MigrationPhase'; export { REPORTER_MIGRATION_PHASES, getReporterMigrationPhase } from './migration/MigrationPhase'; export type { diff --git a/libraries/reporter/src/qualification/AiReporterQualification.ts b/libraries/reporter/src/qualification/AiReporterQualification.ts new file mode 100644 index 0000000000..6d659e6a78 --- /dev/null +++ b/libraries/reporter/src/qualification/AiReporterQualification.ts @@ -0,0 +1,390 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { detectAgent } from '../config/AgentDetection'; +import { isLegacyEmergencyFallbackRequested } from '../reporters/LegacyReporter'; +import { REPORTER_PERFORMANCE_BUDGETS } from '../perf/PerformanceBudgets'; + +/** + * The version of the machine-readable AI reporter qualification result. + * + * @beta + */ +export const AI_REPORTER_QUALIFICATION_SCHEMA_VERSION: '1.0' = '1.0'; + +/** + * The blocking thresholds for the deterministic AI reporter corpus. + * + * @beta + */ +export interface IAiReporterQualificationThresholds { + readonly minimumFailureCases: number; + readonly minimumControlCases: number; + readonly minimumActionableFailurePercent: number; + readonly maximumOutputBytesPerCase: number; + readonly maximumCompactCaseAiOutputBytes: number; + readonly minimumComparableBaselineBytes: number; + readonly maximumPerCaseAiToBaselinePercent: number; + readonly maximumAggregateAiToLegacyPercent: number; + readonly maximumAggregateAiToPlaintextPercent: number; + readonly deterministicRunCount: number; + readonly minimumPrivacyPassPercent: number; + readonly minimumFullLogPassPercent: number; + readonly minimumStdoutContractPassPercent: number; + readonly minimumWarningContractPassPercent: number; +} + +/** + * The frozen qualification thresholds used by CI. + * + * @beta + */ +export const AI_REPORTER_QUALIFICATION_THRESHOLDS: IAiReporterQualificationThresholds = { + minimumFailureCases: 10, + minimumControlCases: 2, + minimumActionableFailurePercent: 100, + maximumOutputBytesPerCase: REPORTER_PERFORMANCE_BUDGETS.maxAiOutputBytes, + maximumCompactCaseAiOutputBytes: 2 * 1024, + minimumComparableBaselineBytes: 1024, + maximumPerCaseAiToBaselinePercent: 100, + maximumAggregateAiToLegacyPercent: 50, + maximumAggregateAiToPlaintextPercent: 50, + deterministicRunCount: 3, + minimumPrivacyPassPercent: 100, + minimumFullLogPassPercent: 100, + minimumStdoutContractPassPercent: 100, + minimumWarningContractPassPercent: 100 +}; + +/** + * A safe, path-free measurement for one deterministic corpus case. + * + * @beta + */ +export interface IAiReporterQualificationCaseResult { + readonly name: string; + readonly scenario: string; + readonly expectedResult: 'succeeded' | 'failed'; + readonly aiOutputBytes: number; + readonly plaintextOutputBytes: number; + readonly legacyOutputBytes: number; + readonly normalizedAiOutputSha256: string; + readonly actionable: boolean; + readonly deterministic: boolean; + readonly privacySafe: boolean; + readonly fullLogValid: boolean; + readonly stdoutContractValid: boolean; + readonly warningContractValid: boolean; + readonly failures: readonly string[]; +} + +/** + * The result of one blocking qualification gate. + * + * @beta + */ +export interface IAiReporterQualificationGateResult { + readonly id: string; + readonly passed: boolean; + readonly actual: number; + readonly threshold: string; + readonly failedCases: readonly string[]; +} + +/** + * The complete machine-readable AI reporter qualification result. + * + * @beta + */ +export interface IAiReporterQualificationResult { + readonly schemaVersion: typeof AI_REPORTER_QUALIFICATION_SCHEMA_VERSION; + readonly passed: boolean; + readonly thresholds: IAiReporterQualificationThresholds; + readonly cases: readonly IAiReporterQualificationCaseResult[]; + readonly gates: readonly IAiReporterQualificationGateResult[]; +} + +function percent(passing: number, total: number): number { + return total === 0 ? 0 : (passing / total) * 100; +} + +function ratioPercent(numerator: number, denominator: number): number { + return denominator === 0 + ? numerator === 0 + ? 0 + : Number.MAX_SAFE_INTEGER + : (numerator / denominator) * 100; +} + +function getHighestRatioCaseNames( + cases: readonly IAiReporterQualificationCaseResult[], + getDenominator: (testCase: IAiReporterQualificationCaseResult) => number +): string[] { + return [...cases] + .sort( + (left, right) => + ratioPercent(right.aiOutputBytes, getDenominator(right)) - + ratioPercent(left.aiOutputBytes, getDenominator(left)) + ) + .slice(0, 3) + .map(({ name }) => name); +} + +function createPerCaseRatioGate( + id: string, + cases: readonly IAiReporterQualificationCaseResult[], + getDenominator: (testCase: IAiReporterQualificationCaseResult) => number, + minimumComparableBaselineBytes: number, + maximumPercent: number +): IAiReporterQualificationGateResult { + const comparableCases: readonly IAiReporterQualificationCaseResult[] = cases.filter( + (testCase) => getDenominator(testCase) >= minimumComparableBaselineBytes + ); + const failedCases: string[] = comparableCases + .filter((testCase) => ratioPercent(testCase.aiOutputBytes, getDenominator(testCase)) > maximumPercent) + .map(({ name }) => name); + return { + id, + passed: failedCases.length === 0, + actual: Math.max( + 0, + ...comparableCases.map((testCase) => ratioPercent(testCase.aiOutputBytes, getDenominator(testCase))) + ), + threshold: `<= ${maximumPercent}% when baseline >= ${minimumComparableBaselineBytes} bytes`, + failedCases + }; +} + +function createPercentageGate( + id: string, + cases: readonly IAiReporterQualificationCaseResult[], + predicate: (testCase: IAiReporterQualificationCaseResult) => boolean, + minimumPercent: number +): IAiReporterQualificationGateResult { + const failedCases: string[] = cases.filter((testCase) => !predicate(testCase)).map(({ name }) => name); + return { + id, + passed: percent(cases.length - failedCases.length, cases.length) >= minimumPercent, + actual: percent(cases.length - failedCases.length, cases.length), + threshold: `>= ${minimumPercent}%`, + failedCases + }; +} + +/** + * Evaluates safe corpus measurements against the frozen blocking thresholds. + * + * @beta + */ +export function evaluateAiReporterQualification( + cases: readonly IAiReporterQualificationCaseResult[], + thresholds: IAiReporterQualificationThresholds = AI_REPORTER_QUALIFICATION_THRESHOLDS +): IAiReporterQualificationResult { + const failureCases: readonly IAiReporterQualificationCaseResult[] = cases.filter( + ({ expectedResult }) => expectedResult === 'failed' + ); + const controlCases: readonly IAiReporterQualificationCaseResult[] = cases.filter( + ({ expectedResult }) => expectedResult === 'succeeded' + ); + const totalAiBytes: number = cases.reduce((sum, testCase) => sum + testCase.aiOutputBytes, 0); + const totalLegacyBytes: number = cases.reduce((sum, testCase) => sum + testCase.legacyOutputBytes, 0); + const totalPlaintextBytes: number = cases.reduce((sum, testCase) => sum + testCase.plaintextOutputBytes, 0); + const aggregateAiToLegacyPercent: number = ratioPercent(totalAiBytes, totalLegacyBytes); + const aggregateAiToPlaintextPercent: number = ratioPercent(totalAiBytes, totalPlaintextBytes); + + const gates: IAiReporterQualificationGateResult[] = [ + { + id: 'corpus.failure-cases', + passed: failureCases.length >= thresholds.minimumFailureCases, + actual: failureCases.length, + threshold: `>= ${thresholds.minimumFailureCases}`, + failedCases: [] + }, + { + id: 'corpus.control-cases', + passed: controlCases.length >= thresholds.minimumControlCases, + actual: controlCases.length, + threshold: `>= ${thresholds.minimumControlCases}`, + failedCases: [] + }, + createPercentageGate( + 'actionability', + failureCases, + ({ actionable }) => actionable, + thresholds.minimumActionableFailurePercent + ), + { + id: 'size.absolute', + passed: cases.every(({ aiOutputBytes }) => aiOutputBytes <= thresholds.maximumOutputBytesPerCase), + actual: Math.max(0, ...cases.map(({ aiOutputBytes }) => aiOutputBytes)), + threshold: `<= ${thresholds.maximumOutputBytesPerCase} bytes`, + failedCases: cases + .filter(({ aiOutputBytes }) => aiOutputBytes > thresholds.maximumOutputBytesPerCase) + .map(({ name }) => name) + }, + { + id: 'size.compact-case', + passed: cases.every( + ({ aiOutputBytes, legacyOutputBytes, plaintextOutputBytes }) => + Math.max(legacyOutputBytes, plaintextOutputBytes) >= thresholds.minimumComparableBaselineBytes || + aiOutputBytes <= thresholds.maximumCompactCaseAiOutputBytes + ), + actual: Math.max( + 0, + ...cases + .filter( + ({ legacyOutputBytes, plaintextOutputBytes }) => + Math.max(legacyOutputBytes, plaintextOutputBytes) < thresholds.minimumComparableBaselineBytes + ) + .map(({ aiOutputBytes }) => aiOutputBytes) + ), + threshold: + `<= ${thresholds.maximumCompactCaseAiOutputBytes} bytes when both baselines are below ` + + `${thresholds.minimumComparableBaselineBytes} bytes`, + failedCases: cases + .filter( + ({ aiOutputBytes, legacyOutputBytes, plaintextOutputBytes }) => + Math.max(legacyOutputBytes, plaintextOutputBytes) < thresholds.minimumComparableBaselineBytes && + aiOutputBytes > thresholds.maximumCompactCaseAiOutputBytes + ) + .map(({ name }) => name) + }, + createPerCaseRatioGate( + 'size.per-case-vs-legacy', + cases, + ({ legacyOutputBytes }) => legacyOutputBytes, + thresholds.minimumComparableBaselineBytes, + thresholds.maximumPerCaseAiToBaselinePercent + ), + createPerCaseRatioGate( + 'size.per-case-vs-plaintext', + cases, + ({ plaintextOutputBytes }) => plaintextOutputBytes, + thresholds.minimumComparableBaselineBytes, + thresholds.maximumPerCaseAiToBaselinePercent + ), + { + id: 'size.vs-legacy', + passed: aggregateAiToLegacyPercent <= thresholds.maximumAggregateAiToLegacyPercent, + actual: aggregateAiToLegacyPercent, + threshold: `<= ${thresholds.maximumAggregateAiToLegacyPercent}%`, + failedCases: + aggregateAiToLegacyPercent <= thresholds.maximumAggregateAiToLegacyPercent + ? [] + : getHighestRatioCaseNames(cases, ({ legacyOutputBytes }) => legacyOutputBytes) + }, + { + id: 'size.vs-plaintext', + passed: aggregateAiToPlaintextPercent <= thresholds.maximumAggregateAiToPlaintextPercent, + actual: aggregateAiToPlaintextPercent, + threshold: `<= ${thresholds.maximumAggregateAiToPlaintextPercent}%`, + failedCases: + aggregateAiToPlaintextPercent <= thresholds.maximumAggregateAiToPlaintextPercent + ? [] + : getHighestRatioCaseNames(cases, ({ plaintextOutputBytes }) => plaintextOutputBytes) + }, + createPercentageGate('determinism', cases, ({ deterministic }) => deterministic, 100), + createPercentageGate( + 'privacy', + cases, + ({ privacySafe }) => privacySafe, + thresholds.minimumPrivacyPassPercent + ), + createPercentageGate( + 'full-log', + cases, + ({ fullLogValid }) => fullLogValid, + thresholds.minimumFullLogPassPercent + ), + createPercentageGate( + 'stdout-contract', + cases, + ({ stdoutContractValid }) => stdoutContractValid, + thresholds.minimumStdoutContractPassPercent + ), + createPercentageGate( + 'warning-contract', + cases, + ({ warningContractValid }) => warningContractValid, + thresholds.minimumWarningContractPassPercent + ) + ]; + + return { + schemaVersion: AI_REPORTER_QUALIFICATION_SCHEMA_VERSION, + passed: gates.every(({ passed }) => passed), + thresholds, + cases, + gates + }; +} + +/** + * Formats failed gates with actionable case names for CI output. + * + * @beta + */ +export function formatAiReporterQualificationFailures(result: IAiReporterQualificationResult): string { + return result.gates + .filter(({ passed }) => !passed) + .map( + ({ id, actual, threshold, failedCases }) => + `${id}: actual=${Number.isFinite(actual) ? actual.toFixed(2) : String(actual)}, ` + + `required=${threshold}` + + (failedCases.length > 0 ? `; cases=${failedCases.join(', ')}` : '') + ) + .join('\n'); +} + +/** + * The isolated decision produced for a future automatic-selection integration. + * + * @beta + */ +export interface IQualifiedAiReporterDecision { + readonly agentDetected: boolean; + readonly eligible: boolean; + readonly reporter?: 'ai'; + readonly reason: + | 'RUSH_REPORTER=legacy' + | 'agent not detected' + | 'qualification unavailable' + | 'qualification failed' + | 'privacy prerequisite unavailable' + | 'qualified'; +} + +/** + * Resolves whether an agent environment is eligible for a future AI reporter selection. + * + * @remarks + * This helper does not alter reporter selection by itself. The pre-major Rush + * frontend remains opt-in-only until rollout integration explicitly consumes a + * passing result. + * + * @beta + */ +export function getQualifiedAiReporterDecision( + env: Record, + configuredAgentEnvironmentVariables: readonly string[], + qualification: IAiReporterQualificationResult | undefined, + privacyPrerequisiteAccepted: boolean = false +): IQualifiedAiReporterDecision { + const agentDetected: boolean = detectAgent(env, configuredAgentEnvironmentVariables); + if (isLegacyEmergencyFallbackRequested(env)) { + return { agentDetected, eligible: false, reason: 'RUSH_REPORTER=legacy' }; + } + if (!agentDetected) { + return { agentDetected: false, eligible: false, reason: 'agent not detected' }; + } + if (!qualification) { + return { agentDetected: true, eligible: false, reason: 'qualification unavailable' }; + } + if (!qualification.passed) { + return { agentDetected: true, eligible: false, reason: 'qualification failed' }; + } + if (!privacyPrerequisiteAccepted) { + return { agentDetected: true, eligible: false, reason: 'privacy prerequisite unavailable' }; + } + return { agentDetected: true, eligible: true, reporter: 'ai', reason: 'qualified' }; +} diff --git a/libraries/reporter/src/qualification/AiReporterQualificationCorpus.ts b/libraries/reporter/src/qualification/AiReporterQualificationCorpus.ts new file mode 100644 index 0000000000..21a45e6461 --- /dev/null +++ b/libraries/reporter/src/qualification/AiReporterQualificationCorpus.ts @@ -0,0 +1,1131 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { createHash } from 'node:crypto'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; + +import type { IReporterEventEnvelope } from '../events/IReporterEventEnvelope'; +import type { ReporterPrivacyClassification } from '../events/ReporterPrivacyClassification'; +import type { IRushRemediationAction } from '../diagnostics/IRushRemediationAction'; +import type { IAiDiagnostic, IAiFinalRecord } from '../reporters/AiReporter'; +import { AiReporter } from '../reporters/AiReporter'; +import { FileReporter, type IFileReporterArtifact } from '../reporters/FileReporter'; +import { JsonReporter } from '../reporters/JsonReporter'; +import { LegacyReporter } from '../reporters/LegacyReporter'; +import { PlaintextReporter } from '../reporters/PlaintextReporter'; +import { + AI_REPORTER_QUALIFICATION_THRESHOLDS, + evaluateAiReporterQualification, + type IAiReporterQualificationCaseResult, + type IAiReporterQualificationGateResult, + type IAiReporterQualificationResult +} from './AiReporterQualification'; + +const FIXED_TIMESTAMP: string = '2026-08-28T08:45:34.000Z'; +const FIXED_TIME_MS: number = Date.parse(FIXED_TIMESTAMP); +const FIXED_PID: number = 4242; +const CLASSIFIED_SECRET: string = 'qualification-fake-secret-token'; +const CLASSIFIED_SECRET_PRODUCER: string = '@secret/qualification-fixture'; +const CLASSIFIED_SECRET_COMPONENT: string = 'SecretQualificationFixture'; +const CLASSIFIED_SECRET_COMMAND: string = 'qualification-secret-command'; +const CLASSIFIED_SECRET_OPERATION: string = 'qualification-secret-operation'; +const CLASSIFIED_SECRET_PROJECT: string = '@private/qualification-secret-project'; +const CLASSIFIED_SECRET_PHASE: string = 'qualification-secret-phase'; +const CLASSIFIED_SECRET_PARENT_SESSION: string = 'qualification-secret-parent-session'; +const CLASSIFIED_SECRET_PARENT_OPERATION: string = 'qualification-secret-parent-operation'; +const CLASSIFIED_SECRET_MESSAGE: string = 'qualification-secret-message-text'; +const CLASSIFIED_SECRET_DIAGNOSTIC: string = 'qualification-secret-diagnostic-summary'; +const PRIVATE_PRODUCER: string = '@private/example-rush-plugin'; +const PRIVATE_COMPONENT: string = 'PrivatePluginImplementation'; +const LOCAL_SENSITIVE_FALLBACK_MESSAGE: string = 'qualification-local-sensitive-fallback-message'; +const OVERSIZED_LOCAL_SENSITIVE_VALUE: string = 'qualification-oversized-local-sensitive-value'; +const OVERSIZED_LOCAL_SENSITIVE_PRODUCER: string = '@private/oversized-qualification-fixture'; +const OVERSIZED_LOCAL_SENSITIVE_COMPONENT: string = 'OversizedPrivateQualificationFixture'; +const OVERSIZED_LOCAL_SENSITIVE_SCOPE: string = '@private/oversized-qualification-project'; + +function createExternalOutput(lines: readonly string[], repetitions: number): string { + const output: string[] = []; + for (let index: number = 0; index < repetitions; index++) { + output.push(lines[index % lines.length].split('{index}').join(String(index + 1))); + } + return `${output.join('\n')}\n`; +} + +interface ICorpusDiagnostic { + readonly code: string; + readonly category: string; + readonly summaryKey: string; + readonly parameters: Readonly< + Record< + string, + { readonly value: string | number | boolean; readonly privacy: ReporterPrivacyClassification } + > + >; + readonly remediation: readonly { + readonly descriptionKey: string; + readonly command?: string; + readonly documentationUrl?: string; + readonly automatedExecutionSafety: 'safe' | 'requires-confirmation' | 'unsafe'; + }[]; + readonly privacy?: ReporterPrivacyClassification; + readonly sourcePackage?: string; + readonly sourceComponent?: string; +} + +interface ICorpusCase { + readonly name: string; + readonly scenario: string; + readonly expectedResult: 'succeeded' | 'failed'; + readonly diagnostic?: ICorpusDiagnostic; + readonly externalOutput?: string; + readonly fallbackMessages?: readonly { + readonly text: string; + readonly privacy: ReporterPrivacyClassification; + }[]; + readonly operationStatus?: 'success' | 'failure' | 'aborted' | 'fromCache'; + readonly warningOnly?: boolean; +} + +interface ICaseRun { + readonly normalizedAiOutput: string; + readonly normalizedPlaintextOutput: string; + readonly normalizedLegacyOutput: string; + readonly result: Omit; +} + +/** @internal */ +export function hasExpectedAiQualificationDiagnostic( + actual: IAiDiagnostic | undefined, + expected: ICorpusDiagnostic +): boolean { + return Boolean( + actual && + actual.code === expected.code && + actual.category === expected.category && + (actual.summaryKey ?? `diagnostic.${actual.code}.summary`) === expected.summaryKey && + Object.keys(actual.context ?? {}).length === Object.keys(expected.parameters).length && + Object.entries(expected.parameters).every(([name, parameter]) => { + const expectedValue: string | number | boolean = + parameter.privacy === 'public' ? parameter.value : `[${parameter.privacy}]`; + return actual.context?.[name] === expectedValue; + }) && + expected.remediation.length > 0 && + actual.remediation?.length === expected.remediation.length && + expected.remediation.every((action, index) => { + const projected: IRushRemediationAction | undefined = actual.remediation?.[index]; + return ( + Boolean(action.command || action.documentationUrl) && + projected?.descriptionKey === action.descriptionKey && + projected.command === action.command && + projected.documentationUrl === action.documentationUrl && + projected.automatedExecutionSafety === action.automatedExecutionSafety + ); + }) + ); +} + +const CORPUS: readonly ICorpusCase[] = [ + { + name: 'bootstrap-unsupported-node', + scenario: 'Rush bootstrap rejects an unsupported Node.js version', + expectedResult: 'failed', + diagnostic: { + code: 'RUSH_ENVIRONMENT_UNSUPPORTED_NODE', + category: 'environment', + summaryKey: 'diagnostic.RUSH_ENVIRONMENT_UNSUPPORTED_NODE.summary', + parameters: { + actualVersion: { value: '16.20.0', privacy: 'public' }, + expectedRange: { value: '>=20.0.0', privacy: 'public' } + }, + remediation: [ + { + descriptionKey: 'remediation.install-supported-node', + documentationUrl: 'https://rushjs.io/pages/maintainer/setup_new_repo/', + automatedExecutionSafety: 'requires-confirmation' + } + ] + } + }, + { + name: 'configuration-invalid-json', + scenario: 'rush.json contains invalid JSON', + expectedResult: 'failed', + diagnostic: { + code: 'RUSH_CONFIG_INVALID_JSON', + category: 'configuration', + summaryKey: 'diagnostic.RUSH_CONFIG_INVALID_JSON.summary', + parameters: { + file: { value: 'rush.json', privacy: 'public' } + }, + remediation: [ + { + descriptionKey: 'remediation.fix-rush-json', + command: 'node common/scripts/install-run-rush.js check', + automatedExecutionSafety: 'safe' + } + ] + } + }, + { + name: 'input-unknown-project', + scenario: 'an invalid --only project is rejected', + expectedResult: 'failed', + diagnostic: { + code: 'RUSH_INPUT_UNKNOWN_PROJECT', + category: 'input', + summaryKey: 'diagnostic.RUSH_INPUT_UNKNOWN_PROJECT.summary', + parameters: { + projectName: { value: '@example/missing', privacy: 'public' } + }, + remediation: [ + { + descriptionKey: 'remediation.list-projects', + command: 'rush list', + automatedExecutionSafety: 'safe' + } + ] + } + }, + { + name: 'dependency-package-manager', + scenario: 'pnpm install exits unsuccessfully', + expectedResult: 'failed', + externalOutput: createExternalOutput( + [ + 'ERR_PNPM_FETCH_401 GET https://registry.example.test/@example/pkg: Unauthorized - 401', + 'Progress: resolved {index}, reused 0, downloaded 0, added 0', + 'The authorization header was rejected while resolving @example/pkg.' + ], + 60 + ), + operationStatus: 'failure', + diagnostic: { + code: 'RUSH_DEPENDENCY_TOOL_FAILED', + category: 'dependency-tool', + summaryKey: 'diagnostic.RUSH_DEPENDENCY_TOOL_FAILED.summary', + parameters: { + command: { value: 'pnpm install', privacy: 'public' }, + exitCode: { value: 1, privacy: 'public' }, + logPath: { value: '/private/install.log', privacy: 'local-sensitive' } + }, + remediation: [ + { + descriptionKey: 'remediation.rush-update-purge', + command: 'rush update --purge', + automatedExecutionSafety: 'requires-confirmation' + } + ] + } + }, + { + name: 'operation-build-failure', + scenario: 'a project build operation fails', + expectedResult: 'failed', + externalOutput: createExternalOutput( + [ + 'src/example-{index}.ts(12,7): error TS2322: Type string is not assignable to type number.', + 'Found 1 error in src/example-{index}.ts', + 'Project @example/app failed during the _phase:build operation.' + ], + 66 + ), + operationStatus: 'failure', + diagnostic: { + code: 'RUSH_OPERATION_FAILED', + category: 'operation', + summaryKey: 'diagnostic.RUSH_OPERATION_FAILED.summary', + parameters: { + projectName: { value: '@example/app', privacy: 'public' } + }, + remediation: [ + { + descriptionKey: 'remediation.rebuild-project', + command: 'rush rebuild --to @example/app', + automatedExecutionSafety: 'safe' + } + ] + } + }, + { + name: 'cache-restore-failure', + scenario: 'a build-cache restore is invalid and requires a local rebuild', + expectedResult: 'failed', + externalOutput: createExternalOutput( + [ + 'Build cache entry cache-entry-42 failed integrity validation for @example/cache-consumer.', + 'Expected archive member lib/index.js but the restored file was missing.', + 'Discarding invalid cache entry and requiring a local rebuild ({index}).' + ], + 30 + ), + operationStatus: 'failure', + diagnostic: { + code: 'RUSH_OPERATION_FAILED', + category: 'operation', + summaryKey: 'diagnostic.RUSH_OPERATION_FAILED.summary', + parameters: { + cacheKey: { value: 'cache-entry-42', privacy: 'public' }, + projectName: { value: '@example/cache-consumer', privacy: 'public' } + }, + remediation: [ + { + descriptionKey: 'remediation.disable-build-cache', + command: 'rush rebuild --to @example/cache-consumer --disable-build-cache', + automatedExecutionSafety: 'safe' + } + ] + } + }, + { + name: 'network-auth-unauthorized', + scenario: 'the registry returns an authentication-shaped failure', + expectedResult: 'failed', + externalOutput: createExternalOutput( + [ + 'GET https://registry.example.test/@example/private returned 401 Unauthorized.', + 'The registry challenge did not include credentials; refresh the configured authentication.', + 'Request attempt {index} failed without exposing an authorization value.' + ], + 24 + ), + diagnostic: { + code: 'RUSH_NETWORK_AUTH_UNAUTHORIZED', + category: 'network-auth', + summaryKey: 'diagnostic.RUSH_NETWORK_AUTH_UNAUTHORIZED.summary', + parameters: { + registryUrl: { value: 'https://registry.example.test/', privacy: 'public' }, + token: { value: CLASSIFIED_SECRET, privacy: 'secret' } + }, + remediation: [ + { + descriptionKey: 'remediation.refresh-registry-auth', + documentationUrl: 'https://rushjs.io/pages/maintainer/npm_registry_auth/', + automatedExecutionSafety: 'requires-confirmation' + } + ] + } + }, + { + name: 'plugin-api-incompatible', + scenario: 'a private plugin is incompatible with the current Rush API', + expectedResult: 'failed', + externalOutput: createExternalOutput( + [ + 'Loading the configured Rush plugin from the repository plugin manifest.', + 'Validating plugin API compatibility before invoking plugin hooks ({index}).', + 'Plugin activation stopped because the declared Rush version range is incompatible.' + ], + 15 + ), + diagnostic: { + code: 'RUSH_PLUGIN_API_INCOMPATIBLE', + category: 'configuration', + summaryKey: 'diagnostic.RUSH_PLUGIN_API_INCOMPATIBLE.summary', + parameters: { + pluginName: { value: PRIVATE_PRODUCER, privacy: 'secret' }, + rushVersion: { value: '5.200.0', privacy: 'public' }, + rushVersionRange: { value: '^5.100.0', privacy: 'public' } + }, + remediation: [ + { + descriptionKey: 'remediation.update-plugin', + command: 'rush update', + automatedExecutionSafety: 'requires-confirmation' + } + ], + privacy: 'local-sensitive', + sourcePackage: PRIVATE_PRODUCER, + sourceComponent: PRIVATE_COMPONENT + } + }, + { + name: 'logical-cancellation', + scenario: 'a command is cancelled and reports an aborted operation', + expectedResult: 'failed', + externalOutput: createExternalOutput( + [ + 'Building @example/app: completed work item {index}.', + 'Cancellation requested; waiting for the active child process to stop.', + 'The _phase:build operation exited before producing final outputs.' + ], + 24 + ), + operationStatus: 'aborted', + diagnostic: { + code: 'RUSH_COMMAND_FAILED', + category: 'operation', + summaryKey: 'diagnostic.RUSH_COMMAND_FAILED.summary', + parameters: { + commandName: { value: 'build', privacy: 'public' }, + reason: { value: 'cancelled', privacy: 'public' } + }, + remediation: [ + { + descriptionKey: 'remediation.retry-command', + command: 'rush build', + automatedExecutionSafety: 'safe' + } + ] + } + }, + { + name: 'internal-unexpected-error', + scenario: 'Rush reports an unexpected internal failure', + expectedResult: 'failed', + externalOutput: createExternalOutput( + [ + 'Unexpected internal failure while finalizing the command graph.', + 'Diagnostic incident incident-42 was recorded for correlation.', + 'See the owner-only full-detail log for stack frame {index}.' + ], + 30 + ), + diagnostic: { + code: 'RUSH_INTERNAL_UNEXPECTED', + category: 'internal', + summaryKey: 'diagnostic.RUSH_INTERNAL_UNEXPECTED.summary', + parameters: { + incident: { value: 'incident-42', privacy: 'public' }, + stack: { value: CLASSIFIED_SECRET, privacy: 'secret' } + }, + remediation: [ + { + descriptionKey: 'remediation.report-rush-bug', + documentationUrl: 'https://github.com/microsoft/rushstack/issues/new/choose', + automatedExecutionSafety: 'unsafe' + } + ] + } + }, + { + name: 'fallback-mixed-privacy', + scenario: 'legacy parser errors include public and local-sensitive fallback messages', + expectedResult: 'failed', + fallbackMessages: [ + { + text: 'The requested command could not be parsed.', + privacy: 'public' + }, + { + text: LOCAL_SENSITIVE_FALLBACK_MESSAGE, + privacy: 'local-sensitive' + } + ] + }, + { + name: 'success-no-warning', + scenario: 'a successful operation emits no warnings', + expectedResult: 'succeeded', + externalOutput: createExternalOutput( + [ + 'Building @example/app source file {index}.', + 'Emitted lib/example-{index}.js and lib/example-{index}.d.ts.', + 'Completed incremental compilation work item {index}.' + ], + 42 + ), + operationStatus: 'success' + }, + { + name: 'success-warning-only', + scenario: 'a successful command emits one bounded warning', + expectedResult: 'succeeded', + externalOutput: createExternalOutput( + [ + 'Restored @example/app output group {index} from the local build cache.', + 'Validated cached output metadata for work item {index}.', + 'The deprecated option warning is represented by a structured diagnostic.' + ], + 24 + ), + operationStatus: 'fromCache', + warningOnly: true, + diagnostic: { + code: 'RUSH_EXTERNAL_TOOL_PROBLEM', + category: 'operation', + summaryKey: 'diagnostic.RUSH_EXTERNAL_TOOL_PROBLEM.summary', + parameters: { + code: { value: 'W42', privacy: 'public' }, + message: { value: 'deprecated option', privacy: 'public' }, + tool: { value: 'fixture-tool', privacy: 'public' } + }, + remediation: [ + { + descriptionKey: 'remediation.remove-deprecated-option', + command: 'rush check', + automatedExecutionSafety: 'safe' + } + ] + } + } +]; + +export function normalizeAiReporterQualificationOutput( + text: string, + logPath: string, + tempRoot: string +): string { + const replacePath = (value: string, machinePath: string, token: string): string => { + const jsonEscapedPath: string = JSON.stringify(machinePath).slice(1, -1); + return value.split(machinePath).join(token).split(jsonEscapedPath).join(token); + }; + return replacePath(replacePath(text, logPath, ''), tempRoot, '').replace( + /\\/g, + '/' + ); +} + +function sha256(text: string): string { + return createHash('sha256').update(text).digest('hex'); +} + +function parseAiOutput(output: string): { + readonly records: readonly Record[]; + readonly valid: boolean; +} { + try { + const records: Record[] = output + .split('\n') + .filter(Boolean) + .map((line) => JSON.parse(line) as Record); + return { records, valid: records.length > 0 && output.endsWith('\n') }; + } catch { + return { records: [], valid: false }; + } +} + +async function runInvocationBudgetGateAsync(tempRoot: string): Promise { + const failedCases: string[] = []; + let maximumBytes: number = 0; + for (const name of ['near-limit-diagnostic', 'watch-early-log', 'watch-late-log']) { + let output: string = ''; + let sequence: number = 0; + const reporter: AiReporter = new AiReporter({ write: (text: string) => (output += text) }); + const emit = (type: IReporterEventEnvelope['type'], payload: unknown): void => { + reporter.report({ + protocolVersion: { major: 1, minor: 1 }, + eventId: `budget-${++sequence}`, + sessionId: 'qualification-budget', + sequence, + timestamp: FIXED_TIMESTAMP, + source: { packageName: '@microsoft/rush-lib', packageVersion: '5.200.0' }, + privacy: 'public', + required: true, + type, + payload + }); + }; + const logPath: string | undefined = + name === 'near-limit-diagnostic' + ? undefined + : path.join(tempRoot, ...Array(name === 'watch-late-log' ? 200 : 1).fill('logs'), 'full.log'); + const emitLog = (): void => + emit('artifactAvailable', { role: 'log', path: logPath, format: 'plaintext', complete: true }); + emit('commandStarted', { commandName: 'build' }); + if (name === 'near-limit-diagnostic') { + emit('diagnosticEmitted', { + diagnosticId: 'root', + code: 'RUSH_OPERATION_FAILED', + category: 'operation', + severity: 'error', + summary: 'x'.repeat(65115) + }); + } else { + if (name === 'watch-early-log') emitLog(); + for (let iterationId: number = 0; iterationId < 1000; iterationId++) { + emit('watchCycleCompleted', { succeeded: true, iterationId }); + } + if (name === 'watch-late-log') emitLog(); + } + emit('commandResult', { succeeded: false, exitCode: 1 }); + await reporter.closeAsync(); + const bytes: number = Buffer.byteLength(output, 'utf8'); + maximumBytes = Math.max(maximumBytes, bytes); + const parsed: ReturnType = parseAiOutput(output); + const final: IAiFinalRecord | undefined = parsed.records.at(-1) as IAiFinalRecord | undefined; + if ( + bytes > AI_REPORTER_QUALIFICATION_THRESHOLDS.maximumOutputBytesPerCase || + !parsed.valid || + final?.kind !== 'ai.final' || + final.result !== 'failed' || + final.exitCode !== 1 || + !Array.isArray(final.diagnostics) || + final.diagnostics.length > 20 || + (logPath !== undefined && (final.log?.path !== logPath || final.log.complete !== true)) + ) { + failedCases.push(name); + } + } + return { + id: 'size.invocation-boundary', + passed: failedCases.length === 0, + actual: maximumBytes, + threshold: + `<= ${AI_REPORTER_QUALIFICATION_THRESHOLDS.maximumOutputBytesPerCase} emitted UTF-8 bytes, ` + + 'including delimiters, with a valid final result and preserved supplied log reference', + failedCases + }; +} + +function createEvents(testCase: ICorpusCase, logPath: string): IReporterEventEnvelope[] { + const events: IReporterEventEnvelope[] = []; + let sequence: number = 0; + const add = ( + type: IReporterEventEnvelope['type'], + payload: unknown, + options: { + readonly privacy?: ReporterPrivacyClassification; + readonly sourcePackage?: string; + readonly sourceComponent?: string; + readonly scope?: IReporterEventEnvelope['scope']; + } = {} + ): void => { + sequence++; + events.push({ + protocolVersion: { major: 1, minor: 1 }, + eventId: `${testCase.name}-event-${sequence}`, + sessionId: `${testCase.name}-session`, + sequence, + timestamp: FIXED_TIMESTAMP, + source: { + packageName: options.sourcePackage ?? '@microsoft/rush-lib', + packageVersion: '5.200.0', + component: options.sourceComponent + }, + scope: options.scope, + privacy: options.privacy ?? 'public', + required: type !== 'activityChanged', + type, + payload + }); + }; + + add('commandStarted', { commandName: 'build' }, { scope: { commandName: 'build' } }); + if (testCase.operationStatus) { + add( + 'operationRegistered', + { operationId: 'fixture#build', projectName: '@example/app', phaseName: '_phase:build' }, + { + scope: { + commandName: 'build', + operationId: 'fixture#build', + projectName: '@example/app', + phaseName: '_phase:build' + } + } + ); + } + if (testCase.externalOutput !== undefined) { + add( + 'externalOutput', + { stream: 'stderr', text: testCase.externalOutput }, + testCase.operationStatus + ? { + privacy: 'local-sensitive', + scope: { + commandName: 'build', + operationId: 'fixture#build', + projectName: '@example/app', + phaseName: '_phase:build' + } + } + : { privacy: 'local-sensitive', scope: { commandName: 'build' } } + ); + } + if (testCase.operationStatus) { + add( + 'operationStatusChanged', + { operationId: 'fixture#build', status: testCase.operationStatus, durationMs: 250 }, + { scope: { commandName: 'build', operationId: 'fixture#build', projectName: '@example/app' } } + ); + add( + 'operationCompleted', + { operationId: 'fixture#build', status: testCase.operationStatus, durationMs: 250 }, + { scope: { commandName: 'build', operationId: 'fixture#build', projectName: '@example/app' } } + ); + } + if (testCase.diagnostic) { + const diagnosticId: string = `${testCase.name}-diagnostic`; + add( + 'diagnosticEmitted', + { + diagnosticId, + code: testCase.diagnostic.code, + category: testCase.diagnostic.category, + severity: testCase.warningOnly ? 'warning' : 'error', + summaryKey: testCase.diagnostic.summaryKey, + parameters: testCase.diagnostic.parameters, + remediation: testCase.diagnostic.remediation, + source: { + kind: 'tool', + toolName: testCase.diagnostic.sourcePackage ?? 'rush' + } + }, + { + privacy: testCase.diagnostic.privacy, + sourcePackage: testCase.diagnostic.sourcePackage, + sourceComponent: testCase.diagnostic.sourceComponent, + scope: { commandName: 'build' } + } + ); + } + for (const message of testCase.fallbackMessages ?? []) { + add( + 'messageEmitted', + { + severity: 'error', + text: message.text + }, + { + privacy: message.privacy, + scope: { commandName: 'build' } + } + ); + } + if (testCase.expectedResult === 'failed') { + add( + 'diagnosticEmitted', + { + diagnosticId: `${testCase.name}-warning`, + code: 'RUSH_EXTERNAL_TOOL_PROBLEM', + category: 'operation', + severity: 'warning', + summaryKey: 'diagnostic.RUSH_EXTERNAL_TOOL_PROBLEM.summary', + parameters: { + tool: { value: 'fixture-tool', privacy: 'public' }, + code: { value: 'W01', privacy: 'public' }, + message: { value: 'secondary warning', privacy: 'public' } + } + }, + { scope: { commandName: 'build' } } + ); + } + add( + 'extension', + { + name: 'qualification.oversized-local-sensitive', + payload: OVERSIZED_LOCAL_SENSITIVE_VALUE.repeat(128) + }, + { + privacy: 'local-sensitive', + sourcePackage: OVERSIZED_LOCAL_SENSITIVE_PRODUCER, + sourceComponent: OVERSIZED_LOCAL_SENSITIVE_COMPONENT, + scope: { + commandName: 'build', + operationId: 'oversized-local-sensitive-operation', + projectName: OVERSIZED_LOCAL_SENSITIVE_SCOPE + } + } + ); + add( + 'extension', + { name: 'private.fixture.secret', payload: { token: CLASSIFIED_SECRET } }, + { + privacy: 'secret', + sourcePackage: CLASSIFIED_SECRET_PRODUCER, + sourceComponent: CLASSIFIED_SECRET_COMPONENT, + scope: { commandName: 'build' } + } + ); + add( + 'artifactAvailable', + { artifactId: `${testCase.name}-log`, role: 'log', path: logPath, format: 'plaintext', complete: true }, + { privacy: 'local-sensitive', scope: { commandName: 'build' } } + ); + add( + 'commandResult', + { + commandName: 'build', + succeeded: testCase.expectedResult === 'succeeded', + exitCode: testCase.expectedResult === 'succeeded' ? 0 : 1 + }, + { scope: { commandName: 'build' } } + ); + add( + 'sessionCompleted', + { exitCode: testCase.expectedResult === 'succeeded' ? 0 : 1 }, + { scope: { commandName: 'build' } } + ); + return events; +} + +function createSecretProjectionProbeEvents(testCase: ICorpusCase): IReporterEventEnvelope[] { + const createProbe = ( + eventId: string, + sequence: number, + type: IReporterEventEnvelope['type'], + payload: unknown, + scope: IReporterEventEnvelope['scope'] = { + commandName: CLASSIFIED_SECRET_COMMAND, + operationId: CLASSIFIED_SECRET_OPERATION, + projectName: CLASSIFIED_SECRET_PROJECT, + phaseName: CLASSIFIED_SECRET_PHASE + } + ): IReporterEventEnvelope => ({ + protocolVersion: { major: 1, minor: 1 }, + eventId, + sessionId: `${testCase.name}-session`, + parentSessionId: CLASSIFIED_SECRET_PARENT_SESSION, + parentOperationId: CLASSIFIED_SECRET_PARENT_OPERATION, + sequence, + sourceSequence: sequence - 10000, + timestamp: FIXED_TIMESTAMP, + source: { + packageName: CLASSIFIED_SECRET_PRODUCER, + packageVersion: '1.0.0', + component: CLASSIFIED_SECRET_COMPONENT + }, + scope, + privacy: 'secret', + required: true, + type, + payload + }); + + return [ + createProbe( + `${testCase.name}-secret-command`, + 10001, + 'commandStarted', + { + commandName: CLASSIFIED_SECRET_COMMAND + }, + { + commandName: CLASSIFIED_SECRET_COMMAND + } + ), + createProbe(`${testCase.name}-secret-operation-registered`, 10002, 'operationRegistered', { + operationId: CLASSIFIED_SECRET_OPERATION, + projectName: CLASSIFIED_SECRET_PROJECT, + phaseName: CLASSIFIED_SECRET_PHASE + }), + createProbe(`${testCase.name}-secret-operation-completed`, 10003, 'operationCompleted', { + operationId: CLASSIFIED_SECRET_OPERATION, + status: 'failure' + }), + createProbe(`${testCase.name}-secret-message`, 10004, 'messageEmitted', { + severity: 'info', + text: CLASSIFIED_SECRET_MESSAGE + }), + createProbe(`${testCase.name}-secret-diagnostic`, 10005, 'diagnosticEmitted', { + diagnosticId: `${testCase.name}-secret-diagnostic`, + code: 'RUSH_INTERNAL_UNEXPECTED', + category: 'internal', + severity: 'info', + summary: CLASSIFIED_SECRET_DIAGNOSTIC + }) + ]; +} + +async function runCaseAsync( + testCase: ICorpusCase, + caseDirectory: string, + tempRoot: string +): Promise { + let aiOutput: string = ''; + let jsonOutput: string = ''; + let plaintextOutput: string = ''; + let legacyOutput: string = ''; + const fileReporter: FileReporter = new FileReporter({ + commonTempFolder: caseDirectory, + actionName: testCase.name, + pid: FIXED_PID, + nowMs: () => FIXED_TIME_MS + }); + const aiReporter: AiReporter = new AiReporter({ write: (text: string) => (aiOutput += text) }); + const jsonReporter: JsonReporter = new JsonReporter({ + write: (text: string) => (jsonOutput += text), + maxRecordBytes: 1024 + }); + const plaintextReporter: PlaintextReporter = new PlaintextReporter({ + write: (text: string) => (plaintextOutput += text), + variant: 'detailed', + color: false, + nowMs: () => FIXED_TIME_MS + }); + const legacyReporter: LegacyReporter = new LegacyReporter({ + write: (text: string) => (legacyOutput += text), + maxParallelism: 4 + }); + + await fileReporter.initializeAsync(); + const logPath: string = fileReporter.getArtifact().path!; + const events: readonly IReporterEventEnvelope[] = createEvents(testCase, logPath); + for (const event of events) { + fileReporter.report(event); + aiReporter.report(event); + jsonReporter.report(event); + plaintextReporter.report(event); + legacyReporter.report(event); + } + for (const event of createSecretProjectionProbeEvents(testCase)) { + // Operation grouping is a separate file-sidecar policy. Exercise shared + // file redaction with records that do not enter the grouping path. + if (event.type === 'messageEmitted' || event.type === 'diagnosticEmitted') { + fileReporter.report(event); + } + aiReporter.report(event); + jsonReporter.report(event); + } + await fileReporter.closeAsync(); + await aiReporter.closeAsync(); + await jsonReporter.closeAsync(); + await plaintextReporter.closeAsync(); + await legacyReporter.closeAsync(); + + const normalizedAiOutput: string = normalizeAiReporterQualificationOutput(aiOutput, logPath, tempRoot); + const normalizedPlaintextOutput: string = normalizeAiReporterQualificationOutput( + plaintextOutput, + logPath, + tempRoot + ); + const normalizedLegacyOutput: string = normalizeAiReporterQualificationOutput( + legacyOutput, + logPath, + tempRoot + ); + const parsedAi: { + readonly records: readonly Record[]; + readonly valid: boolean; + } = parseAiOutput(aiOutput); + const parsedJson: { + readonly records: readonly Record[]; + readonly valid: boolean; + } = parseAiOutput(jsonOutput); + const final: IAiFinalRecord | undefined = parsedAi.records.at(-1) as IAiFinalRecord | undefined; + const diagnostic: ICorpusDiagnostic | undefined = testCase.diagnostic; + const matchingDiagnostic: IAiDiagnostic | undefined = diagnostic + ? final?.diagnostics.find(({ code }) => code === diagnostic.code) + : undefined; + const fallbackMessages: readonly { + readonly text: string; + readonly privacy: ReporterPrivacyClassification; + }[] = testCase.fallbackMessages ?? []; + const publicFallbackMessages: readonly string[] = fallbackMessages + .filter(({ privacy }) => privacy === 'public') + .map(({ text }) => text); + const expectedFallbackDiagnostic: ICorpusDiagnostic = { + code: 'RUSH_COMMAND_FAILED', + category: 'command', + summaryKey: 'diagnostic.RUSH_COMMAND_FAILED.summary', + parameters: { commandName: { value: 'build', privacy: 'public' } }, + remediation: [ + { + descriptionKey: 'remediation.review-command-usage', + command: 'rush build --help', + automatedExecutionSafety: 'safe' + } + ] + }; + const actionable: boolean = + testCase.expectedResult === 'succeeded' + ? true + : diagnostic + ? Boolean( + final?.result === 'failed' && + final.errorCodes.includes(diagnostic.code) && + hasExpectedAiQualificationDiagnostic(matchingDiagnostic, diagnostic) + ) + : Boolean( + fallbackMessages.length > 0 && + final?.result === 'failed' && + final.errorCodes.includes('RUSH_COMMAND_FAILED') && + final.errorCount === fallbackMessages.length && + final.diagnosticCategoryCounts.command === fallbackMessages.length && + final.diagnostics.length === publicFallbackMessages.length && + final.diagnostics.every( + (projected, index) => + projected.severity === 'error' && + projected.summary === publicFallbackMessages[index] && + hasExpectedAiQualificationDiagnostic(projected, expectedFallbackDiagnostic) + ) && + final.truncated + ); + + const artifact: IFileReporterArtifact = fileReporter.getArtifact(); + const aiLogPath: string | undefined = final?.log?.path; + const logExists: boolean = aiLogPath !== undefined && fs.existsSync(aiLogPath); + const logContent: string = + logExists && aiLogPath !== undefined ? await fs.promises.readFile(aiLogPath, 'utf8') : ''; + const ownerOnly: boolean = + process.platform === 'win32' || + (logExists && aiLogPath !== undefined && (await fs.promises.stat(aiLogPath)).mode % 0o1000 === 0o600); + const failureCorrelated: boolean = + testCase.expectedResult === 'succeeded' || + (diagnostic + ? logContent.includes(diagnostic.code) && + logContent.includes(`${testCase.name}-diagnostic`) && + logContent.includes(`${testCase.name}-session`) + : fallbackMessages.length > 0 && + fallbackMessages.every(({ text }) => logContent.includes(text)) && + logContent.includes(`${testCase.name}-session`)); + const localSensitiveProducerPreserved: boolean = + diagnostic?.sourcePackage === undefined || + (logContent.includes(diagnostic.sourcePackage) && + (diagnostic.sourceComponent === undefined || logContent.includes(diagnostic.sourceComponent))); + const fullLogValid: boolean = Boolean( + final?.log && + final.log.path === artifact.path && + final.log.format === 'plaintext' && + final.log.complete === artifact.complete && + artifact.available && + artifact.complete && + aiLogPath && + path.isAbsolute(aiLogPath) && + logExists && + ownerOnly && + logContent.includes('"type":"commandStarted"') && + logContent.includes('"type":"commandResult"') && + logContent.includes('"type":"sessionCompleted"') && + (testCase.externalOutput === undefined || logContent.includes(testCase.externalOutput.trim())) && + failureCorrelated && + localSensitiveProducerPreserved && + logContent.includes(OVERSIZED_LOCAL_SENSITIVE_VALUE) && + logContent.includes(OVERSIZED_LOCAL_SENSITIVE_PRODUCER) && + logContent.includes(OVERSIZED_LOCAL_SENSITIVE_COMPONENT) && + logContent.includes(OVERSIZED_LOCAL_SENSITIVE_SCOPE) + ); + const oversizedMarker: Record | undefined = parsedJson.records.find( + ({ payload }) => + ( + payload as + | { readonly name?: string; readonly payload?: { readonly originalType?: string } } + | undefined + )?.name === 'rush.reporter.record-too-large' && + (payload as { readonly payload?: { readonly originalType?: string } }).payload?.originalType === + 'extension' + ); + const oversizedMarkerValid: boolean = + oversizedMarker?.privacy === 'local-sensitive' && + oversizedMarker.scope === undefined && + (oversizedMarker.source as { readonly packageName?: string; readonly packageVersion?: string }) + ?.packageName === '[private-producer]' && + (oversizedMarker.source as { readonly packageName?: string; readonly packageVersion?: string }) + ?.packageVersion === '[private-version]'; + const machinePresentedOutput: string = `${aiOutput}\n${jsonOutput}`; + const humanPresentedOutput: string = `${plaintextOutput}\n${legacyOutput}`; + const allLocalOutput: string = `${machinePresentedOutput}\n${humanPresentedOutput}\n${logContent}`; + const privacySafe: boolean = + !allLocalOutput.includes(CLASSIFIED_SECRET) && + !allLocalOutput.includes(CLASSIFIED_SECRET_PRODUCER) && + !allLocalOutput.includes(CLASSIFIED_SECRET_COMPONENT) && + !allLocalOutput.includes(CLASSIFIED_SECRET_COMMAND) && + !allLocalOutput.includes(CLASSIFIED_SECRET_OPERATION) && + !allLocalOutput.includes(CLASSIFIED_SECRET_PROJECT) && + !allLocalOutput.includes(CLASSIFIED_SECRET_PHASE) && + !allLocalOutput.includes(CLASSIFIED_SECRET_PARENT_SESSION) && + !allLocalOutput.includes(CLASSIFIED_SECRET_PARENT_OPERATION) && + !allLocalOutput.includes(CLASSIFIED_SECRET_MESSAGE) && + !allLocalOutput.includes(CLASSIFIED_SECRET_DIAGNOSTIC) && + !machinePresentedOutput.includes(LOCAL_SENSITIVE_FALLBACK_MESSAGE) && + !machinePresentedOutput.includes(OVERSIZED_LOCAL_SENSITIVE_VALUE) && + !machinePresentedOutput.includes(OVERSIZED_LOCAL_SENSITIVE_PRODUCER) && + !machinePresentedOutput.includes(OVERSIZED_LOCAL_SENSITIVE_COMPONENT) && + !machinePresentedOutput.includes(OVERSIZED_LOCAL_SENSITIVE_SCOPE) && + !aiOutput.includes(PRIVATE_PRODUCER) && + !aiOutput.includes(PRIVATE_COMPONENT) && + !humanPresentedOutput.includes(PRIVATE_PRODUCER) && + !humanPresentedOutput.includes(PRIVATE_COMPONENT); + const warningContractValid: boolean = + testCase.expectedResult === 'failed' + ? final?.warningCount === 1 && final.diagnostics.every(({ severity }) => severity === 'error') + : testCase.warningOnly + ? final?.warningCount === 1 && final.diagnostics.some(({ severity }) => severity === 'warning') + : final?.warningCount === 0; + const failures: string[] = []; + if (!actionable) failures.push('missing stable code/category/context/remediation'); + if (!privacySafe) failures.push('classified secret or private producer identity leaked'); + if (!fullLogValid) failures.push('full log path, permissions, completeness, or correlation invalid'); + if (!parsedAi.valid || !parsedJson.valid || !oversizedMarkerValid) { + failures.push('machine stdout or oversized-record marker contract regressed'); + } + if (!warningContractValid) failures.push('warning suppression/detail contract regressed'); + + return { + normalizedAiOutput, + normalizedPlaintextOutput, + normalizedLegacyOutput, + result: { + name: testCase.name, + scenario: testCase.scenario, + expectedResult: testCase.expectedResult, + aiOutputBytes: Buffer.byteLength(aiOutput, 'utf8'), + plaintextOutputBytes: Buffer.byteLength(plaintextOutput, 'utf8'), + legacyOutputBytes: Buffer.byteLength(legacyOutput, 'utf8'), + actionable, + privacySafe, + fullLogValid, + stdoutContractValid: parsedAi.valid && parsedJson.valid && oversizedMarkerValid, + warningContractValid, + failures + } + }; +} + +/** + * Runs the deterministic, network-free AI reporter qualification corpus. + * + * @remarks + * External package-manager, cache, registry, plugin, cancellation, and internal + * failures are represented by stable canonical event fixtures. Each case runs + * through AI, detailed plaintext, legacy, and full-detail file reporters three + * times. Machine-specific paths are normalized before hashing and are never + * stored in the returned result. + * + * @beta + */ +export async function runAiReporterQualificationCorpusAsync(): Promise { + const tempRoot: string = await fs.promises.mkdtemp( + path.join(os.tmpdir(), 'rush-ai-reporter-qualification-') + ); + try { + const runs: ICaseRun[][] = []; + for ( + let runIndex: number = 0; + runIndex < AI_REPORTER_QUALIFICATION_THRESHOLDS.deterministicRunCount; + runIndex++ + ) { + const runDirectory: string = path.join(tempRoot, `run-${runIndex}`); + await fs.promises.mkdir(runDirectory); + const run: ICaseRun[] = []; + for (const testCase of CORPUS) { + const caseDirectory: string = path.join(runDirectory, testCase.name); + await fs.promises.mkdir(caseDirectory); + run.push(await runCaseAsync(testCase, caseDirectory, tempRoot)); + } + runs.push(run); + } + + const results: IAiReporterQualificationCaseResult[] = runs[0].map( + (firstRun: ICaseRun, caseIndex: number) => { + const normalizedOutputs: readonly string[] = runs.map( + (run: readonly ICaseRun[]) => run[caseIndex].normalizedAiOutput + ); + const deterministic: boolean = normalizedOutputs.every( + (output: string) => output === normalizedOutputs[0] + ); + const failures: string[] = [...firstRun.result.failures]; + if (!deterministic) { + failures.push('normalized AI output differed across repeated runs'); + } + return { + ...firstRun.result, + deterministic, + normalizedAiOutputSha256: sha256(firstRun.normalizedAiOutput), + failures + }; + } + ); + const qualification: IAiReporterQualificationResult = evaluateAiReporterQualification(results); + const invocationBudget: IAiReporterQualificationGateResult = await runInvocationBudgetGateAsync(tempRoot); + return { + ...qualification, + passed: qualification.passed && invocationBudget.passed, + gates: [...qualification.gates, invocationBudget] + }; + } finally { + await fs.promises.rm(tempRoot, { recursive: true, force: true }); + } +} diff --git a/libraries/reporter/src/reporters/AiReporter.ts b/libraries/reporter/src/reporters/AiReporter.ts index 818be5c1fd..a3c0995caf 100644 --- a/libraries/reporter/src/reporters/AiReporter.ts +++ b/libraries/reporter/src/reporters/AiReporter.ts @@ -3,8 +3,10 @@ import type { IReporterProtocolVersion } from '../events/ReporterProtocolVersion'; import type { IReporterEventEnvelope } from '../events/IReporterEventEnvelope'; +import type { ReporterJsonValue } from '../events/ReporterJsonValue'; import type { IReporter } from '../manager/IReporter'; import type { IRushRemediationAction } from '../diagnostics/IRushRemediationAction'; +import type { IClassifiedDiagnosticValue } from '../diagnostics/IClassifiedDiagnosticValue'; import { REPORTER_PERFORMANCE_BUDGETS } from '../perf/PerformanceBudgets'; import { REPORTER_PROTOCOL_VERSION } from '../protocol/ReporterProtocol'; @@ -21,10 +23,12 @@ const TERMINAL_STATUSES: ReadonlySet = new Set([ ]); interface IAiDiagnosticState { - readonly errorDiagnostics: IAiDiagnostic[]; - readonly warningDiagnostics: IAiDiagnostic[]; + readonly errorDiagnostics: ICollectedAiDiagnostic[]; + readonly warningDiagnostics: ICollectedAiDiagnostic[]; readonly errorCodes: Set; readonly diagnosticCategoryCounts: { [category: string]: number }; + suppressedSecretErrorCount: number; + suppressedSecretWarningCount: number; errorDiagnosticsTruncated: boolean; warningDiagnosticsTruncated: boolean; errorCount: number; @@ -41,12 +45,19 @@ interface IAiWatchCycleState { watchCompleted: boolean; } +interface IPendingAiProgress { + readonly kind: 'ai.status' | 'ai.watchCycle'; + readonly line: string; +} + function createDiagnosticState(): IAiDiagnosticState { return { errorDiagnostics: [], warningDiagnostics: [], errorCodes: new Set(), diagnosticCategoryCounts: {}, + suppressedSecretErrorCount: 0, + suppressedSecretWarningCount: 0, errorDiagnosticsTruncated: false, warningDiagnosticsTruncated: false, errorCount: 0, @@ -60,13 +71,24 @@ function createDiagnosticState(): IAiDiagnosticState { * @beta */ export interface IAiDiagnostic { + readonly diagnosticId?: string; readonly code: string; readonly category: string; readonly severity: string; readonly summary?: string; + /** + * A nonstandard summary key. The standard `diagnostic.${code}.summary` key is implicit. + */ + readonly summaryKey?: string; + readonly detailKey?: string; + readonly context?: Readonly>; readonly remediation?: readonly IRushRemediationAction[]; } +interface ICollectedAiDiagnostic extends IAiDiagnostic { + readonly causeDiagnosticIds?: readonly string[]; +} + /** * The AI reporter's log reference. * @@ -111,7 +133,8 @@ export interface IAiReporterOptions { readonly write: (text: string) => void; /** - * The maximum size of the final record in bytes. Defaults to 64 KiB. + * The maximum UTF-8 size of the entire invocation, including NDJSON delimiters. + * Defaults to 64 KiB. */ readonly maxBytes?: number; @@ -156,6 +179,10 @@ export class AiReporter implements IReporter { private _pendingResult: { succeeded: boolean; exitCode: number } | undefined; private _legacyIterationId: number; private _latestIterationId: number; + private readonly _pendingProgress: IPendingAiProgress[] = []; + private _pendingProgressBytes: number = 0; + private _writtenBytes: number = 0; + private _progressTruncated: boolean = false; public constructor(options: IAiReporterOptions) { this._write = options.write; @@ -190,17 +217,33 @@ export class AiReporter implements IReporter { } public report(event: IReporterEventEnvelope): void { + if (this._finalEmitted) { + return; + } this._protocolVersion = event.protocolVersion; + if (event.privacy === 'secret') { + switch (event.type) { + case 'diagnosticEmitted': + case 'messageEmitted': + case 'commandResult': + case 'sessionCompleted': + break; + default: + return; + } + } + if ( + this._logPath !== undefined && + event.type !== 'artifactAvailable' && + event.type !== 'commandResult' && + event.type !== 'sessionCompleted' + ) { + this._flushPendingProgress(); + } switch (event.type) { case 'commandStarted': { this._commandName = (event.payload as { commandName: string }).commandName; - this._write( - `${JSON.stringify({ - kind: 'ai.status', - protocolVersion: this._protocolVersion, - commandName: this._commandName - })}\n` - ); + this._writeProgressRecord({ kind: 'ai.status', protocolVersion: this._protocolVersion }); break; } case 'operationRegistered': { @@ -255,11 +298,11 @@ export class AiReporter implements IReporter { break; } case 'diagnosticEmitted': { - const diagnostic: IAiDiagnostic & { iterationId?: number } = event.payload as IAiDiagnostic & { - iterationId?: number; + const diagnostic: { readonly iterationId?: number } = event.payload as { + readonly iterationId?: number; }; this._collectDiagnostic( - diagnostic, + event, diagnostic.iterationId === undefined ? this._globalDiagnostics : this._getWatchCycle(diagnostic.iterationId).diagnostics @@ -273,15 +316,13 @@ export class AiReporter implements IReporter { }; const cycle: IAiWatchCycleState = this._getWatchCycle(payload.iterationId); const succeeded: boolean = payload.succeeded === true; - this._write( - `${JSON.stringify({ - kind: 'ai.watchCycle', - protocolVersion: this._protocolVersion, - succeeded, - operationCounts: { ...cycle.operationCounts }, - failedProjects: [...cycle.failedProjects] - })}\n` - ); + this._writeProgressRecord({ + kind: 'ai.watchCycle', + protocolVersion: this._protocolVersion, + succeeded, + operationCounts: { ...cycle.operationCounts }, + failedProjects: [...cycle.failedProjects] + }); if (payload.iterationId === undefined) { this._legacyIterationId++; } @@ -338,7 +379,9 @@ export class AiReporter implements IReporter { } public async flushAsync(): Promise { - /* no-op */ + if (this._logPath !== undefined) { + this._flushPendingProgress(); + } } public async closeAsync(): Promise { @@ -351,35 +394,51 @@ export class AiReporter implements IReporter { } } - private _collectDiagnostic(diagnostic: IAiDiagnostic, state: IAiDiagnosticState): void { + private _collectDiagnostic(event: IReporterEventEnvelope, state: IAiDiagnosticState): void { + const diagnostic: IAiDiagnostic & { + readonly causeDiagnosticIds?: readonly string[]; + readonly parameters?: Readonly>; + } = event.payload as IAiDiagnostic & { + readonly causeDiagnosticIds?: readonly string[]; + readonly parameters?: Readonly>; + }; + if (event.privacy === 'secret') { + if (diagnostic.severity === 'error') { + state.suppressedSecretErrorCount++; + } else if (diagnostic.severity === 'warning') { + state.suppressedSecretWarningCount++; + } + return; + } if (diagnostic.category !== undefined) { state.diagnosticCategoryCounts[diagnostic.category] = (state.diagnosticCategoryCounts[diagnostic.category] ?? 0) + 1; } + const collected: ICollectedAiDiagnostic = { + diagnosticId: diagnostic.diagnosticId, + code: diagnostic.code, + category: diagnostic.category, + severity: diagnostic.severity, + summary: diagnostic.summary, + summaryKey: + diagnostic.summaryKey === `diagnostic.${diagnostic.code}.summary` ? undefined : diagnostic.summaryKey, + detailKey: diagnostic.detailKey, + context: this._projectDiagnosticContext(diagnostic.parameters), + remediation: diagnostic.remediation, + causeDiagnosticIds: diagnostic.causeDiagnosticIds + }; if (diagnostic.severity === 'error') { state.errorCount++; state.errorCodes.add(diagnostic.code); if (state.errorDiagnostics.length < this._maxDetailedDiagnostics) { - state.errorDiagnostics.push({ - code: diagnostic.code, - category: diagnostic.category, - severity: 'error', - summary: diagnostic.summary, - remediation: diagnostic.remediation - }); + state.errorDiagnostics.push(collected); } else { state.errorDiagnosticsTruncated = true; } } else if (diagnostic.severity === 'warning') { state.warningCount++; if (state.warningDiagnostics.length < this._maxDetailedDiagnostics) { - state.warningDiagnostics.push({ - code: diagnostic.code, - category: diagnostic.category, - severity: 'warning', - summary: diagnostic.summary, - remediation: diagnostic.remediation - }); + state.warningDiagnostics.push(collected); } else { state.warningDiagnosticsTruncated = true; } @@ -418,6 +477,137 @@ export class AiReporter implements IReporter { } } + private _projectDiagnosticContext( + parameters: Readonly> | undefined + ): Readonly> | undefined { + if (!parameters) { + return undefined; + } + const context: Record = {}; + for (const name of Object.keys(parameters).sort()) { + const parameter: IClassifiedDiagnosticValue = parameters[name]; + context[name] = parameter.privacy === 'public' ? parameter.value : (`[${parameter.privacy}]` as const); + } + return Object.keys(context).length > 0 ? context : undefined; + } + + private _orderDiagnostics(diagnostics: readonly ICollectedAiDiagnostic[]): IAiDiagnostic[] { + const byId: Map = new Map(); + for (const diagnostic of diagnostics) { + if (diagnostic.diagnosticId) { + byId.set(diagnostic.diagnosticId, diagnostic); + } + } + + const ordered: IAiDiagnostic[] = []; + const visited: Set = new Set(); + const visiting: Set = new Set(); + const visit = (diagnostic: ICollectedAiDiagnostic): void => { + if (visited.has(diagnostic) || visiting.has(diagnostic)) { + return; + } + visiting.add(diagnostic); + for (const causeId of diagnostic.causeDiagnosticIds ?? []) { + const cause: ICollectedAiDiagnostic | undefined = byId.get(causeId); + if (cause) { + visit(cause); + } + } + visiting.delete(diagnostic); + visited.add(diagnostic); + ordered.push({ + diagnosticId: diagnostic.diagnosticId, + code: diagnostic.code, + category: diagnostic.category, + severity: diagnostic.severity, + summary: diagnostic.summary, + summaryKey: diagnostic.summaryKey, + detailKey: diagnostic.detailKey, + context: diagnostic.context, + remediation: diagnostic.remediation + }); + }; + + for (const diagnostic of diagnostics) { + visit(diagnostic); + } + return ordered; + } + + private _getFinalReserveBytes(): number { + const minimal: IAiFinalRecord = { + kind: 'ai.final', + protocolVersion: this._protocolVersion, + result: 'succeeded', + exitCode: Number.MIN_SAFE_INTEGER, + scope: { failedProjects: [] }, + errorCodes: [], + diagnosticCategoryCounts: {}, + diagnostics: [], + errorCount: Number.MAX_SAFE_INTEGER, + warningCount: Number.MAX_SAFE_INTEGER, + operationCounts: {}, + ...(this._logPath === undefined + ? {} + : { log: { path: this._logPath, format: this._logFormat, complete: false } }), + truncated: true + }; + return Math.max( + Math.min(this._maxBytes, Math.max(MIN_AI_MAX_BYTES, Math.floor(this._maxBytes / 2))), + Buffer.byteLength(JSON.stringify(minimal), 'utf8') + 1 + ); + } + + private _writeProgressRecord( + record: Readonly> & { readonly kind: IPendingAiProgress['kind'] } + ): void { + const line: string = `${JSON.stringify(record)}\n`; + if (this._logPath === undefined) { + // Delay progress until the log reservation is known; the pending queue is itself byte-bounded. + const bytes: number = Buffer.byteLength(line, 'utf8'); + if (this._pendingProgressBytes + bytes <= this._maxBytes) { + this._pendingProgress.push({ kind: record.kind, line }); + this._pendingProgressBytes += bytes; + } else { + this._progressTruncated = true; + } + } else { + this._writeProgressLine(line); + } + } + + private _flushPendingProgress(): void { + const pending: IPendingAiProgress[] = this._pendingProgress.splice(0); + this._pendingProgressBytes = 0; + for (const { kind, line } of pending) { + // A completed result supersedes an unrendered start acknowledgement, not watch history. + if (kind === 'ai.status' && (this._pendingResult !== undefined || this._finalEmitted)) { + continue; + } + this._writeProgressLine(line); + } + } + + private _writeProgressLine(line: string): void { + if ( + this._writtenBytes + Buffer.byteLength(line, 'utf8') + this._getFinalReserveBytes() <= + this._maxBytes + ) { + this._writeLine(line); + } else { + this._progressTruncated = true; + } + } + + private _writeLine(line: string): void { + const bytes: number = Buffer.byteLength(line, 'utf8'); + if (this._writtenBytes + bytes > this._maxBytes) { + throw new Error(`AI reporter output exceeds the invocation budget of ${this._maxBytes} bytes.`); + } + this._writtenBytes += bytes; + this._write(line); + } + private _emitFinal(succeeded: boolean, exitCode: number): void { if (this._finalEmitted) { return; @@ -428,29 +618,51 @@ export class AiReporter implements IReporter { const cycleDiagnostics: IAiDiagnosticState = cycle.diagnostics; const errorCountWithoutFallback: number = this._globalDiagnostics.errorCount + cycleDiagnostics.errorCount; - const warningCount: number = this._globalDiagnostics.warningCount + cycleDiagnostics.warningCount; - const collectedErrorDiagnostics: IAiDiagnostic[] = [ + const suppressedSecretErrorCount: number = + this._globalDiagnostics.suppressedSecretErrorCount + cycleDiagnostics.suppressedSecretErrorCount; + const suppressedSecretWarningCount: number = + this._globalDiagnostics.suppressedSecretWarningCount + cycleDiagnostics.suppressedSecretWarningCount; + const warningCount: number = + this._globalDiagnostics.warningCount + cycleDiagnostics.warningCount + suppressedSecretWarningCount; + const collectedErrorDiagnostics: IAiDiagnostic[] = this._orderDiagnostics([ ...this._globalDiagnostics.errorDiagnostics, ...cycleDiagnostics.errorDiagnostics - ]; - const collectedWarningDiagnostics: IAiDiagnostic[] = [ + ]); + const collectedWarningDiagnostics: IAiDiagnostic[] = this._orderDiagnostics([ ...this._globalDiagnostics.warningDiagnostics, ...cycleDiagnostics.warningDiagnostics - ]; + ]); const fallbackDiagnostics: IAiDiagnostic[] = errorCountWithoutFallback === 0 - ? this._fallbackErrorMessages.map((summary) => ({ - code: 'RUSH_COMMAND_FAILED', - category: 'command', - severity: 'error', - summary - })) + ? this._fallbackErrorMessages.map( + (summary: string): IAiDiagnostic => ({ + code: 'RUSH_COMMAND_FAILED', + category: 'command', + severity: 'error', + summary, + context: + this._commandName === undefined ? { tool: 'rush' } : { commandName: this._commandName }, + remediation: [ + { + descriptionKey: 'remediation.review-command-usage', + command: + this._commandName !== undefined && /^[a-z][a-z0-9:-]*$/i.test(this._commandName) + ? `rush ${this._commandName} --help` + : 'rush --help', + automatedExecutionSafety: 'safe' + } + ] + }) + ) : []; const hasFallbackErrors: boolean = errorCountWithoutFallback === 0 && this._fallbackErrorCount > 0; const errorDiagnostics: IAiDiagnostic[] = hasFallbackErrors ? fallbackDiagnostics : collectedErrorDiagnostics; - const errorCount: number = errorCountWithoutFallback + (hasFallbackErrors ? this._fallbackErrorCount : 0); + const errorCount: number = + errorCountWithoutFallback + + suppressedSecretErrorCount + + (hasFallbackErrors ? this._fallbackErrorCount : 0); const errorCodes: string[] = hasFallbackErrors ? ['RUSH_COMMAND_FAILED'] : [...new Set([...this._globalDiagnostics.errorCodes, ...cycleDiagnostics.errorCodes])].sort(); @@ -497,14 +709,22 @@ export class AiReporter implements IReporter { truncated: hasFailures ? this._globalDiagnostics.errorDiagnosticsTruncated || cycleDiagnostics.errorDiagnosticsTruncated || + suppressedSecretErrorCount > 0 || (hasFallbackErrors && this._fallbackErrorsTruncated) - : this._globalDiagnostics.warningDiagnosticsTruncated || cycleDiagnostics.warningDiagnosticsTruncated + : this._globalDiagnostics.warningDiagnosticsTruncated || + cycleDiagnostics.warningDiagnosticsTruncated || + suppressedSecretWarningCount > 0 }; if (this._logPath !== undefined) { record.log = { path: this._logPath, format: this._logFormat, complete: this._artifactComplete }; } + this._flushPendingProgress(); + record.truncated ||= this._progressTruncated; + const finalBudgetBytes: number = this._maxBytes - this._writtenBytes; + const serializedBytes = (): number => Buffer.byteLength(JSON.stringify(record), 'utf8') + 1; + // Enforce the byte cap by progressively trimming detailed diagnostics, then // error codes, then failed projects, so the record always fits the budget. const trimTargets: Array<{ get: () => unknown[]; set: (value: unknown[]) => void }> = [ @@ -519,29 +739,33 @@ export class AiReporter implements IReporter { } ]; for (const target of trimTargets) { - while (Buffer.byteLength(JSON.stringify(record), 'utf8') > this._maxBytes && target.get().length > 0) { + while (serializedBytes() > finalBudgetBytes && target.get().length > 0) { target.set(target.get().slice(0, target.get().length - 1)); record.truncated = true; } - if (Buffer.byteLength(JSON.stringify(record), 'utf8') <= this._maxBytes) { + if (serializedBytes() <= finalBudgetBytes) { break; } } let serialized: string = JSON.stringify(record); - if (Buffer.byteLength(serialized, 'utf8') > this._maxBytes) { + if (Buffer.byteLength(serialized, 'utf8') + 1 > finalBudgetBytes) { record.scope = { failedProjects: [] }; record.errorCodes = []; record.diagnosticCategoryCounts = {}; record.diagnostics = []; record.operationCounts = {}; - delete record.log; record.truncated = true; serialized = JSON.stringify(record); } - if (Buffer.byteLength(serialized, 'utf8') > this._maxBytes) { + if (Buffer.byteLength(serialized, 'utf8') + 1 > finalBudgetBytes && record.log !== undefined) { + // This can only fit by omitting an intrinsically oversized reference, even with no progress output. + delete record.log; + serialized = JSON.stringify(record); + } + if (Buffer.byteLength(serialized, 'utf8') + 1 > finalBudgetBytes) { throw new Error(`The minimal AI final record exceeds maxBytes=${this._maxBytes}`); } - this._write(`${serialized}\n`); + this._writeLine(`${serialized}\n`); } } diff --git a/libraries/reporter/src/reporters/ReporterRedaction.ts b/libraries/reporter/src/reporters/ReporterRedaction.ts index 37bc0d2866..5fe52c87a5 100644 --- a/libraries/reporter/src/reporters/ReporterRedaction.ts +++ b/libraries/reporter/src/reporters/ReporterRedaction.ts @@ -37,19 +37,28 @@ export function redactReporterEvent(event: IReporterEventEnvelope): IRe } let payload: unknown = event.payload; + const source: IReporterEventEnvelope['source'] = event.source; if (event.type === 'diagnosticEmitted') { - const diagnostic: { readonly parameters?: Readonly> } = - event.payload as { - readonly parameters?: Readonly>; - }; + const diagnostic: { + readonly parameters?: Readonly>; + readonly source?: unknown; + } = event.payload as { + readonly parameters?: Readonly>; + readonly source?: unknown; + }; + const redactedDiagnostic: { + parameters?: Record; + source?: unknown; + } = { ...diagnostic }; if (diagnostic.parameters) { const parameters: Record = {}; for (const [name, classified] of Object.entries(diagnostic.parameters)) { parameters[name] = classified.privacy === 'secret' ? { value: '[secret]', privacy: 'secret' } : classified; } - payload = { ...diagnostic, parameters }; + redactedDiagnostic.parameters = parameters; } + payload = redactedDiagnostic; } - return { ...event, payload }; + return { ...event, source, payload }; } diff --git a/libraries/reporter/src/test/AiReporterQualification.test.ts b/libraries/reporter/src/test/AiReporterQualification.test.ts new file mode 100644 index 0000000000..0ce23650f1 --- /dev/null +++ b/libraries/reporter/src/test/AiReporterQualification.test.ts @@ -0,0 +1,316 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { + AiReporter, + AI_REPORTER_QUALIFICATION_THRESHOLDS, + evaluateAiReporterQualification, + formatAiReporterQualificationFailures, + getQualifiedAiReporterDecision, + runAiReporterQualificationCorpusAsync, + type IAiDiagnostic, + type IReporterEventEnvelope, + type IAiReporterQualificationCaseResult, + type IAiReporterQualificationGateResult, + type IAiReporterQualificationResult +} from '../index'; +import { + hasExpectedAiQualificationDiagnostic, + normalizeAiReporterQualificationOutput +} from '../qualification/AiReporterQualificationCorpus'; + +describe('AI reporter deterministic qualification corpus', () => { + let qualification: IAiReporterQualificationResult; + + // Three file-backed corpus passes can exceed Jest's default setup allowance on Windows CI. + beforeAll(async () => { + qualification = await runAiReporterQualificationCorpusAsync(); + }, 15000); + + it('passes every blocking gate with machine-readable safe results', () => { + if (!qualification.passed) { + throw new Error(formatAiReporterQualificationFailures(qualification)); + } + expect(qualification.schemaVersion).toBe('1.0'); + expect(qualification.cases).toHaveLength(13); + expect(qualification.cases.filter(({ expectedResult }) => expectedResult === 'failed')).toHaveLength(11); + expect(qualification.cases.every(({ failures }) => failures.length === 0)).toBe(true); + const serialized: string = JSON.stringify(qualification); + expect(serialized).not.toContain('rush-ai-reporter-qualification-'); + expect(serialized).not.toContain('qualification-fake-secret-token'); + expect(serialized).not.toContain('qualification-secret-command'); + expect(serialized).not.toContain('qualification-secret-operation'); + expect(serialized).not.toContain('@private/qualification-secret-project'); + expect(serialized).not.toContain('qualification-secret-phase'); + expect(serialized).not.toContain('qualification-secret-parent-session'); + expect(serialized).not.toContain('qualification-secret-parent-operation'); + expect(serialized).not.toContain('qualification-secret-message-text'); + expect(serialized).not.toContain('qualification-secret-diagnostic-summary'); + expect(serialized).not.toContain('qualification-local-sensitive-fallback-message'); + expect(serialized).not.toContain('qualification-oversized-local-sensitive-value'); + expect(serialized).not.toContain('@private/oversized-qualification-fixture'); + expect(serialized).not.toContain('@private/example-rush-plugin'); + }); + + it('enforces the documented size and repeat thresholds', () => { + expect(AI_REPORTER_QUALIFICATION_THRESHOLDS).toMatchObject({ + minimumActionableFailurePercent: 100, + maximumOutputBytesPerCase: 64 * 1024, + maximumCompactCaseAiOutputBytes: 2 * 1024, + minimumComparableBaselineBytes: 1024, + maximumPerCaseAiToBaselinePercent: 100, + maximumAggregateAiToLegacyPercent: 50, + maximumAggregateAiToPlaintextPercent: 50, + deterministicRunCount: 3, + minimumPrivacyPassPercent: 100, + minimumFullLogPassPercent: 100, + minimumStdoutContractPassPercent: 100, + minimumWarningContractPassPercent: 100 + }); + expect(qualification.gates.find(({ id }) => id === 'size.invocation-boundary')).toMatchObject({ + passed: true, + failedCases: [] + }); + }); + + it('measures unnormalized emitted UTF-8 output including every delimiter', async () => { + const byteLength: typeof Buffer.byteLength = Buffer.byteLength; + const byteLengthSpy: jest.SpiedFunction = jest.spyOn(Buffer, 'byteLength'); + try { + const result: IAiReporterQualificationResult = await runAiReporterQualificationCorpusAsync(); + const outputByLogPath: Map = new Map(); + for (const [value] of byteLengthSpy.mock.calls) { + if ( + typeof value !== 'string' || + !value.startsWith('{"kind":"ai.') || + !value.includes('"kind":"ai.final"') || + !value.endsWith('\n') + ) { + continue; + } + const final: { kind?: string; log?: { path?: string } } = JSON.parse( + value.trimEnd().split('\n').at(-1)! + ); + if (final.kind === 'ai.final' && final.log?.path !== undefined) { + const previous: string | undefined = outputByLogPath.get(final.log.path); + if (previous === undefined || byteLength(value, 'utf8') > byteLength(previous, 'utf8')) { + outputByLogPath.set(final.log.path, value); + } + } + } + const capturedOutput: string[] = [...outputByLogPath.values()].slice(0, result.cases.length); + expect(capturedOutput).toHaveLength(result.cases.length); + expect(capturedOutput.every((output) => output.endsWith('\n'))).toBe(true); + expect(capturedOutput.every((output) => !output.includes(''))).toBe(true); + expect(capturedOutput.map((output) => byteLength(output, 'utf8'))).toEqual( + result.cases.map(({ aiOutputBytes }) => aiOutputBytes) + ); + } finally { + byteLengthSpy.mockRestore(); + } + }, 15000); + + it('normalizes Windows and POSIX paths without storing machine-specific separators', () => { + expect( + normalizeAiReporterQualificationOutput( + '{"path":"C:\\\\repo\\\\temp\\\\rush.log"}', + 'C:\\repo\\temp\\rush.log', + 'C:\\repo\\temp' + ) + ).toBe('{"path":""}'); + expect( + normalizeAiReporterQualificationOutput( + '{"path":"/repo/temp/rush.log","root":"/repo/temp"}', + '/repo/temp/rush.log', + '/repo/temp' + ) + ).toBe('{"path":"","root":""}'); + }); + + it('reports actionable per-case failures when a blocking gate regresses', () => { + const cases: IAiReporterQualificationCaseResult[] = qualification.cases.map( + (testCase: IAiReporterQualificationCaseResult, index: number) => + index === 0 ? { ...testCase, actionable: false } : testCase + ); + const failed: IAiReporterQualificationResult = evaluateAiReporterQualification(cases); + + expect(failed.passed).toBe(false); + expect(formatAiReporterQualificationFailures(failed)).toContain( + 'actionability: actual=90.91, required=>= 100%; cases=bootstrap-unsupported-node' + ); + }); + + it('fails with an actionable case list when the AI reporter omits its log reference', async () => { + const originalReport: typeof AiReporter.prototype.report = AiReporter.prototype.report; + const reportSpy: jest.SpiedFunction = jest + .spyOn(AiReporter.prototype, 'report') + .mockImplementation(function (this: AiReporter, event: IReporterEventEnvelope): void { + if (event.type !== 'artifactAvailable') { + originalReport.call(this, event); + } + }); + try { + const failed: IAiReporterQualificationResult = await runAiReporterQualificationCorpusAsync(); + expect(failed.passed).toBe(false); + expect(formatAiReporterQualificationFailures(failed)).toContain( + 'full-log: actual=0.00, required=>= 100%; cases=' + ); + expect( + failed.cases.every(({ failures }) => + failures.includes('full log path, permissions, completeness, or correlation invalid') + ) + ).toBe(true); + } finally { + reportSpy.mockRestore(); + } + }); + + it('fails qualification when a renderer substitutes unrelated remediation', async () => { + const report: typeof AiReporter.prototype.report = AiReporter.prototype.report; + const reportSpy: jest.SpiedFunction = jest + .spyOn(AiReporter.prototype, 'report') + .mockImplementation(function (this: AiReporter, event: IReporterEventEnvelope): void { + const payload: { remediation?: unknown } = event.payload as { remediation?: unknown }; + report.call( + this, + event.type === 'diagnosticEmitted' && payload.remediation !== undefined + ? { + ...event, + payload: { + ...payload, + remediation: [ + { + descriptionKey: 'remediation.unrelated', + command: 'rush --version', + automatedExecutionSafety: 'safe' + } + ] + } + } + : event + ); + }); + try { + const result: IAiReporterQualificationResult = await runAiReporterQualificationCorpusAsync(); + const actionability: IAiReporterQualificationGateResult | undefined = result.gates.find( + ({ id }) => id === 'actionability' + ); + expect(actionability?.passed).toBe(false); + expect(actionability?.failedCases).toContain('bootstrap-unsupported-node'); + expect(actionability?.failedCases).toContain('configuration-invalid-json'); + expect(result.passed).toBe(false); + } finally { + reportSpy.mockRestore(); + } + }, 15000); +}); + +describe('AI qualification actionable diagnostic contract', () => { + const expected: Parameters[1] = { + code: 'RUSH_COMMAND_FAILED', + category: 'command', + summaryKey: 'diagnostic.RUSH_COMMAND_FAILED.summary', + parameters: { commandName: { value: 'build', privacy: 'public' } }, + remediation: [ + { + descriptionKey: 'remediation.review-command-usage', + command: 'rush build --help', + automatedExecutionSafety: 'safe' + } + ] + }; + const valid: IAiDiagnostic = { + code: expected.code, + category: expected.category, + severity: 'error', + summary: 'The requested command could not be parsed.', + context: { commandName: 'build' }, + remediation: expected.remediation + }; + + it('accepts genuinely retained fallback context and the correct usage action', () => { + expect(hasExpectedAiQualificationDiagnostic(valid, expected)).toBe(true); + }); + + it.each>([ + { context: undefined, remediation: undefined }, + { context: { commandName: 'other' } }, + { remediation: [] }, + { remediation: [{ ...expected.remediation[0], command: 'rush --version' }] }, + { remediation: [{ ...expected.remediation[0], automatedExecutionSafety: 'unsafe' }] } + ])('rejects missing or incorrect fallback evidence: %j', (change) => { + expect(hasExpectedAiQualificationDiagnostic({ ...valid, ...change }, expected)).toBe(false); + }); +}); + +describe('qualified AI reporter decision', () => { + function passedQualification(): IAiReporterQualificationResult { + const emptyCases: readonly IAiReporterQualificationCaseResult[] = []; + return { + schemaVersion: '1.0', + passed: true, + thresholds: AI_REPORTER_QUALIFICATION_THRESHOLDS, + cases: emptyCases, + gates: [] + }; + } + + it('recognizes agent variables without activating selection before the privacy prerequisite', () => { + expect(getQualifiedAiReporterDecision({ COPILOT_CLI: '1' }, [], passedQualification())).toMatchObject({ + agentDetected: true, + eligible: false, + reason: 'privacy prerequisite unavailable' + }); + expect( + getQualifiedAiReporterDecision({ MY_AGENT: 'yes' }, ['MY_AGENT'], passedQualification()) + ).toMatchObject({ + agentDetected: true, + eligible: false, + reason: 'privacy prerequisite unavailable' + }); + }); + + it('returns a reusable AI decision only after qualification and privacy are accepted', () => { + expect( + getQualifiedAiReporterDecision({ COPILOT_CLI: '1' }, [], passedQualification(), true) + ).toMatchObject({ + agentDetected: true, + eligible: true, + reporter: 'ai', + reason: 'qualified' + }); + }); + + it('blocks selection when qualification is absent or failed', () => { + expect(getQualifiedAiReporterDecision({ COPILOT_CLI: '1' }, [], undefined, true)).toMatchObject({ + eligible: false, + reason: 'qualification unavailable' + }); + expect( + getQualifiedAiReporterDecision( + { COPILOT_CLI: '1' }, + [], + { ...passedQualification(), passed: false }, + true + ) + ).toMatchObject({ + eligible: false, + reason: 'qualification failed' + }); + }); + + it('keeps RUSH_REPORTER=legacy authoritative even after qualification passes', () => { + expect( + getQualifiedAiReporterDecision( + { COPILOT_CLI: '1', RUSH_REPORTER: 'legacy' }, + [], + passedQualification(), + true + ) + ).toMatchObject({ + agentDetected: true, + eligible: false, + reason: 'RUSH_REPORTER=legacy' + }); + }); +}); diff --git a/libraries/reporter/src/test/FileReporter.test.ts b/libraries/reporter/src/test/FileReporter.test.ts index b58ea33eda..4e80362abd 100644 --- a/libraries/reporter/src/test/FileReporter.test.ts +++ b/libraries/reporter/src/test/FileReporter.test.ts @@ -490,6 +490,37 @@ describe('FileReporter', () => { }); }); + it('retains local-sensitive producer identity but redacts secret producer identity', async () => { + await withTempDir(async (base: string) => { + const reporter: FileReporter = new FileReporter({ commonTempFolder: base, nowMs: () => FIXED_NOW }); + reporter.report({ + ...ev('extension', { name: 'local.plugin.event' }, 'local-sensitive'), + source: { + packageName: '@private/example-rush-plugin', + packageVersion: '1.0.0', + component: 'PrivatePluginImplementation' + } + }); + reporter.report({ + ...ev('extension', { name: 'secret.plugin.event' }, 'secret'), + source: { + packageName: '@secret/example-rush-plugin', + packageVersion: '2.0.0', + component: 'SecretPluginImplementation' + } + }); + await reporter.closeAsync(); + + const content: string = await fs.promises.readFile(reporter.getArtifact().path!, 'utf8'); + expect(content).toContain('@private/example-rush-plugin'); + expect(content).toContain('PrivatePluginImplementation'); + expect(content).not.toContain('@secret/example-rush-plugin'); + expect(content).not.toContain('SecretPluginImplementation'); + expect(content).toContain('[private-producer]'); + expect(content).toContain('[private-version]'); + }); + }); + it('deletes logs older than the retention window and caps the session count', async () => { await withTempDir(async (base: string) => { const logsDir: string = path.join(base, RUSH_LOGS_DIR_NAME); diff --git a/libraries/reporter/src/test/JsonAiReporter.test.ts b/libraries/reporter/src/test/JsonAiReporter.test.ts index d418f99e88..beac11924b 100644 --- a/libraries/reporter/src/test/JsonAiReporter.test.ts +++ b/libraries/reporter/src/test/JsonAiReporter.test.ts @@ -10,6 +10,13 @@ import { type ITelemetryAggregate } from '../index'; +const SECRET_COMMAND: string = 'qualification-secret-command'; +const SECRET_OPERATION: string = 'qualification-secret-operation'; +const SECRET_PROJECT: string = '@private/qualification-secret-project'; +const SECRET_PHASE: string = 'qualification-secret-phase'; +const SECRET_PARENT_SESSION: string = 'qualification-secret-parent-session'; +const SECRET_PARENT_OPERATION: string = 'qualification-secret-parent-operation'; + function ev( type: string, payload: unknown = {}, @@ -57,21 +64,202 @@ describe('JsonReporter', () => { let output: string = ''; const reporter: JsonReporter = new JsonReporter({ write: (text: string) => (output += text), - maxRecordBytes: 512 + maxRecordBytes: 768 + }); + reporter.report({ + ...ev( + 'externalOutput', + { stream: 'stdout', text: 'x'.repeat(1000) }, + { operationId: 'operation-a', projectName: '@example/project' } + ), + sessionId: 'child-session', + parentSessionId: 'parent-session', + parentOperationId: 'parent-operation', + sourceSequence: 3 }); - reporter.report(ev('externalOutput', { stream: 'stdout', text: 'x'.repeat(1000) })); const records: Record[] = parseLines(output); expect(records).toHaveLength(1); expect((records[0].payload as { name: string }).name).toBe('rush.reporter.record-too-large'); expect(records[0]).toMatchObject({ timestamp: '2026-01-01T00:00:00.000Z', + sessionId: 'child-session', + parentSessionId: 'parent-session', + parentOperationId: 'parent-operation', + sourceSequence: 3, source: { packageName: '@microsoft/rush-lib', packageVersion: '5.177.2' }, + scope: { operationId: 'operation-a', projectName: '@example/project' }, privacy: 'public', required: true, type: 'extension' }); - expect(Buffer.byteLength(output.trim(), 'utf8')).toBeLessThanOrEqual(512); + expect(Buffer.byteLength(output.trim(), 'utf8')).toBeLessThanOrEqual(768); + }); + + it('does not expose a secret diagnostic value in an oversized redacted record', () => { + let output: string = ''; + const reporter: JsonReporter = new JsonReporter({ + write: (text: string) => (output += text), + maxRecordBytes: 512 + }); + reporter.report({ + ...ev('diagnosticEmitted', { + code: 'RUSH_DEPENDENCY_TOOL_FAILED', + parameters: { + token: { value: 'qualification-fake-secret-token', privacy: 'secret' }, + detail: { value: 'x'.repeat(1000), privacy: 'public' } + } + }), + source: { + packageName: '@private/oversized-reporter', + packageVersion: '1.0.0', + component: 'OversizedPrivateComponent' + }, + scope: { + operationId: 'oversized-private-operation', + projectName: '@private/oversized-project' + }, + privacy: 'local-sensitive' + }); + + expect(output).not.toContain('qualification-fake-secret-token'); + expect(output).not.toContain('@private/oversized-reporter'); + expect(output).not.toContain('OversizedPrivateComponent'); + expect(output).not.toContain('oversized-private-operation'); + expect(output).not.toContain('@private/oversized-project'); + expect(parseLines(output)[0]).toMatchObject({ + source: { + packageName: '[private-producer]', + packageVersion: '[private-version]' + }, + privacy: 'local-sensitive', + payload: { + name: 'rush.reporter.record-too-large' + } + }); + expect(parseLines(output)[0].scope).toBeUndefined(); + }); + + it('redacts local-sensitive message text from JSON stdout without dropping envelope metadata', () => { + let output: string = ''; + const reporter: JsonReporter = new JsonReporter({ write: (text: string) => (output += text) }); + reporter.report({ + ...ev( + 'messageEmitted', + { + severity: 'error', + text: 'qualification-local-sensitive-message' + }, + { + operationId: 'operation-a', + projectName: '@example/project' + } + ), + privacy: 'local-sensitive' + }); + + expect(output).not.toContain('qualification-local-sensitive-message'); + expect(parseLines(output)[0]).toMatchObject({ + source: { + packageName: '@microsoft/rush-lib', + packageVersion: '5.177.2' + }, + scope: { + operationId: 'operation-a', + projectName: '@example/project' + }, + privacy: 'local-sensitive', + payload: { + severity: 'error', + text: '[local-sensitive]' + } + }); + }); + + it('allowlists metadata for normal and oversized secret envelopes', () => { + let output: string = ''; + const reporter: JsonReporter = new JsonReporter({ + write: (text: string) => (output += text), + maxRecordBytes: 512 + }); + const secretMetadata: Partial> = { + parentSessionId: SECRET_PARENT_SESSION, + parentOperationId: SECRET_PARENT_OPERATION, + sourceSequence: 7, + source: { + packageName: '@private/qualification-secret-producer', + packageVersion: '1.0.0', + component: 'QualificationSecretComponent' + }, + scope: { + commandName: SECRET_COMMAND, + operationId: SECRET_OPERATION, + projectName: SECRET_PROJECT, + phaseName: SECRET_PHASE + }, + privacy: 'secret' + }; + reporter.report({ + ...ev('messageEmitted', { + severity: 'error', + text: 'qualification-secret-message-text' + }), + ...secretMetadata, + eventId: 'secret-message' + } as IReporterEventEnvelope); + reporter.report({ + ...ev('diagnosticEmitted', { + code: 'RUSH_INTERNAL_UNEXPECTED', + summary: 'qualification-secret-diagnostic-summary', + detail: 'x'.repeat(2000) + }), + ...secretMetadata, + eventId: 'secret-diagnostic', + sequence: 2 + } as IReporterEventEnvelope); + + const records: Record[] = parseLines(output); + expect(records).toHaveLength(2); + for (const record of records) { + expect(Object.keys(record).sort()).toEqual( + [ + 'eventId', + 'payload', + 'privacy', + 'protocolVersion', + 'required', + 'sequence', + 'sessionId', + 'source', + 'sourceSequence', + 'timestamp', + 'type' + ].sort() + ); + expect(record).toMatchObject({ + source: { + packageName: '[private-producer]', + packageVersion: '[private-version]' + }, + privacy: 'secret', + payload: '[secret]' + }); + } + for (const sentinel of [ + SECRET_COMMAND, + SECRET_OPERATION, + SECRET_PROJECT, + SECRET_PHASE, + SECRET_PARENT_SESSION, + SECRET_PARENT_OPERATION, + '@private/qualification-secret-producer', + 'QualificationSecretComponent', + 'qualification-secret-message-text', + 'qualification-secret-diagnostic-summary' + ]) { + expect(output).not.toContain(sentinel); + } + expect(output).not.toContain('rush.reporter.record-too-large'); }); it('redacts secret diagnostic fields from stdout', () => { @@ -136,6 +324,7 @@ describe('AiReporter', () => { events: IReporterEventEnvelope[], options?: { maxBytes?: number } ): { + output: string; records: Record[]; final: IAiFinalRecord; } { @@ -149,7 +338,7 @@ describe('AiReporter', () => { } void reporter.closeAsync(); const records: Record[] = parseLines(output); - return { records, final: records[records.length - 1] as unknown as IAiFinalRecord }; + return { output, records, final: records[records.length - 1] as unknown as IAiFinalRecord }; } it('fails closed when commandResult is missing', async () => { @@ -199,11 +388,121 @@ describe('AiReporter', () => { expect.objectContaining({ category: 'command', severity: 'error', - summary: 'The project \"missing\" passed to \"--only\" does not exist in rush.json.' + summary: 'The project \"missing\" passed to \"--only\" does not exist in rush.json.', + context: { commandName: 'build' }, + remediation: [ + { + descriptionKey: 'remediation.review-command-usage', + command: 'rush build --help', + automatedExecutionSafety: 'safe' + } + ] }) ]); }); + it('counts non-public fallback errors without exposing their message text', () => { + const { final } = run([ + ev('commandStarted', { commandName: 'build' }), + { + ...ev('messageEmitted', { + severity: 'error', + text: 'qualification-local-sensitive-message' + }), + privacy: 'local-sensitive' + }, + { + ...ev('messageEmitted', { + severity: 'error', + text: 'qualification-secret-message' + }), + parentSessionId: SECRET_PARENT_SESSION, + parentOperationId: SECRET_PARENT_OPERATION, + scope: { + commandName: SECRET_COMMAND, + operationId: SECRET_OPERATION, + projectName: SECRET_PROJECT, + phaseName: SECRET_PHASE + }, + privacy: 'secret' + }, + ev('artifactAvailable', { + role: 'log', + path: '/protected/rush.log', + format: 'plaintext', + complete: true + }), + ev('commandResult', { commandName: 'build', succeeded: false, exitCode: 1 }) + ]); + + expect(final.errorCodes).toEqual(['RUSH_COMMAND_FAILED']); + expect(final.errorCount).toBe(2); + expect(final.diagnosticCategoryCounts.command).toBe(2); + expect(final.diagnostics).toEqual([]); + expect(final.truncated).toBe(true); + expect(final.log).toEqual({ + path: '/protected/rush.log', + format: 'plaintext', + complete: true + }); + expect(JSON.stringify(final)).not.toContain('qualification-local-sensitive-message'); + expect(JSON.stringify(final)).not.toContain('qualification-secret-message'); + expect(JSON.stringify(final)).not.toContain(SECRET_COMMAND); + expect(JSON.stringify(final)).not.toContain(SECRET_OPERATION); + expect(JSON.stringify(final)).not.toContain(SECRET_PROJECT); + expect(JSON.stringify(final)).not.toContain(SECRET_PHASE); + expect(JSON.stringify(final)).not.toContain(SECRET_PARENT_SESSION); + expect(JSON.stringify(final)).not.toContain(SECRET_PARENT_OPERATION); + }); + + it('ignores secret lifecycle context in AI status and final scope', () => { + const secretEnvelope: Partial> = { + parentSessionId: SECRET_PARENT_SESSION, + parentOperationId: SECRET_PARENT_OPERATION, + sourceSequence: 7, + scope: { + commandName: SECRET_COMMAND, + operationId: SECRET_OPERATION, + projectName: SECRET_PROJECT, + phaseName: SECRET_PHASE + }, + privacy: 'secret' + }; + const { records, final } = run([ + { + ...ev('commandStarted', { commandName: SECRET_COMMAND }), + ...secretEnvelope + } as IReporterEventEnvelope, + { + ...ev('operationRegistered', { + operationId: SECRET_OPERATION, + projectName: SECRET_PROJECT, + phaseName: SECRET_PHASE + }), + ...secretEnvelope + } as IReporterEventEnvelope, + { + ...ev('operationCompleted', { + operationId: SECRET_OPERATION, + status: 'failure' + }), + ...secretEnvelope + } as IReporterEventEnvelope, + ev('commandResult', { commandName: 'build', succeeded: false, exitCode: 1 }) + ]); + + expect(records.filter(({ kind }) => kind === 'ai.status')).toEqual([]); + expect(final.scope).toEqual({ failedProjects: [] }); + expect(final.operationCounts).toEqual({}); + const serialized: string = JSON.stringify(records); + expect(serialized).not.toContain(SECRET_COMMAND); + expect(serialized).not.toContain(SECRET_OPERATION); + expect(serialized).not.toContain(SECRET_PROJECT); + expect(serialized).not.toContain(SECRET_PHASE); + expect(serialized).not.toContain(SECRET_PARENT_SESSION); + expect(serialized).not.toContain(SECRET_PARENT_OPERATION); + }); + it('counts fallback errors even when detailed diagnostics are disabled', async () => { let output: string = ''; const reporter: AiReporter = new AiReporter({ @@ -223,7 +522,7 @@ describe('AiReporter', () => { expect(final.truncated).toBe(true); }); - it('emits a status record and a bounded final record with scope, codes, and log', () => { + it('coalesces an unrendered start into a complete final record without losing scope, codes, or log', () => { const { records, final } = run([ ev('commandStarted', { commandName: 'build' }), ev('operationRegistered', { operationId: 'op1', projectName: 'project-a' }), @@ -239,7 +538,7 @@ describe('AiReporter', () => { ev('commandResult', { commandName: 'build', succeeded: false, exitCode: 1 }) ]); - expect(records[0].kind).toBe('ai.status'); + expect(records.map(({ kind }) => kind)).toEqual(['ai.final']); expect(final.kind).toBe('ai.final'); expect(final.result).toBe('failed'); expect(final.exitCode).toBe(1); @@ -251,6 +550,113 @@ describe('AiReporter', () => { expect(final.log).toEqual({ path: '/abs/rush.log', format: 'plaintext', complete: true }); }); + it.each(['event', 'flush'])( + 'preserves active-command status at the next %s boundary', + async (boundary: string) => { + let output: string = ''; + const reporter: AiReporter = new AiReporter({ write: (text: string) => (output += text) }); + reporter.report(ev('commandStarted', { commandName: 'build' })); + reporter.report( + ev('artifactAvailable', { role: 'log', path: '/abs/rush.log', format: 'plaintext', complete: false }) + ); + expect(output).toBe(''); + if (boundary === 'event') { + reporter.report(ev('activityChanged', { text: 'building' })); + } else { + await reporter.flushAsync(); + } + expect(parseLines(output).map(({ kind }) => kind)).toEqual(['ai.status']); + reporter.report(ev('commandResult', { succeeded: true, exitCode: 0 })); + reporter.report(ev('artifactAvailable', { role: 'log', path: '/abs/rush.log', complete: true })); + await reporter.closeAsync(); + const final: IAiFinalRecord = parseLines(output).at(-1) as unknown as IAiFinalRecord; + expect(final).toMatchObject({ + result: 'succeeded', + exitCode: 0, + scope: { commandName: 'build' }, + log: { path: '/abs/rush.log', complete: true }, + truncated: false + }); + } + ); + + it('preserves the complete final record when buffered start status is coalesced', async () => { + const output: string[] = ['', '']; + const reporters: AiReporter[] = output.map( + (value: string, index: number) => new AiReporter({ write: (text: string) => (output[index] += text) }) + ); + const events: IReporterEventEnvelope[] = [ + ev('commandStarted', { commandName: 'build' }), + ev('operationRegistered', { operationId: 'op', projectName: 'project' }), + ev('operationCompleted', { operationId: 'op', status: 'failure' }), + ev('diagnosticEmitted', { + diagnosticId: 'root', + code: 'RUSH_OPERATION_FAILED', + category: 'operation', + severity: 'error', + parameters: { + project: { value: 'project', privacy: 'public' }, + token: { value: 'not-for-output', privacy: 'secret' } + }, + remediation: [{ descriptionKey: 'retry', command: 'rush rebuild', automatedExecutionSafety: 'safe' }] + }), + ev('diagnosticEmitted', { + code: 'RUSH_EXTERNAL_TOOL_PROBLEM', + category: 'operation', + severity: 'warning' + }), + ev('artifactAvailable', { + role: 'log', + path: '/absolute/full.log', + format: 'plaintext', + complete: true + }) + ]; + for (const reporter of reporters) { + for (const event of events) reporter.report(event); + } + await reporters[0].flushAsync(); + for (const reporter of reporters) { + reporter.report(ev('commandResult', { succeeded: false, exitCode: 1 })); + await reporter.closeAsync(); + } + const activeRecords: Record[] = parseLines(output[0]); + const completedRecords: Record[] = parseLines(output[1]); + expect(activeRecords.map(({ kind }) => kind)).toEqual(['ai.status', 'ai.final']); + expect(completedRecords.map(({ kind }) => kind)).toEqual(['ai.final']); + expect(completedRecords[0]).toEqual(activeRecords[1]); + expect(output[1]).not.toContain('not-for-output'); + expect(Buffer.byteLength(output[0]) - Buffer.byteLength(output[1])).toBe( + Buffer.byteLength(`${JSON.stringify(activeRecords[0])}\n`) + ); + }); + + it('preserves ordered watch history when a terminal result supersedes buffered start status', () => { + const { records, final } = run([ + ev('commandStarted', { commandName: 'build' }), + ev('watchCycleCompleted', { succeeded: false, iterationId: 1 }), + ev('watchCycleCompleted', { succeeded: true, iterationId: 2 }), + ev('artifactAvailable', { role: 'log', path: '/abs/rush.log', complete: true }), + ev('commandResult', { succeeded: true, exitCode: 0 }) + ]); + expect(records.map(({ kind }) => kind)).toEqual(['ai.watchCycle', 'ai.watchCycle', 'ai.final']); + expect(records.slice(0, -1).map(({ succeeded }) => succeeded)).toEqual([false, true]); + expect(final.result).toBe('succeeded'); + expect(final.log?.path).toBe('/abs/rush.log'); + }); + + it('keeps buffered progress write failures in the synchronous reporter failure boundary', () => { + const failure: Error = new Error('output failed'); + const reporter: AiReporter = new AiReporter({ + write: () => { + throw failure; + } + }); + reporter.report(ev('commandStarted', { commandName: 'build' })); + reporter.report(ev('artifactAvailable', { role: 'log', path: '/abs/rush.log' })); + expect(() => reporter.report(ev('activityChanged', {}))).toThrow(failure); + }); + it('excludes silent operations from AI result counts', () => { const { final } = run([ ev('commandStarted', { commandName: 'build' }), @@ -380,10 +786,107 @@ describe('AiReporter', () => { } void reporter.closeAsync(); const finalLine: string = output.trim().split('\n').pop() ?? ''; - expect(Buffer.byteLength(finalLine, 'utf8')).toBeLessThanOrEqual(512); + expect(Buffer.byteLength(output, 'utf8')).toBeLessThanOrEqual(512); expect((JSON.parse(finalLine) as IAiFinalRecord).truncated).toBe(true); }); + it('includes status records and newline delimiters in the invocation budget', () => { + const { output, final } = run([ + ev('commandStarted', { commandName: 'build' }), + ev('diagnosticEmitted', { + diagnosticId: 'root', + code: 'RUSH_OPERATION_FAILED', + category: 'operation', + severity: 'error', + summary: 'x'.repeat(65115) + }), + ev('commandResult', { succeeded: false, exitCode: 1 }) + ]); + + expect(Buffer.byteLength(output, 'utf8')).toBeLessThanOrEqual(64 * 1024); + expect(final).toMatchObject({ kind: 'ai.final', result: 'failed', exitCode: 1, truncated: true }); + expect(output.endsWith('\n')).toBe(true); + }); + + it.each([true, false])( + 'bounds watch output and retains the log when published early=%s', + async (early: boolean) => { + let output: string = ''; + const reporter: AiReporter = new AiReporter({ write: (text: string) => (output += text) }); + const logPath: string = `/protected/${'logs/'.repeat(200)}full.log`; + const artifact: IReporterEventEnvelope = ev('artifactAvailable', { + role: 'log', + path: logPath, + format: 'plaintext', + complete: true + }); + reporter.report(ev('commandStarted', { commandName: 'build' })); + if (early) reporter.report(artifact); + for (let iterationId: number = 0; iterationId < 1000; iterationId++) { + reporter.report(ev('watchCycleCompleted', { succeeded: true, iterationId })); + } + if (!early) reporter.report(artifact); + reporter.report(ev('commandResult', { succeeded: false, exitCode: 1 })); + await reporter.closeAsync(); + + const records: Record[] = parseLines(output); + const final: IAiFinalRecord = records.at(-1) as unknown as IAiFinalRecord; + expect(Buffer.byteLength(output, 'utf8')).toBeLessThanOrEqual(64 * 1024); + expect(records.filter(({ kind }) => kind === 'ai.watchCycle').length).toBeLessThan(1000); + expect(final).toMatchObject({ result: 'failed', exitCode: 1, truncated: true }); + expect(final.log).toEqual({ path: logPath, format: 'plaintext', complete: true }); + expect(final.diagnostics.length).toBeLessThanOrEqual(20); + } + ); + + it('reserves a late log reference before spending a tight progress budget', async () => { + let output: string = ''; + const reporter: AiReporter = new AiReporter({ + write: (text: string) => (output += text), + maxBytes: 2048 + }); + const logPath: string = `/protected/${'logs/'.repeat(250)}full.log`; + reporter.report(ev('commandStarted', { commandName: 'build' })); + for (let iterationId: number = 0; iterationId < 100; iterationId++) { + reporter.report(ev('watchCycleCompleted', { succeeded: true, iterationId })); + } + expect(output).toBe(''); + reporter.report(ev('artifactAvailable', { role: 'log', path: logPath, complete: true })); + reporter.report(ev('commandResult', { succeeded: false, exitCode: 1 })); + await reporter.closeAsync(); + + const final: IAiFinalRecord = parseLines(output).at(-1) as unknown as IAiFinalRecord; + expect(Buffer.byteLength(output, 'utf8')).toBeLessThanOrEqual(2048); + expect(final.log?.path).toBe(logPath); + expect(final).toMatchObject({ result: 'failed', exitCode: 1, truncated: true }); + }); + + it('budgets escaped paths and multibyte diagnostics without exposing secrets', () => { + const logPath: string = `C:\\logs\\${'x'.repeat(80)}\\report-\u00e9.log`; + const { output, final } = run( + [ + ev('commandStarted', { commandName: 'build' }), + ev('diagnosticEmitted', { + code: 'RUSH_OPERATION_FAILED', + category: 'operation', + severity: 'error', + parameters: { + message: { value: '\u00e9\u{1f680}'.repeat(200), privacy: 'public' }, + token: { value: 'budget-secret-value', privacy: 'secret' } + } + }), + ev('artifactAvailable', { role: 'log', path: logPath, complete: true }), + ev('commandResult', { succeeded: false, exitCode: 1 }) + ], + { maxBytes: 512 } + ); + + expect(Buffer.byteLength(output, 'utf8')).toBeLessThanOrEqual(512); + expect(final.log?.path).toBe(logPath); + expect(final).toMatchObject({ kind: 'ai.final', result: 'failed', exitCode: 1 }); + expect(output).not.toContain('budget-secret-value'); + }); + it('falls back to a minimal bounded record when fixed fields are oversized', () => { const events: IReporterEventEnvelope[] = [ ev('commandStarted', { commandName: 'x'.repeat(5000) }), @@ -438,6 +941,130 @@ describe('AiReporter', () => { expect(final.truncated).toBe(false); }); + it('orders root-cause diagnostics before diagnostics that reference them', () => { + const { final } = run([ + ev('diagnosticEmitted', { + diagnosticId: 'outer', + code: 'RUSH_OPERATION_FAILED', + category: 'operation', + severity: 'error', + causeDiagnosticIds: ['root'] + }), + ev('diagnosticEmitted', { + diagnosticId: 'root', + code: 'RUSH_DEPENDENCY_TOOL_FAILED', + category: 'dependency-tool', + severity: 'error' + }), + ev('commandResult', { commandName: 'build', succeeded: false, exitCode: 1 }) + ]); + + expect(final.diagnostics.map(({ diagnosticId }) => diagnosticId)).toEqual(['root', 'outer']); + }); + + it('projects classified context without exposing secret values', () => { + const { final } = run([ + ev('diagnosticEmitted', { + diagnosticId: 'auth', + code: 'RUSH_NETWORK_AUTH_UNAUTHORIZED', + category: 'network-auth', + severity: 'error', + summaryKey: 'diagnostic.RUSH_NETWORK_AUTH_UNAUTHORIZED.summary', + parameters: { + registryUrl: { value: 'https://registry.example.test/', privacy: 'public' }, + token: { value: 'qualification-fake-secret-token', privacy: 'secret' } + } + }), + ev('commandResult', { commandName: 'install', succeeded: false, exitCode: 1 }) + ]); + + expect(final.diagnostics[0]).toMatchObject({ + context: { + registryUrl: 'https://registry.example.test/', + token: '[secret]' + } + }); + expect(final.diagnostics[0].summaryKey).toBeUndefined(); + expect(JSON.stringify(final)).not.toContain('qualification-fake-secret-token'); + }); + + it('preserves nonstandard summary keys rather than treating them as derived metadata', () => { + const { final } = run([ + ev('diagnosticEmitted', { + code: 'RUSH_OPERATION_FAILED', + category: 'operation', + severity: 'error', + summaryKey: 'plugin.custom.failure-summary' + }), + ev('commandResult', { succeeded: false, exitCode: 1 }) + ]); + expect(final.diagnostics[0].summaryKey).toBe('plugin.custom.failure-summary'); + }); + + it('counts secret diagnostics while marking their omitted details as truncated', () => { + const { final } = run([ + { + ...ev('diagnosticEmitted', { + diagnosticId: 'secret-error', + code: 'RUSH_INTERNAL_UNEXPECTED', + category: 'internal', + severity: 'error', + summary: 'qualification-fake-secret-token' + }), + parentSessionId: SECRET_PARENT_SESSION, + parentOperationId: SECRET_PARENT_OPERATION, + scope: { + commandName: SECRET_COMMAND, + operationId: SECRET_OPERATION, + projectName: SECRET_PROJECT, + phaseName: SECRET_PHASE + }, + privacy: 'secret' + }, + ev('commandResult', { commandName: 'build', succeeded: false, exitCode: 1 }) + ]); + + expect(final.errorCount).toBe(1); + expect(final.errorCodes).toEqual([]); + expect(final.diagnosticCategoryCounts).toEqual({}); + expect(final.diagnostics).toEqual([]); + expect(final.truncated).toBe(true); + expect(JSON.stringify(final)).not.toContain('qualification-fake-secret-token'); + expect(JSON.stringify(final)).not.toContain(SECRET_COMMAND); + expect(JSON.stringify(final)).not.toContain(SECRET_OPERATION); + expect(JSON.stringify(final)).not.toContain(SECRET_PROJECT); + expect(JSON.stringify(final)).not.toContain(SECRET_PHASE); + expect(JSON.stringify(final)).not.toContain(SECRET_PARENT_SESSION); + expect(JSON.stringify(final)).not.toContain(SECRET_PARENT_OPERATION); + }); + + it('preserves fallback errors when secret diagnostics are also suppressed', () => { + const { final } = run([ + { + ...ev('diagnosticEmitted', { + diagnosticId: 'secret-error', + code: 'RUSH_INTERNAL_UNEXPECTED', + category: 'internal', + severity: 'error', + summary: 'qualification-fake-secret-token' + }), + privacy: 'secret' + }, + ev('messageEmitted', { severity: 'error', text: 'First visible fallback error.' }), + ev('messageEmitted', { severity: 'error', text: 'Second visible fallback error.' }), + ev('commandResult', { commandName: 'build', succeeded: false, exitCode: 1 }) + ]); + + expect(final.errorCount).toBe(3); + expect(final.errorCodes).toEqual(['RUSH_COMMAND_FAILED']); + expect(final.diagnostics.map(({ summary }) => summary)).toEqual([ + 'First visible fallback error.', + 'Second visible fallback error.' + ]); + expect(final.truncated).toBe(true); + expect(JSON.stringify(final)).not.toContain('qualification-fake-secret-token'); + }); + it('excludes raw external output and keeps stdout pure JSON', () => { let output: string = ''; const reporter: AiReporter = new AiReporter({ write: (text: string) => (output += text) });