diff --git a/eng/pipelines/scripts/regenerate-runner.js b/eng/pipelines/scripts/regenerate-runner.js new file mode 100644 index 000000000000..6afbce97b6e3 --- /dev/null +++ b/eng/pipelines/scripts/regenerate-runner.js @@ -0,0 +1,1064 @@ +#!/usr/bin/env node + +const { execSync, spawn } = require("child_process"); +const fs = require("fs"); +const path = require("path"); + +function getArg(name, defaultValue = "") { + const index = process.argv.indexOf(name); + if (index < 0 || index + 1 >= process.argv.length) return defaultValue; + return process.argv[index + 1]; +} + +const sdkRoot = process.cwd(); +const specRepoRoot = getArg("--specRepoRoot"); +const maxWorkers = Number(getArg("--maxWorkers", "4")); +const buildWorkers = Number(getArg("--buildWorkers", "1")); +const maxPackages = Number(getArg("--maxPackages", "0")); +const skipBuild = getArg("--skipBuild", "false").toLowerCase() === "true"; +const emitterVersion = getArg("--emitterVersion", ""); +const directoryListFile = getArg("--directoryList", ""); +const resultOutputDir = getArg("--resultOutputDir", ""); +// No per-process timeout: aligned with azure-sdk-for-net / azure-sdk-for-go +// regeneration scripts, which rely on the ADO job-level timeout +// (timeoutInMinutes, default 60min) and let individual tsp-client / pnpm +// invocations run to completion. If a single package hangs, the whole shard +// will eventually be killed by ADO — that's the accepted trade-off. + +if (!specRepoRoot || !fs.existsSync(specRepoRoot)) { + console.error(`ERROR: spec repo root not found: ${specRepoRoot}`); + process.exit(1); +} + +function normalizePath(p) { + return p.replace(/\\/g, "/"); +} + +// Per-package spec lookup goes through each SDK's tsp-location.yaml only. +// We deliberately do NOT scan the cloned spec repo to discover tspconfigs: +// (1) the matrix-stage filter `-OnlyTypeSpec true` already keeps only packages +// that have tsp-location.yaml on the SDK side, so any spec-index "rescue" +// would never fire; (2) reviewers (mentor comment #3) want the regeneration +// contract to be exactly "what tsp-location.yaml points at", with the small +// set of packages that lack tsp-location.yaml surfaced in the PR description +// so the spec/SDK team can on-board them properly. +// +// Helper: parse `directory:` (and optionally `commit:`/`repo:`) out of a +// tsp-location.yaml. Returns null if the file is missing the directory field +// or has an obviously-broken value. +function readTspLocation(tspLocationPath) { + let content; + try { + content = fs.readFileSync(tspLocationPath, "utf8"); + } catch { + return { error: "read failed" }; + } + if (/^\s*repo\s*:\s*\.\./m.test(content)) { + return { error: "relative repo" }; + } + const directoryMatch = content.match(/^\s*directory\s*:\s*(.+?)\s*$/m); + if (!directoryMatch) { + return { error: "missing 'directory:' field" }; + } + const directory = directoryMatch[1].trim(); + const commitMatch = content.match(/^\s*commit\s*:\s*(.+?)\s*$/m); + const repoMatch = content.match(/^\s*repo\s*:\s*(.+?)\s*$/m); + return { + directory, + commit: commitMatch ? commitMatch[1].trim() : "", + repo: repoMatch ? repoMatch[1].trim() : "", + }; +} + +function tsPrefix(startMs) { + const now = new Date(); + const hh = String(now.getUTCHours()).padStart(2, "0"); + const mm = String(now.getUTCMinutes()).padStart(2, "0"); + const ss = String(now.getUTCSeconds()).padStart(2, "0"); + const ms = String(now.getUTCMilliseconds()).padStart(3, "0"); + const elapsedSec = ((Date.now() - startMs) / 1000).toFixed(1).padStart(6, " "); + return `[${hh}:${mm}:${ss}.${ms}Z +${elapsedSec}s] `; +} + +// Wrap a streaming chunk handler so every completed line is prefixed with a +// real timestamp + elapsed-seconds marker. Partial lines (no trailing newline) +// are buffered until the next chunk so we never break mid-line. This makes +// downloaded per-package logs actually useful for diagnosing slow phases — +// instead of relying on ADO's coarse, flush-time stamps. +function makeLineStamper(startMs, onLine) { + let buf = ""; + function flushLine(line) { + onLine(tsPrefix(startMs) + line + "\n"); + } + return { + push(chunk) { + buf += chunk; + let idx; + while ((idx = buf.indexOf("\n")) >= 0) { + const line = buf.slice(0, idx).replace(/\r$/, ""); + buf = buf.slice(idx + 1); + flushLine(line); + } + }, + end() { + if (buf.length > 0) { + flushLine(buf); + buf = ""; + } + }, + }; +} + +function runCommand(cmd, args, cwd) { + return new Promise((resolve) => { + const startMs = Date.now(); + let output = ""; + let lastOutputMs = Date.now(); + const proc = spawn(cmd, args, { cwd, shell: true }); + + const appendLine = (line) => { output += line; lastOutputMs = Date.now(); }; + const stdoutStamper = makeLineStamper(startMs, appendLine); + const stderrStamper = makeLineStamper(startMs, appendLine); + + // Header line so the log shows exactly when the command was launched. + output += `${tsPrefix(startMs)}$ ${cmd} ${args.join(" ")} (cwd=${cwd})\n`; + + // Heartbeat: while the child is alive, every HEARTBEAT_INTERVAL_MS print a + // timestamped progress line showing how long it's been silent and the + // child process's current CPU / RSS. This makes tsp-compile's silent + // multi-minute compile phase observable in the log instead of looking + // like a hang. We deliberately do NOT touch lastOutputMs here — heartbeats + // are runner-side metadata, not real child output. + const HEARTBEAT_INTERVAL_MS = 30 * 1000; + let lastResourceSnapshot = null; + // Walk the process tree rooted at `rootPid` (the shell wrapper spawned by + // shell:true), summing CPU jiffies and RSS across all descendants. We have + // to do this because spawn(..., { shell: true }) makes proc.pid point to + // /bin/sh, not the actual tsp / node / npx worker that's doing the work — + // so reading just proc.pid yields CPU 0% / RSS 2MB even when tsp compile + // is burning a CPU core. We use /proc//task//children (a + // space-separated list of direct child pids) to walk recursively. + function collectDescendants(rootPid) { + const all = new Set(); + const stack = [rootPid]; + while (stack.length) { + const pid = stack.pop(); + if (all.has(pid)) continue; + all.add(pid); + let taskDir; + try { + taskDir = fs.readdirSync(`/proc/${pid}/task`); + } catch { continue; } + for (const tid of taskDir) { + let childrenStr; + try { + childrenStr = fs.readFileSync(`/proc/${pid}/task/${tid}/children`, "utf8"); + } catch { continue; } + for (const c of childrenStr.split(/\s+/)) { + const n = Number(c); + if (Number.isInteger(n) && n > 0 && !all.has(n)) stack.push(n); + } + } + } + return [...all]; + } + + function readPidStat(pid) { + try { + const stat = fs.readFileSync(`/proc/${pid}/stat`, "utf8"); + const rparen = stat.lastIndexOf(")"); + const tail = stat.slice(rparen + 2).split(" "); + const utime = Number(tail[11]); + const stime = Number(tail[12]); + return utime + stime; + } catch { return 0; } + } + + function readPidRssKb(pid) { + try { + const status = fs.readFileSync(`/proc/${pid}/status`, "utf8"); + const m = status.match(/VmRSS:\s*(\d+)\s*kB/); + return m ? Number(m[1]) : 0; + } catch { return 0; } + } + + function readProcSnapshot(rootPid) { + // Sum CPU jiffies + RSS over the whole process tree rooted at rootPid. + try { + const pids = collectDescendants(rootPid); + let totalJiffies = 0; + let totalRssKb = 0; + for (const pid of pids) { + totalJiffies += readPidStat(pid); + totalRssKb += readPidRssKb(pid); + } + const nowMs = Date.now(); + let cpuPct = null; + if (lastResourceSnapshot) { + const dJiffies = totalJiffies - lastResourceSnapshot.totalJiffies; + const dMs = nowMs - lastResourceSnapshot.nowMs; + const clkTck = 100; // standard on Linux + if (dMs > 0) cpuPct = Math.round((dJiffies * 1000) / (clkTck * dMs) * 100); + } + lastResourceSnapshot = { totalJiffies, nowMs }; + return { + cpuPct, + rssMB: Math.round(totalRssKb / 1024), + procCount: pids.length, + }; + } catch { + return null; + } + } + const heartbeatTimer = setInterval(() => { + const silentSec = Math.round((Date.now() - lastOutputMs) / 1000); + const snap = readProcSnapshot(proc.pid); + const parts = [`silent ${silentSec}s`]; + if (snap) { + if (snap.cpuPct !== null) parts.push(`CPU ${snap.cpuPct}%`); + parts.push(`RSS ${snap.rssMB}MB`); + parts.push(`procs ${snap.procCount}`); + } + output += `${tsPrefix(startMs)}[heartbeat] still running (${parts.join(", ")})\n`; + }, HEARTBEAT_INTERVAL_MS); + + proc.stdout.on("data", (d) => { stdoutStamper.push(d.toString()); }); + proc.stderr.on("data", (d) => { stderrStamper.push(d.toString()); }); + proc.on("close", (code) => { + stdoutStamper.end(); + stderrStamper.end(); + clearInterval(heartbeatTimer); + resolve({ code, output }); + }); + proc.on("error", (err) => { + stdoutStamper.end(); + stderrStamper.end(); + clearInterval(heartbeatTimer); + output += `${tsPrefix(startMs)}[runner] spawn error: ${err.message}\n`; + resolve({ code: 1, output }); + }); + }); +} + +function extractError(output) { + const lines = output.split("\n"); + const specMissing = lines.filter((l) => /tspconfig\.yaml not found/.test(l)).slice(0, 1); + if (specMissing.length > 0) return specMissing[0].trim(); + + const compileErrs = lines.filter((l) => /\.tsp:\d+:\d+ - error /.test(l)).slice(0, 3); + if (compileErrs.length > 0) { + return compileErrs + .map((line) => { + const match = line.match(/- (error .+)$/); + return match ? match[1].trim() : line.trim(); + }) + .join(" | "); + } + + const npmErrs = lines.filter((l) => /^npm error/.test(l) && l.trim() !== "npm error").slice(0, 2); + if (npmErrs.length > 0) return npmErrs.map((l) => l.trim()).join(" | "); + + const gitErrs = lines.filter((l) => /fatal:|git clone failed/.test(l)).slice(0, 2); + if (gitErrs.length > 0) return gitErrs.map((l) => l.trim()).join(" | "); + + const buildErrs = lines.filter((l) => /Invalid config|warp build threw|does not exist|Cannot find/.test(l)).slice(0, 2); + if (buildErrs.length > 0) return buildErrs.map((l) => l.trim()).join(" | "); + + const generalErrs = lines.filter((l) => /Failed to generate|Error:/.test(l) && !/tsp-client-config\.yaml/.test(l)).slice(0, 2); + if (generalErrs.length > 0) return generalErrs.map((l) => l.trim()).join(" | "); + + // Fallback: show last meaningful lines + const lastLines = lines.filter((l) => l.trim().length > 0).slice(-3); + if (lastLines.length > 0) return lastLines.map((l) => l.trim()).join(" | "); + + return "Unknown error"; +} + +// Sanitize a package directory like "sdk/foo/arm-foo" into a safe filename. +function safeLogName(pkg) { + return pkg.replace(/[\\/]/g, "__"); +} + +// Write the full log for a single package to /logs//.log, +// and ALWAYS emit an ADO-collapsible group (##[group]) with the full log so +// every package's run — success or failure — is folded by default and can be +// expanded on demand from the ADO UI. Failures are still easy to find via the +// inline "❌ [N/M] FAILED" lines and the SUMMARY section at the end. +function recordPackageLog(phase, pkg, success, output) { + if (resultOutputDir) { + try { + const logDir = path.join(resultOutputDir, "logs", phase); + fs.mkdirSync(logDir, { recursive: true }); + const logFile = path.join(logDir, `${safeLogName(pkg)}.log`); + fs.writeFileSync(logFile, output); + } catch (err) { + console.log(` Warning: failed to write log for ${pkg}: ${err.message}`); + } + } + const status = success ? "OK" : "FAILED"; + console.log(`##[group]${phase} log [${status}]: ${pkg}`); + console.log(output); + console.log("##[endgroup]"); +} + +// Comment #5 (mentor): replace the custom breaking-change-detector.js with the +// official @azure-tools/js-sdk-release-tools `update-changelog` CLI. That tool +// uses typescript-codegen-breaking-change-detector internally and writes a +// CHANGELOG.md section (### Breaking Changes / ### Features Added / ...). It +// compares against the latest npm-published version, which is a better +// regression baseline than git HEAD for "candidate emitter" runs. +const CHANGELOG_TOOL_DIR = "eng/tools/js-sdk-release-tools"; +let changelogToolReady = false; + +async function installChangelogTool() { + if (changelogToolReady) return true; + const toolDirAbs = path.join(sdkRoot, CHANGELOG_TOOL_DIR); + if (!fs.existsSync(path.join(toolDirAbs, "package.json"))) { + console.log(`WARNING: ${CHANGELOG_TOOL_DIR}/package.json not found; skipping changelog generation.`); + return false; + } + console.log(`Installing ${CHANGELOG_TOOL_DIR} dependencies for update-changelog...`); + const install = await runCommand("npm", ["--prefix", CHANGELOG_TOOL_DIR, "ci"], sdkRoot); + if (install.code !== 0) { + console.log(`WARNING: npm ci for ${CHANGELOG_TOOL_DIR} failed; skipping changelog generation.`); + console.log(install.output.slice(-1000)); + return false; + } + changelogToolReady = true; + console.log("update-changelog tool installed."); + return true; +} + +// Run `update-changelog --sdkRepoPath --packagePath ` for a single +// package. Returns { success, hasBreaking, hasChanges, output }. Never throws; +// any failure is recorded and the caller decides what to do. +async function runUpdateChangelog(packageRelPath) { + if (!changelogToolReady) { + return { success: false, hasBreaking: false, hasChanges: false, output: "(changelog tool not installed)" }; + } + const result = await runCommand( + "npm", + [ + "--prefix", + CHANGELOG_TOOL_DIR, + "exec", + "--no", + "--", + "update-changelog", + "--sdkRepoPath", + sdkRoot, + "--packagePath", + path.join(sdkRoot, packageRelPath), + ], + sdkRoot + ); + const success = result.code === 0; + // Detect breaking by scanning the package's CHANGELOG.md for an unreleased + // "### Breaking Changes" heading. update-changelog rewrites the top section + // in place; we look only at the first ~200 lines to avoid matching old + // entries. + let hasBreaking = false; + let hasChanges = false; + try { + const changelogPath = path.join(sdkRoot, packageRelPath, "CHANGELOG.md"); + if (fs.existsSync(changelogPath)) { + const head = fs.readFileSync(changelogPath, "utf8").split("\n").slice(0, 200).join("\n"); + // Stop at the second "## " (next released version) so we only inspect + // the top entry. + const firstVersion = head.indexOf("\n## "); + const top = firstVersion >= 0 ? head.slice(0, head.indexOf("\n## ", firstVersion + 1) > 0 ? head.indexOf("\n## ", firstVersion + 1) : head.length) : head; + hasBreaking = /^###\s+Breaking Changes\b/m.test(top); + hasChanges = /^###\s+(Features Added|Breaking Changes|Bugs Fixed|Other Changes)\b/m.test(top); + } + } catch (err) { + // best-effort detection — don't fail the whole package because we couldn't + // parse the changelog + } + return { success, hasBreaking, hasChanges, output: result.output }; +} + +// Recursively delete nested duplicate workspace directories of the form +// sdk/X/Y/sdk/X/Y created by tsp-client when emitter-output-dir is misresolved. +// These break pnpm install because two workspaces share the same package name. +function cleanupNestedDuplicateWorkspaces(root) { + const nestedPattern = /^sdk[/\\][^/\\]+[/\\][^/\\]+[/\\]sdk[/\\]/; + function findNested(dir, results) { + let entries; + try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return results; } + for (const entry of entries) { + if (entry.name === "node_modules" || entry.name === ".git") continue; + const full = path.join(dir, entry.name); + if (entry.isDirectory()) { + findNested(full, results); + } else if (entry.name === "package.json") { + const rel = normalizePath(path.relative(root, full)); + if (nestedPattern.test(rel)) results.push(full); + } + } + return results; + } + const packageJsonPaths = findNested(path.join(root, "sdk"), []); + if (packageJsonPaths.length === 0) return 0; + + // Delete the inner duplicate "sdk" folder so the outer workspace stays. + // Example: sdk/azurestackhci/arm-azurestackhci/sdk -> delete this whole inner sdk + const innerSdkRoots = new Set(); + for (const pkgJson of packageJsonPaths) { + const rel = normalizePath(path.relative(root, pkgJson)); + const segments = rel.split("/"); + const innerIndex = segments.indexOf("sdk", 1); + if (innerIndex > 0) { + const innerSdkRel = segments.slice(0, innerIndex + 1).join("/"); + innerSdkRoots.add(path.join(root, innerSdkRel)); + } + } + for (const innerSdk of innerSdkRoots) { + try { + fs.rmSync(innerSdk, { recursive: true, force: true }); + console.log(` Removed nested duplicate workspace: ${path.relative(root, innerSdk)}`); + } catch (err) { + console.log(` Warning: failed to remove ${innerSdk}: ${err.message}`); + } + } + return innerSdkRoots.size; +} + +// ============ Pre-build workaround helpers ============ +// These work around bugs in the dev emitter where generated packages are missing +// config files or dependencies needed for the new warp build system. + +function cleanupTempTypeSpecFiles(root) { + // TempTypeSpecFiles contain package.json with name "typescript-emitter-package" + // which causes turbo "duplicate workspace" errors + const sdkDir = path.join(root, "sdk"); + let removed = 0; + function scan(dir, depth) { + if (depth > 4) return; + let entries; + try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; } + for (const e of entries) { + if (!e.isDirectory() || e.name === "node_modules" || e.name === ".git") continue; + const full = path.join(dir, e.name); + if (e.name === "TempTypeSpecFiles") { + fs.rmSync(full, { recursive: true, force: true }); + removed++; + } else { + scan(full, depth + 1); + } + } + } + scan(sdkDir, 0); + if (removed > 0) console.log(` Removed ${removed} TempTypeSpecFiles directories`); +} + +function scaffoldWarpConfigs(root) { + // The dev emitter generates warp.config.yml referencing config/tsconfig.src.*.json + // but doesn't create those files. We scaffold them here. + const sdkDir = path.join(root, "sdk"); + let count = 0; + function scan(dir, depth) { + if (depth > 3) return; + let entries; + try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; } + for (const e of entries) { + if (!e.isDirectory() || e.name === "node_modules" || e.name === ".git") continue; + const full = path.join(dir, e.name); + const warpPath = path.join(full, "warp.config.yml"); + if (fs.existsSync(warpPath)) { + const configDir = path.join(full, "config"); + // Calculate relative path depth to eng/tsconfigs + const rel = path.relative(configDir, path.join(root, "eng", "tsconfigs")).replace(/\\/g, "/"); + const targets = { + "tsconfig.src.browser.json": `${rel}/src.browser.json`, + "tsconfig.src.cjs.json": `${rel}/src.cjs.json`, + "tsconfig.src.esm.json": `${rel}/src.esm.json`, + "tsconfig.src.react-native.json": `${rel}/src.react-native.json`, + }; + let needed = false; + for (const file of Object.keys(targets)) { + if (!fs.existsSync(path.join(configDir, file))) { needed = true; break; } + } + if (needed) { + fs.mkdirSync(configDir, { recursive: true }); + for (const [file, ext] of Object.entries(targets)) { + const fp = path.join(configDir, file); + if (!fs.existsSync(fp)) { + fs.writeFileSync(fp, JSON.stringify({ extends: ext, include: ["../src/index.ts"] }, null, 2) + "\n"); + } + } + count++; + } + } else if (depth < 2) { + scan(full, depth + 1); + } + } + } + scan(sdkDir, 0); + if (count > 0) console.log(` Scaffolded config/ for ${count} packages`); +} + +function patchMissingDependencies(root, successPkgs) { + // The dev emitter sometimes generates code importing @azure/logger etc. + // without adding them to package.json dependencies. Also patches missing + // devDependencies for warp build targets (e.g., react-native). + const knownVersions = { + "@azure/logger": "^1.2.0", + "@azure/core-util": "^1.12.0", + "@azure/core-lro": "^3.1.0", + "@azure/core-paging": "^1.6.2", + "@azure/abort-controller": "^2.1.2", + }; + const knownDevDeps = { + "react-native": "catalog:testing", + }; + let count = 0; + + for (const pkg of successPkgs) { + const pkgDir = path.join(root, pkg.pkg); + const pkgJsonPath = path.join(pkgDir, "package.json"); + if (!fs.existsSync(pkgJsonPath)) continue; + + // Scan .ts files for @azure/* imports + const srcDir = path.join(pkgDir, "src"); + const imports = new Set(); + function scanTs(dir) { + let entries; + try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; } + for (const e of entries) { + const full = path.join(dir, e.name); + if (e.isDirectory()) { scanTs(full); continue; } + if (!e.name.endsWith(".ts")) continue; + try { + const content = fs.readFileSync(full, "utf8"); + const matches = content.matchAll(/from\s+"(@azure[^"]+)"/g); + for (const m of matches) imports.add(m[1]); + } catch {} + } + } + scanTs(srcDir); + + const pkgJson = JSON.parse(fs.readFileSync(pkgJsonPath, "utf8")); + let patched = false; + + // Patch missing runtime dependencies + const deps = pkgJson.dependencies || {}; + for (const imp of imports) { + if (!deps[imp] && knownVersions[imp]) { + deps[imp] = knownVersions[imp]; + patched = true; + } + } + pkgJson.dependencies = deps; + + // Patch missing devDependencies for warp targets + const warpPath = path.join(pkgDir, "warp.config.yml"); + if (fs.existsSync(warpPath)) { + const warpContent = fs.readFileSync(warpPath, "utf8"); + const devDeps = pkgJson.devDependencies || {}; + for (const [dep, ver] of Object.entries(knownDevDeps)) { + if (warpContent.includes(dep) && !devDeps[dep]) { + devDeps[dep] = ver; + patched = true; + } + } + pkgJson.devDependencies = devDeps; + } + + if (patched) { + fs.writeFileSync(pkgJsonPath, JSON.stringify(pkgJson, null, 2) + "\n"); + count++; + } + } + if (count > 0) console.log(` Patched dependencies in ${count} packages`); +} + +function detectNestedDuplicateWorkspaces() { + // Find package.json files at nested paths like sdk/a/b/sdk/c/d/package.json + const nestedPattern = /^sdk[/\\][^/\\]+[/\\][^/\\]+[/\\]sdk[/\\]/; + function findNested(dir, results) { + let entries; + try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return results; } + for (const entry of entries) { + if (entry.name === "node_modules" || entry.name === ".git") continue; + const full = path.join(dir, entry.name); + if (entry.isDirectory()) { + findNested(full, results); + } else if (entry.name === "package.json") { + const rel = normalizePath(path.relative(sdkRoot, full)); + if (nestedPattern.test(rel)) results.push(rel); + } + } + return results; + } + const packageJsonPaths = findNested(path.join(sdkRoot, "sdk"), []); + + if (packageJsonPaths.length === 0) { + console.log("Pre-flight: no nested duplicate workspaces found."); + return; + } + + console.log("Pre-flight: nested duplicate workspace directories found:"); + for (const packageJsonPath of packageJsonPaths) { + console.log(` - ${path.dirname(packageJsonPath)}`); + } + throw new Error("Nested duplicate workspaces will make pnpm/turbo fail. Remove them before running the pipeline."); +} + +// Resolve one SDK directory's spec location from its tsp-location.yaml. +// Returns { ok: true, package } if the SDK has a usable tsp-location.yaml, +// or { ok: false, reason } if it should be reported as skipped in the PR. +function resolvePackageFromTspLocation(sdkDir) { + const dir = normalizePath(sdkDir); + const pkgDir = path.join(sdkRoot, sdkDir); + const tspLocationPath = path.join(pkgDir, "tsp-location.yaml"); + if (!fs.existsSync(tspLocationPath)) { + return { ok: false, reason: "no tsp-location.yaml" }; + } + const parsed = readTspLocation(tspLocationPath); + if (parsed.error) { + return { ok: false, reason: parsed.error }; + } + const localSpecPath = normalizePath(path.join(specRepoRoot, parsed.directory)); + const tspConfigPath = path.join(localSpecPath, "tspconfig.yaml"); + if (!fs.existsSync(tspConfigPath)) { + return { ok: false, reason: `stale path: ${parsed.directory} (tspconfig.yaml not in cloned spec)` }; + } + return { + ok: true, + package: { + pkg: dir, + pkgDir, + tspConfigPath, + localSpecPath, + source: "tsp-location", + tspLocationDirectory: parsed.directory, + tspLocationCommit: parsed.commit, + tspLocationRepo: parsed.repo, + }, + }; +} + +// Single-job mode: scan sdk/ for arm-* dirs and resolve each via tsp-location.yaml. +// Returns { packages, skippedNoTspLocation } — packages without a usable +// tsp-location.yaml are surfaced for the PR description, not silently dropped. +function classifyPackages() { + function findArmDirs(dir, results) { + let entries; + try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return results; } + for (const entry of entries) { + if (entry.name === "node_modules" || entry.name === ".git") continue; + const full = path.join(dir, entry.name); + if (entry.isDirectory()) { + if (entry.name.startsWith("arm-")) { + results.push(normalizePath(path.relative(sdkRoot, full))); + } else { + findArmDirs(full, results); + } + } + } + return results; + } + const allArmDirs = findArmDirs(path.join(sdkRoot, "sdk"), []).sort(); + + const packages = []; + const skippedNoTspLocation = []; + + for (const dir of allArmDirs) { + const r = resolvePackageFromTspLocation(dir); + if (r.ok) packages.push(r.package); + else skippedNoTspLocation.push({ name: normalizePath(dir), reason: r.reason }); + } + + console.log(""); + console.log(` ✅ Will run: ${packages.length} packages (resolved via tsp-location.yaml)`); + if (skippedNoTspLocation.length > 0) { + console.log(` ⚠️ Skipped: ${skippedNoTspLocation.length} arm-* packages without a usable tsp-location.yaml`); + console.log(` (these are surfaced in the PR description for follow-up on-boarding)`); + for (const item of skippedNoTspLocation) console.log(` [SKIP] ${item.name} — ${item.reason}`); + } + console.log(""); + + return { packages, skippedNoTspLocation }; +} + +// Matrix mode: caller passes a directoryList JSON (entries like +// "advisor/arm-advisor"). We resolve each entry the same way as single-job +// mode (tsp-location.yaml only). When matrix gen runs with `-OnlyTypeSpec +// true`, every entry in the list will already have tsp-location.yaml — the +// skipped list normally stays empty here. The list is still computed (and +// emitted) so an edge-case (e.g. yaml gets deleted between matrix-gen and +// shard run) is visible in result.json. +function classifyFromDirectoryList(directoryListPath) { + console.log(`Reading directory list from: ${directoryListPath}`); + const entries = JSON.parse(fs.readFileSync(directoryListPath, "utf8")); + if (!Array.isArray(entries) || entries.length === 0) { + console.error("ERROR: directoryList is empty or not an array"); + process.exit(1); + } + console.log(` Directory list contains ${entries.length} packages`); + + const packages = []; + const skippedNoTspLocation = []; + + for (const entry of entries) { + if (entry.includes("..") || path.isAbsolute(entry)) { + console.error(`ERROR: invalid directory list entry (must be relative): ${entry}`); + process.exit(1); + } + + const sdkDir = normalizePath(path.join("sdk", entry)); + if (!fs.existsSync(path.join(sdkRoot, sdkDir))) { + skippedNoTspLocation.push({ name: sdkDir, reason: "directory does not exist" }); + continue; + } + const r = resolvePackageFromTspLocation(sdkDir); + if (r.ok) packages.push(r.package); + else skippedNoTspLocation.push({ name: sdkDir, reason: r.reason }); + } + + console.log(` ✅ Will run: ${packages.length} packages`); + if (skippedNoTspLocation.length > 0) { + console.log(` ⚠️ Skipped: ${skippedNoTspLocation.length} packages without a usable tsp-location.yaml`); + for (const item of skippedNoTspLocation) console.log(` [SKIP] ${item.name} — ${item.reason}`); + } + console.log(""); + + return { packages, skippedNoTspLocation }; +} + +async function regenerateAll(packages) { + console.log("===== Step 4: Regeneration (tsp-client init --update-if-exists) ====="); + + // Set npm config to avoid ERESOLVE errors during tsp-client's internal npm install + try { + execSync("npm config set legacy-peer-deps true", { encoding: "utf8" }); + console.log("Set npm legacy-peer-deps=true to avoid ERESOLVE conflicts"); + } catch (e) { + console.log("Warning: failed to set npm config:", e.message); + } + + const results = []; + let completed = 0; + let activePromises = []; + + async function processPackage(pkg) { + const start = Date.now(); + let output = ""; + let success = false; + + output += `Package dir : ${pkg.pkgDir}\n`; + output += `Spec source : ${pkg.source}\n`; + output += `tsp-config : ${pkg.tspConfigPath}\n`; + output += `local-spec : ${pkg.localSpecPath}\n`; + + if (!fs.existsSync(pkg.tspConfigPath)) { + output += `ERROR: tspconfig.yaml not found at ${pkg.tspConfigPath} (spec may have been renamed/deleted)\n`; + completed++; + const duration = ((Date.now() - start) / 1000).toFixed(1); + console.log(` ❌ [${completed}/${packages.length}] ${pkg.pkg} - FAILED (${duration}s)`); + console.log(` ${extractError(output)}`); + recordPackageLog("regenerate", pkg.pkg, false, output); + results.push({ pkg: pkg.pkg, success: false, duration, output }); + return; + } + + // Use `tsp-client init --update-if-exists` with the latest main spec from + // the locally cloned azure-rest-api-specs repo. The emitter runs against + // whatever api-version the spec's tspconfig.yaml declares by default, so + // the diff reflects the combined (emitter + spec-evolution) signal. + // (api-version pinning machinery was removed after the A/B test in PR + // #38604 showed it didn't materially reduce breaking-change noise — only + // 1 package differed between pinned and unpinned runs out of 95.) + // Single attempt, no retry — aligned with azure-sdk-for-net / azure-sdk-for-go. + const tspClientArgs = ["init", "--update-if-exists", "-c", pkg.tspConfigPath, "--local-spec-repo", pkg.localSpecPath, "--debug"]; + output += `tsp-client : init --local-spec-repo (latest main)\n`; + + const result = await runCommand("tsp-client", tspClientArgs, pkg.pkgDir); + output += `\n${result.output}`; + output += `\nExit code: ${result.code}\n`; + success = result.code === 0; + + const duration = ((Date.now() - start) / 1000).toFixed(1); + completed++; + if (success) { + console.log(` ✅ [${completed}/${packages.length}] ${pkg.pkg} - SUCCESS (${duration}s)`); + } else { + console.log(` ❌ [${completed}/${packages.length}] ${pkg.pkg} - FAILED (${duration}s)`); + console.log(` ${extractError(output)}`); + } + recordPackageLog("regenerate", pkg.pkg, success, output); + results.push({ pkg: pkg.pkg, success, duration, output }); + } + + for (const pkg of packages) { + const promise = processPackage(pkg).then(() => { + activePromises = activePromises.filter((p) => p !== promise); + }); + activePromises.push(promise); + if (activePromises.length >= maxWorkers) await Promise.race(activePromises); + } + await Promise.allSettled(activePromises); + + // Same ADO group-collapse sentinel as in buildAll. Most of the time the + // next phase's banner "===== Step 5 =====" already triggers ADO to close + // the last regenerate group, but when build is skipped (--skipBuild, or + // all regens failed) the last regenerate group would otherwise be stuck + // in "open / not collapsible" state. + console.log("##[section]All per-package regenerate logs complete"); + + return results; +} + +async function buildAll(regenResults) { + const successPkgs = regenResults.filter((result) => result.success); + if (skipBuild || successPkgs.length === 0) { + if (skipBuild) console.log("Build verification skipped (SkipBuild=true)"); + return { buildResults: [], skipped: true }; + } + + console.log(""); + console.log("===== Step 5: Build Verification (pnpm build --filter) ====="); + console.log(`Building ${successPkgs.length} packages with ${buildWorkers} workers`); + console.log(""); + + // Pre-build fix 1: Clean up TempTypeSpecFiles to avoid turbo "duplicate workspace" errors + console.log("Cleaning up TempTypeSpecFiles directories..."); + cleanupTempTypeSpecFiles(sdkRoot); + + // Pre-build fix 1b: Clean up nested duplicate workspaces created by tsp-client + // (e.g. sdk/foo/arm-foo/sdk/foo/arm-foo/package.json), which would otherwise + // make pnpm install reject the whole workspace. + console.log("Cleaning up nested duplicate workspace directories..."); + const nestedRemoved = cleanupNestedDuplicateWorkspaces(sdkRoot); + if (nestedRemoved === 0) { + console.log(" No nested duplicate workspaces found."); + } + + // Pre-build fix 2: Scaffold missing config/tsconfig files for warp-enabled packages + console.log("Scaffolding missing config/tsconfig files..."); + scaffoldWarpConfigs(sdkRoot); + + // Pre-build fix 3: Patch missing dependencies in package.json + console.log("Checking for missing dependencies in generated packages..."); + patchMissingDependencies(sdkRoot, successPkgs); + + console.log("Running pnpm install at repo root..."); + const pnpmInstall = await runCommand("pnpm", ["install", "--no-frozen-lockfile"], sdkRoot); + if (pnpmInstall.code !== 0) { + console.log("ERROR: pnpm install failed. Cannot proceed with build verification."); + console.log(pnpmInstall.output.slice(-1000)); + return { buildResults: [], skipped: true }; + } + console.log("pnpm install completed"); + + // Install the official changelog tool once for the whole shard, so each + // successful build can immediately run update-changelog. + await installChangelogTool(); + + // Pre-build fix 4: Build core dependencies first + console.log("Pre-building core dependencies..."); + const coreFilters = [ + "@azure/core-rest-pipeline", "@azure/core-client", "@azure/core-auth", + "@azure/core-lro", "@azure/logger", "@azure/core-paging" + ]; + const coreArgs = ["build"]; + for (const f of coreFilters) { coreArgs.push("--filter", f); } + const coreBuild = await runCommand("pnpm", coreArgs, sdkRoot); + if (coreBuild.code !== 0) { + console.log("WARNING: Core dependencies build had issues:"); + console.log(coreBuild.output.slice(-2000)); + } else { + console.log("Core dependencies built"); + } + + const buildResults = []; + let buildCompleted = 0; + let activePromises = []; + const buildTotal = successPkgs.length; + + async function buildPackage(pkg) { + const pkgDir = path.join(sdkRoot, pkg.pkg); + let filterName = pkg.pkg; + try { + const packageJson = JSON.parse(fs.readFileSync(path.join(pkgDir, "package.json"), "utf8")); + if (packageJson.name) filterName = packageJson.name; + } catch {} + + const start = Date.now(); + const build = await runCommand("pnpm", ["build", "--filter", filterName], sdkRoot); + const duration = ((Date.now() - start) / 1000).toFixed(1); + buildCompleted++; + + let changelog = null; + if (build.code === 0) { + console.log(` ✅ [BUILD ${buildCompleted}/${buildTotal}] ${pkg.pkg} - BUILD OK (${duration}s)`); + // Comment #5: run the official update-changelog for every package whose + // build succeeded. Changelog failures are logged but never demote the + // build status — the regen+build matrix is the gate, the changelog is a + // signal. + const cl = await runUpdateChangelog(pkg.pkg); + changelog = { success: cl.success, hasBreaking: cl.hasBreaking, hasChanges: cl.hasChanges }; + if (cl.success) { + const marker = cl.hasBreaking ? "⚠️ breaking" : cl.hasChanges ? "changes" : "no changes"; + console.log(` changelog: ${marker}`); + } else { + console.log(` changelog: FAILED (see log)`); + } + recordPackageLog("changelog", pkg.pkg, cl.success, cl.output); + buildResults.push({ pkg: pkg.pkg, success: true, duration, phase: "done", output: build.output, changelog }); + } else { + console.log(` ❌ [BUILD ${buildCompleted}/${buildTotal}] ${pkg.pkg} - BUILD FAILED (${duration}s)`); + console.log(` ${extractError(build.output)}`); + buildResults.push({ pkg: pkg.pkg, success: false, duration, phase: "pnpm build", output: build.output, changelog: null }); + } + recordPackageLog("build", pkg.pkg, build.code === 0, build.output); + } + + for (const pkg of successPkgs) { + const promise = buildPackage(pkg).then(() => { + activePromises = activePromises.filter((p) => p !== promise); + }); + activePromises.push(promise); + if (activePromises.length >= buildWorkers) await Promise.race(activePromises); + } + await Promise.allSettled(activePromises); + + // ADO logging quirk: a ##[group] only renders as a collapsible block when + // there is following non-group content as an anchor. Without this sentinel + // the very last build package's group stays in "open" state and the user + // cannot collapse it. Emitting a ##[section] line acts as a hard boundary + // and forces ADO to close (and thus make collapsible) all preceding groups. + console.log("##[section]All per-package build logs complete"); + + return { buildResults, skipped: false }; +} + +async function main() { + detectNestedDuplicateWorkspaces(); + + console.log("===== Step 3: Resolve spec paths via tsp-location.yaml ====="); + let packages; + let skippedNoTspLocation; + + if (directoryListFile) { + // Matrix mode: package list provided by GenerateMatrix job + console.log("(Matrix mode: using --directoryList)"); + ({ packages, skippedNoTspLocation } = classifyFromDirectoryList(directoryListFile)); + } else { + // Single-job mode: scan all ARM packages + ({ packages, skippedNoTspLocation } = classifyPackages()); + if (maxPackages > 0) { + packages = packages.slice(0, maxPackages); + console.log(`Limited to first ${maxPackages} packages`); + } + } + + console.log(`Processing ${packages.length} packages with ${maxWorkers} regen workers`); + console.log(`Build workers: ${buildWorkers}`); + console.log(""); + + const regenResults = await regenerateAll(packages); + const { buildResults, skipped: buildSkipped } = await buildAll(regenResults); + + const regenSuccess = regenResults.filter((result) => result.success).length; + const regenFail = regenResults.filter((result) => !result.success).length; + const buildOk = buildResults.filter((result) => result.success).length; + const buildFail = buildResults.filter((result) => !result.success).length; + + console.log(""); + console.log("========== SUMMARY =========="); + if (emitterVersion) console.log(`Emitter: @azure-tools/typespec-ts@${emitterVersion}`); + console.log(""); + console.log("--- Regeneration (tsp-client init --update-if-exists) ---"); + console.log(`Total: ${packages.length} | Success: ${regenSuccess} | Failed: ${regenFail}`); + console.log(`Success Rate: ${packages.length === 0 ? "0.0" : ((regenSuccess * 100) / packages.length).toFixed(1)}%`); + + if (!buildSkipped && regenSuccess > 0) { + console.log(""); + console.log("--- Build Verification (pnpm build) ---"); + console.log(`Total: ${regenSuccess} | Build OK: ${buildOk} | Build Failed: ${buildFail}`); + console.log(`Build Pass Rate: ${((buildOk * 100) / regenSuccess).toFixed(1)}%`); + } + + if (regenFail > 0) { + console.log(""); + console.log("Regeneration failures:"); + for (const result of regenResults.filter((r) => !r.success)) { + console.log(` - ${result.pkg}: ${extractError(result.output)}`); + } + } + if (buildFail > 0) { + console.log(""); + console.log("Build failures:"); + for (const result of buildResults.filter((r) => !r.success)) { + console.log(` - ${result.pkg} (failed at ${result.phase}): ${extractError(result.output)}`); + } + } + console.log("============================="); + + // Write result summary as JSON for artifact collection in matrix mode + if (resultOutputDir) { + const resultSummary = { + emitterVersion, + directoryList: directoryListFile || null, + packages: packages.map((p) => p.pkg), + // Comment #3: packages without a usable tsp-location.yaml are surfaced + // here so the Summary stage can list them in the PR description for + // on-boarding follow-up by the spec/SDK team. + skippedNoTspLocation: skippedNoTspLocation || [], + regeneration: { + total: packages.length, + success: regenSuccess, + failed: regenFail, + failures: regenResults.filter((r) => !r.success).map((r) => ({ + pkg: r.pkg, + error: extractError(r.output), + })), + }, + build: buildSkipped ? null : { + total: regenSuccess, + success: buildOk, + failed: buildFail, + failures: buildResults.filter((r) => !r.success).map((r) => ({ + pkg: r.pkg, + phase: r.phase, + error: extractError(r.output), + })), + }, + // Comment #5: changelog signal sourced from the official update-changelog + // tool. `breakingPackages` is the simple actionable list reviewers care + // about — they can read each package's CHANGELOG.md in the PR diff for + // detail. + changelog: buildSkipped ? null : (() => { + const withChangelog = buildResults.filter((r) => r.success && r.changelog); + const generated = withChangelog.filter((r) => r.changelog.success); + const breaking = generated.filter((r) => r.changelog.hasBreaking); + const changes = generated.filter((r) => r.changelog.hasChanges); + const failed = withChangelog.filter((r) => !r.changelog.success).map((r) => r.pkg); + return { + total: withChangelog.length, + generated: generated.length, + failed: failed.length, + withChanges: changes.length, + withBreaking: breaking.length, + breakingPackages: breaking.map((r) => r.pkg), + failedPackages: failed, + }; + })(), + }; + + if (!fs.existsSync(resultOutputDir)) { + fs.mkdirSync(resultOutputDir, { recursive: true }); + } + const resultFile = path.join(resultOutputDir, "result.json"); + fs.writeFileSync(resultFile, JSON.stringify(resultSummary, null, 2)); + console.log(`Result summary written to: ${resultFile}`); + } + + if (regenFail > 0 || buildFail > 0) process.exit(1); +} + +main().catch((error) => { + console.error(error); + process.exit(1); +}); diff --git a/eng/pipelines/sdk-regenerate.yml b/eng/pipelines/sdk-regenerate.yml new file mode 100644 index 000000000000..b1c964120bf4 --- /dev/null +++ b/eng/pipelines/sdk-regenerate.yml @@ -0,0 +1,755 @@ +# JS SDK TypeSpec Breaking Change Check Pipeline (ARM Only) +# Purpose: Validate that the latest emitter (@azure-tools/typespec-ts) doesn't break existing ARM SDK packages +# Flow: Fetch latest emitter version → update emitter-package.json → regenerate lock file +# → clone latest azure-rest-api-specs (local spec source) +# → tsp-client init --update-if-exists -c --local-spec-repo (concurrent) +# → pnpm build (concurrent) + +trigger: + branches: + include: + - main + paths: + include: + - eng/emitter-package.json + - eng/common/scripts/TypeSpec-* + - eng/common/tsp-client/ + - eng/pipelines/sdk-regenerate.yml + - eng/pipelines/scripts/regenerate-runner.js + +pr: + branches: + include: + - main + - feature/* + - hotfix/* + paths: + include: + - eng/emitter-package.json + - eng/common/scripts/TypeSpec-* + - eng/common/tsp-client/ + - eng/pipelines/sdk-regenerate.yml + - eng/pipelines/scripts/regenerate-runner.js + +schedules: + - cron: "0 2 * * 1" + displayName: Weekly TypeSpec break check (Monday 2AM UTC) + branches: + include: + - main + +parameters: + - name: EmitterVersion + displayName: "Emitter version to test (use 'empty' or leave default for latest dev tag)" + type: string + default: 'empty' + - name: MaxWorkers + displayName: "Regenerate concurrent workers" + type: number + default: 2 + - name: BuildWorkers + displayName: "Build concurrent workers" + type: number + default: 2 + - name: RegenerationJobCount + displayName: "Matrix jobs for regeneration" + type: number + default: 10 + - name: MinimumPerJob + displayName: "Minimum packages per matrix job" + type: number + default: 10 + - name: SkipBuild + displayName: "Skip build verification" + type: boolean + default: false + - name: QuickTest + displayName: "Quick test mode (only first 3 valid packages: arm-alertprocessingrules, arm-alertrulerecommendations, arm-alertsmanagement). Overrides Package filter, Matrix jobs, Minimum per job." + type: boolean + default: false + - name: PackageFilter + displayName: "Package filter (wildcard, comma-separated). Ignored when Quick test mode is checked. 'arm-*' = all" + type: string + default: 'arm-*' + - name: SpecRepoBranch + displayName: "azure-rest-api-specs branch to clone" + type: string + default: 'main' + - name: CreatePullRequest + displayName: "Create PR with generated changes" + type: boolean + default: false + - name: ChangePushMode + displayName: "Changes included in PR commit" + type: string + default: 'all' + values: + - all + - api-md + - name: PullRequestTargetBranch + displayName: "PR target branch" + type: string + default: 'main' + - name: PullRequestBranch + displayName: "PR source branch (empty = auto)" + type: string + default: '' + - name: PullRequestRepoOwner + displayName: "PR target repo OWNER (empty = same as source repo, e.g. 'wxl534' for fork)" + type: string + default: '' + - name: PullRequestRepoName + displayName: "PR target repo NAME (empty = same as source repo name)" + type: string + default: '' + - name: ForkTokenVariableName + displayName: "ADO secret variable name holding the GitHub PAT (only needed when pushing to a fork)" + type: string + default: '' + +pool: + vmImage: 'ubuntu-latest' + +variables: + sdkRoot: $(Build.SourcesDirectory) + specRepoRoot: $(Agent.BuildDirectory)/azure-rest-api-specs + nodeVersion: '22.x' + +stages: + - stage: BreakCheck + displayName: 'TypeSpec emitter breaking-change check' + jobs: + - job: Setup + displayName: 'Setup' + steps: + # Step 1: Install Node.js + - task: NodeTool@0 + displayName: 'Step 1: Install Node.js' + inputs: + versionSpec: $(nodeVersion) + + # Step 2: Resolve emitter version + - script: | + set -euo pipefail + + # Treat empty / common sentinel values ('empty', 'latest', 'auto', 'dev') + # as "use the latest dev tag from npm", because the ADO manual-run UI + # forces string params to be non-empty. + RAW_INPUT='${{ parameters.EmitterVersion }}' + NORMALIZED=$(echo "$RAW_INPUT" | tr '[:upper:]' '[:lower:]' | xargs) + if [ -z "$NORMALIZED" ] || [ "$NORMALIZED" = "empty" ] || [ "$NORMALIZED" = "latest" ] || [ "$NORMALIZED" = "auto" ] || [ "$NORMALIZED" = "dev" ]; then + EMITTER_VERSION=$(npm view @azure-tools/typespec-ts dist-tags.dev) + if [ -z "$EMITTER_VERSION" ]; then + echo "##[error]npm view returned empty dev tag for @azure-tools/typespec-ts" + exit 1 + fi + echo "Resolved from npm dev tag: $EMITTER_VERSION" + else + EMITTER_VERSION="$RAW_INPUT" + echo "Using specified version: $EMITTER_VERSION" + fi + + echo "##vso[task.setvariable variable=emitterVersion;isOutput=true]$EMITTER_VERSION" + displayName: 'Step 2: Resolve emitter version' + name: resolveVersion + + - job: GenerateMatrix + displayName: 'Generate package matrix' + dependsOn: Setup + variables: + ${{ if eq(parameters.QuickTest, true) }}: + effJobCount: 3 + effMinPerJob: 1 + effFilter: 'arm-alertprocessingrules,arm-alertrulerecommendations,arm-alertsmanagement' + ${{ else }}: + effJobCount: ${{ parameters.RegenerationJobCount }} + effMinPerJob: ${{ parameters.MinimumPerJob }} + effFilter: ${{ parameters.PackageFilter }} + steps: + # Builds the matrix and records arm-* dirs without tsp-location.yaml + # (filtered out of the matrix, surfaced later in the PR description). + - task: PowerShell@2 + displayName: 'Build matrix (split + scan skipped)' + name: generateMatrix + inputs: + pwsh: true + targetType: inline + workingDirectory: $(Build.SourcesDirectory) + script: | + $matrixOutDir = "$(Build.ArtifactStagingDirectory)/matrix" + + Write-Host "===== Split ARM packages into matrix batches =====" + & "$(Build.SourcesDirectory)/eng/common/scripts/New-RegenerateMatrix.ps1" ` + -OutputDirectory $matrixOutDir ` + -OutputVariableName matrix ` + -JobCount $(effJobCount) ` + -MinimumPerJob $(effMinPerJob) ` + -OnlyTypeSpec $true ` + -DirectoryFilterPattern '$(effFilter)' + + Write-Host "===== Scan ARM packages without tsp-location.yaml =====" + $srcRoot = "$(Build.SourcesDirectory)" + $sdkDir = Join-Path $srcRoot 'sdk' + $missing = @() + if (Test-Path $sdkDir) { + Get-ChildItem -Path $sdkDir -Directory -Recurse -Filter 'arm-*' -ErrorAction SilentlyContinue | + ForEach-Object { + $tspLoc = Join-Path $_.FullName 'tsp-location.yaml' + if (-not (Test-Path $tspLoc)) { + $relative = ($_.FullName.Substring($srcRoot.Length + 1)) -replace '\\','/' + $missing += @{ name = $relative; reason = 'no tsp-location.yaml' } + } + } + } + $outPath = Join-Path $matrixOutDir 'skipped-no-tsp-location.json' + New-Item -ItemType Directory -Force -Path (Split-Path $outPath) | Out-Null + ConvertTo-Json @($missing) -Depth 3 | Out-File -FilePath $outPath -Encoding utf8 + Write-Host "Wrote $($missing.Count) skipped packages to $outPath" + + - task: PublishPipelineArtifact@1 + displayName: 'Publish matrix artifacts' + inputs: + targetPath: $(Build.ArtifactStagingDirectory)/matrix + artifact: matrix_artifacts + + - job: RegenerateAndBuild + displayName: 'RegenerateAndBuild' + dependsOn: + - Setup + - GenerateMatrix + timeoutInMinutes: 180 + strategy: + matrix: $[ dependencies.GenerateMatrix.outputs['generateMatrix.matrix'] ] + variables: + emitterVersion: $[ dependencies.Setup.outputs['resolveVersion.emitterVersion'] ] + steps: + - task: NodeTool@0 + displayName: 'Install Node.js' + inputs: + versionSpec: $(nodeVersion) + + - download: current + displayName: 'Download matrix artifacts' + artifact: matrix_artifacts + + # Installs tsp-client + pnpm, shallow-clones azure-rest-api-specs + # (tsp-client init --local-spec-repo only needs current state), and + # updates emitter-package.json + lock file to pin the candidate + # emitter version. + - script: | + set -euo pipefail + + echo "===== Install tsp-client and pnpm =====" + npm install -g @azure-tools/typespec-client-generator-cli + npm install -g pnpm + npm config set legacy-peer-deps true + + echo "===== Clone azure-rest-api-specs (local spec source) =====" + if [ -d "$(specRepoRoot)/.git" ]; then + echo "Spec repo already exists at $(specRepoRoot), pulling latest..." + cd $(specRepoRoot) + git fetch origin ${{ parameters.SpecRepoBranch }} --depth 1 + git checkout ${{ parameters.SpecRepoBranch }} + git reset --hard origin/${{ parameters.SpecRepoBranch }} + else + echo "Cloning azure-rest-api-specs (branch ${{ parameters.SpecRepoBranch }}, shallow)..." + git clone --depth 1 --branch ${{ parameters.SpecRepoBranch }} \ + https://github.com/Azure/azure-rest-api-specs.git $(specRepoRoot) + fi + echo "Spec repo ready at $(specRepoRoot)" + cd $(specRepoRoot) && git log -1 --oneline + + echo "===== Update emitter + lock file =====" + cd $(sdkRoot) + node -e " + const fs = require('fs'); + const pkg = JSON.parse(fs.readFileSync('eng/emitter-package.json', 'utf8')); + pkg.dependencies['@azure-tools/typespec-ts'] = '$(emitterVersion)'; + // Inject transitive peer deps required by typespec-client-generator-core + // that main's emitter-package.json sometimes omits. Pin to the same + // version as @typespec/events (sibling package, always present). + // Without these, tsp-client init fails with 'Cannot find package @typespec/xml'. + const tspVer = pkg.dependencies['@typespec/events'] || '0.82.0'; + if (!pkg.dependencies['@typespec/xml']) pkg.dependencies['@typespec/xml'] = tspVer; + if (!pkg.dependencies['@typespec/sse']) pkg.dependencies['@typespec/sse'] = tspVer; + fs.writeFileSync('eng/emitter-package.json', JSON.stringify(pkg, null, 2) + '\n'); + " + cd eng + cp emitter-package.json package.json + npm install --package-lock-only --ignore-scripts 2>/dev/null + if [ -f package-lock.json ]; then + cp package-lock.json emitter-package-lock.json + rm -f package.json package-lock.json + rm -rf node_modules + fi + displayName: 'Prepare workspace (deps + specs + emitter)' + + # Step 3: Regenerate (tsp-client init) → build (pnpm build) for this batch + - script: | + cd $(sdkRoot) + mkdir -p "$(Build.ArtifactStagingDirectory)/results/$(JobKey)" + node eng/pipelines/scripts/regenerate-runner.js \ + --specRepoRoot "$(specRepoRoot)" \ + --maxWorkers "${{ parameters.MaxWorkers }}" \ + --buildWorkers "${{ parameters.BuildWorkers }}" \ + --skipBuild "${{ parameters.SkipBuild }}" \ + --emitterVersion "$(emitterVersion)" \ + --directoryList "$(Pipeline.Workspace)/matrix_artifacts/$(DirectoryList)" \ + --resultOutputDir "$(Build.ArtifactStagingDirectory)/results/$(JobKey)" + displayName: 'Regenerate & build ARM package batch' + + # Shows the per-batch diff; if CreatePullRequest is on, also writes + # changes.patch (filtered by ChangePushMode) for the PR job to apply. + - script: | + cd $(sdkRoot) + echo "===== Show generated changes =====" + git diff --stat + CREATE_PR="${{ parameters.CreatePullRequest }}" + if [ "$CREATE_PR" = "True" ] || [ "$CREATE_PR" = "true" ]; then + echo "===== Create patch for PR job =====" + PATCH_FILE="$(Build.ArtifactStagingDirectory)/results/$(JobKey)/changes.patch" + MODE="${{ parameters.ChangePushMode }}" + if [ "$MODE" = "api-md" ]; then + git diff --binary -- ':(glob)sdk/**/review/*.api.md' ':(glob)sdk/**/CHANGELOG.md' > "$PATCH_FILE" + else + git diff --binary -- sdk/ > "$PATCH_FILE" + fi + echo "Patch mode: $MODE" + echo "Patch file: $PATCH_FILE" + wc -l "$PATCH_FILE" || true + fi + displayName: 'Capture changes (diff + patch)' + condition: always() + + - task: PublishPipelineArtifact@1 + displayName: 'Publish batch result' + condition: always() + inputs: + targetPath: $(Build.ArtifactStagingDirectory)/results/$(JobKey) + artifact: regen_result_$(JobKey) + + - job: Summary + displayName: 'Summarize matrix results' + dependsOn: + - Setup + - RegenerateAndBuild + condition: succeededOrFailed() + variables: + emitterVersion: $[ dependencies.Setup.outputs['resolveVersion.emitterVersion'] ] + steps: + - task: NodeTool@0 + displayName: 'Install Node.js' + inputs: + versionSpec: $(nodeVersion) + + - download: current + displayName: 'Download result artifacts' + + - script: | + node - <<'NODE' + const fs = require("fs"); + const path = require("path"); + + const root = process.env.PIPELINE_WORKSPACE; + const resultFiles = []; + (function walk(dir) { + if (!fs.existsSync(dir)) return; + for (const e of fs.readdirSync(dir, { withFileTypes: true })) { + const full = path.join(dir, e.name); + if (e.isDirectory()) walk(full); + else if (e.name === "result.json") resultFiles.push(full); + } + })(root); + + const t = { + packages: 0, + regen: { total: 0, success: 0, failed: 0, failures: [] }, + build: { total: 0, success: 0, failed: 0, failures: [] }, + changelog: { total: 0, generated: 0, failed: 0, withChanges: 0, withBreaking: 0, breakingPackages: [], failedPackages: [] }, + skippedNoTspLocation: [], + }; + const addNums = (dst, src, keys) => { if (src) for (const k of keys) dst[k] += src[k] ?? 0; }; + + for (const file of resultFiles) { + const r = JSON.parse(fs.readFileSync(file, "utf8")); + t.packages += r.packages?.length ?? 0; + addNums(t.regen, r.regeneration, ["total","success","failed"]); + addNums(t.build, r.build, ["total","success","failed"]); + addNums(t.changelog, r.changelog, ["total","generated","failed","withChanges","withBreaking"]); + t.regen.failures.push(...(r.regeneration?.failures ?? [])); + t.build.failures.push(...(r.build?.failures ?? [])); + t.changelog.breakingPackages.push(...(r.changelog?.breakingPackages ?? [])); + t.changelog.failedPackages.push(...(r.changelog?.failedPackages ?? [])); + if (Array.isArray(r.skippedNoTspLocation)) t.skippedNoTspLocation.push(...r.skippedNoTspLocation); + } + + // Setup-stage scan catches arm-* dirs filtered out before matrix (no tsp-location.yaml). + const setupSkipped = path.join(root, "matrix_artifacts", "skipped-no-tsp-location.json"); + if (fs.existsSync(setupSkipped)) { + try { + const arr = JSON.parse(fs.readFileSync(setupSkipped, "utf8")); + if (Array.isArray(arr)) t.skippedNoTspLocation.push(...arr); + } catch (err) { console.log(`Warning: ${err.message}`); } + } + + // Dedup + sort lists once, reuse for console + aggregated JSON. + const skipMap = new Map(); + for (const it of t.skippedNoTspLocation) if (!skipMap.has(it.name)) skipMap.set(it.name, it.reason); + const skipped = [...skipMap.entries()].sort(([a],[b]) => a.localeCompare(b)).map(([name, reason]) => ({ name, reason })); + const breaking = [...new Set(t.changelog.breakingPackages)].sort(); + const cgFailed = [...new Set(t.changelog.failedPackages)].sort(); + const pct = (n, d) => d === 0 ? "0.0" : (n * 100 / d).toFixed(1); + + console.log("========== MATRIX SUMMARY =========="); + console.log(`Result files: ${resultFiles.length} | Packages: ${t.packages}`); + console.log(`Regen: ${t.regen.success}/${t.regen.total} OK (${pct(t.regen.success, t.regen.total)}%) | Failed: ${t.regen.failed}`); + console.log(`Build: ${t.build.success}/${t.build.total} OK (${pct(t.build.success, t.build.total)}%) | Failed: ${t.build.failed}`); + console.log(`Changelog: ${t.changelog.generated}/${t.changelog.total} OK | Changes: ${t.changelog.withChanges} | Breaking: ${t.changelog.withBreaking} | Failed: ${t.changelog.failed}`); + + const printList = (title, items, fmt = x => x) => { + if (!items.length) return; + console.log(`\n${title} (${items.length}):`); + for (const it of items) console.log(` - ${fmt(it)}`); + }; + printList("Breaking changes (see CHANGELOG.md in PR diff)", breaking); + printList("Changelog tool failures (see logs/changelog/.log)", cgFailed); + printList("Skipped — no tsp-location.yaml (filtered by matrix gen)", skipped, ({ name, reason }) => `${name} (${reason})`); + printList("Regen failures", t.regen.failures, f => `${f.pkg}: ${f.error}`); + printList("Build failures", t.build.failures, f => `${f.pkg} (${f.phase}): ${f.error}`); + console.log("===================================="); + + const aggregated = { + emitterVersion: process.env.EMITTER_VERSION || null, + regeneration: { total: t.regen.total, success: t.regen.success, failed: t.regen.failed }, + build: { total: t.build.total, success: t.build.success, failed: t.build.failed }, + changelog: { + total: t.changelog.total, generated: t.changelog.generated, failed: t.changelog.failed, + withChanges: t.changelog.withChanges, withBreaking: t.changelog.withBreaking, + breakingPackages: breaking, failedPackages: cgFailed, + }, + skippedNoTspLocation: skipped, + }; + const outDir = path.join(process.env.BUILD_ARTIFACTSTAGINGDIRECTORY || ".", "summary"); + fs.mkdirSync(outDir, { recursive: true }); + fs.writeFileSync(path.join(outDir, "aggregated-results.json"), JSON.stringify(aggregated, null, 2)); + console.log(`Wrote aggregated-results.json (${skipped.length} skipped, ${breaking.length} breaking)`); + + if (resultFiles.length === 0 || t.regen.failed > 0 || t.build.failed > 0) process.exit(1); + NODE + displayName: 'Aggregate regeneration results' + env: + EMITTER_VERSION: $(emitterVersion) + + - task: PublishPipelineArtifact@1 + displayName: 'Publish aggregated summary' + condition: always() + inputs: + targetPath: $(Build.ArtifactStagingDirectory)/summary + artifact: regen_summary + + - ${{ if parameters.CreatePullRequest }}: + - job: CreatePR + displayName: 'Create PR with generated changes' + dependsOn: + - Setup + - Summary + condition: succeeded() + variables: + emitterVersion: $[ dependencies.Setup.outputs['resolveVersion.emitterVersion'] ] + steps: + - checkout: self + + # Only request the org-level GitHub App token when we're NOT pushing to a fork + # (otherwise the fork user's PAT is used and this template would needlessly + # require Azure Key Vault access). + - ${{ if eq(parameters.PullRequestRepoOwner, '') }}: + - template: /eng/common/pipelines/templates/steps/login-to-github.yml + parameters: + TokenOwners: + - Azure + + - download: current + displayName: 'Download result artifacts' + + - script: | + set -euo pipefail + cd $(Build.SourcesDirectory) + + # Determine target repo (defaults to the source repo when no fork override) + overrideOwner='${{ parameters.PullRequestRepoOwner }}' + overrideName='${{ parameters.PullRequestRepoName }}' + + sourceRepoFull="$(Build.Repository.Name)" + if [[ "$sourceRepoFull" == */* ]]; then + srcOwner="${sourceRepoFull%%/*}" + srcName="${sourceRepoFull##*/}" + else + srcOwner="Azure" + srcName="$sourceRepoFull" + fi + + repoOwner="${overrideOwner:-$srcOwner}" + repoName="${overrideName:-$srcName}" + isFork="false" + if [ "$repoOwner" != "$srcOwner" ] || [ "$repoName" != "$srcName" ]; then + isFork="true" + fi + + # Resolve the token to use for pushing. + # FORK_TOKEN / GH_TOKEN_VAL are injected via the env: block below so + # that ADO can correctly substitute secret variables at runtime. + if [ "$isFork" = "true" ]; then + pushToken="${FORK_TOKEN:-}" + if [ -z "$pushToken" ]; then + echo "ERROR: Pushing to fork ${repoOwner}/${repoName} but no fork PAT was provided." + echo "Set parameter ForkTokenVariableName to the name of an ADO secret variable containing a GitHub PAT with repo scope," + echo "and make sure that secret variable actually exists in this pipeline." + exit 1 + fi + echo "Pushing to fork: ${repoOwner}/${repoName} (using user-provided PAT)" + else + pushToken="${GH_TOKEN_VAL:-}" + echo "Pushing to source repo: ${repoOwner}/${repoName} (using GH App token)" + fi + + # Normalize sentinel values ('empty', 'auto', '') so that + # picking the default-named option in the ADO UI (which can't + # leave the field truly empty) still triggers the auto-branch + # logic below. Without this, branch=='empty' literally became + # the PR head ref and every run reused PR #1 (see PR #38604 + # comment thread). + branch="${{ parameters.PullRequestBranch }}" + branch_lc=$(echo "$branch" | tr '[:upper:]' '[:lower:]') + if [ -z "$branch" ] || [ "$branch_lc" = "empty" ] || [ "$branch_lc" = "auto" ]; then + branch="sdk-regenerate-$(Build.BuildId)" + fi + echo "Using PR branch: $branch" + + # Base the regeneration branch on the PR target branch (default: + # main) rather than the currently checked-out HEAD. Otherwise + # the PR would carry every commit that has accumulated on the + # pipeline-development branch (feature/break-check) since it + # forked from main — reviewers can't distinguish the actual + # regeneration delta from pipeline-development noise. By + # branching off origin/, the PR shows exactly one + # commit: the regeneration changes patched in below. + target="${{ parameters.PullRequestTargetBranch }}" + git fetch origin "$target" --depth=1 + git checkout -B "$branch" "origin/$target" + + # Track files that couldn't be applied due to upstream-vs-shard + # race conditions (e.g. an [AutoPR @azure-arm-foo] commit landed + # on `origin/` between shard checkout and CreatePR + # checkout, so the shard's patch context no longer matches HEAD + # for that file). Instead of aborting the entire PR for a few + # conflicting files, we: + # 1. let `git apply --3way` apply every hunk it can, + # 2. detect the UU (unmerged) files it left behind, + # 3. reset just those files to HEAD (drop the conflicting + # changes — they will be picked up by the NEXT pipeline + # run, which will then see the upstream commit as the + # baseline), + # 4. record the dropped files so the PR body can list them. + # This keeps the rest of the (clean) regeneration changes intact + # and turns "lose 200+ packages because 2 conflicted" into + # "lose only the 2 actually-conflicted files". + FAILED_LOG="$(Pipeline.Workspace)/failed-patches.txt" + : > "$FAILED_LOG" + mapfile -t patches < <(find "$(Pipeline.Workspace)" -type f -name changes.patch | sort) + echo "Found ${#patches[@]} patch file(s)" + for p in "${patches[@]}"; do + if [ ! -s "$p" ]; then + continue + fi + echo "Applying patch: $p" + if git apply --3way "$p"; then + continue + fi + echo "Patch had conflicts: $p" + # Collect UU (both-modified) files left by --3way. + uu_files=$(git diff --name-only --diff-filter=U) + if [ -z "$uu_files" ]; then + # No UU state means the patch was rejected outright (no + # hunks could be applied at all). Nothing in the working + # tree to clean up; just record the patch itself and move + # on so other patches can still proceed. + echo "PATCH_REJECTED $p" >> "$FAILED_LOG" + continue + fi + echo "Conflicting files (will be reset to HEAD and skipped):" + echo "$uu_files" + while IFS= read -r f; do + [ -z "$f" ] && continue + git checkout HEAD -- "$f" 2>/dev/null || git rm -f "$f" 2>/dev/null || true + echo "$f" >> "$FAILED_LOG" + done <<< "$uu_files" + done + + # Surface the conflict list in this step's log too — easier + # than digging into the PR body when triaging. + if [ -s "$FAILED_LOG" ]; then + echo "----- Files dropped due to upstream conflicts -----" + cat "$FAILED_LOG" + echo "---------------------------------------------------" + fi + + # Restore emitter-package.json / lock file so the PR only + # contains real SDK code diffs (these were temporarily bumped + # by the pipeline to the version under test and are not meant + # to be part of the PR). + git checkout -- eng/emitter-package.json eng/emitter-package-lock.json 2>/dev/null || true + + mode="${{ parameters.ChangePushMode }}" + if [ "$mode" = "api-md" ]; then + git add -A -- ':(glob)sdk/**/review/*.api.md' ':(glob)sdk/**/CHANGELOG.md' + else + git add -A sdk/ + fi + + if git diff --cached --quiet; then + echo "No changes selected for mode=$mode. Skipping PR creation." + echo "##vso[task.setvariable variable=HasPrChanges]false" + exit 0 + fi + + git -c user.name="azure-sdk" -c user.email="azuresdk@microsoft.com" commit -m "TypeSpec regeneration changes (${mode}) for emitter $(emitterVersion)" + + git remote remove pr-origin 2>/dev/null || true + git remote add pr-origin "https://x-access-token:${pushToken}@github.com/${repoOwner}/${repoName}.git" + git push pr-origin "$branch" --force + + echo "##vso[task.setvariable variable=HasPrChanges]true" + echo "##vso[task.setvariable variable=PRBranchName]$branch" + echo "##vso[task.setvariable variable=RepoOwner]$repoOwner" + echo "##vso[task.setvariable variable=RepoNameOnly]$repoName" + echo "##vso[task.setvariable variable=IsFork]$isFork" + echo "##vso[task.setvariable variable=PushToken;issecret=true]$pushToken" + displayName: 'Apply patches, commit and push branch' + env: + # Map secret variables into the script via env. This is the only + # reliable way for ADO to inject secret variables — `$(name)` direct + # substitution and `$[ variables[...] ]` indirect lookup do NOT work + # reliably for secret variables. + ${{ if ne(parameters.ForkTokenVariableName, '') }}: + FORK_TOKEN: $(${{ parameters.ForkTokenVariableName }}) + ${{ if eq(parameters.PullRequestRepoOwner, '') }}: + GH_TOKEN_VAL: $(GH_TOKEN) + + - task: PowerShell@2 + displayName: 'Create pull request' + condition: and(succeeded(), eq(variables['HasPrChanges'], 'true')) + inputs: + pwsh: true + targetType: inline + # Build a rich PR description from the aggregated results JSON + # that the Summary stage published (artifact: regen_summary), + # then call Submit-PullRequest.ps1. The body lists every + # package with breaking changes AND every package that was + # skipped because it does not yet have a tsp-location.yaml — + # so the upstream spec/SDK teams can see at a glance which + # packages they still need to on-board, without digging + # through ADO pipeline logs. + script: | + $ErrorActionPreference = 'Stop' + $aggPath = Join-Path '$(Pipeline.Workspace)' 'regen_summary/aggregated-results.json' + $lines = New-Object System.Collections.Generic.List[string] + $lines.Add("Generated by ``$(Build.DefinitionName)`` build [$(Build.BuildNumber)]($(System.CollectionUri)$(System.TeamProject)/_build/results?buildId=$(Build.BuildId)).") + $lines.Add("Emitter version: ``$(emitterVersion)``. Push mode: ``${{ parameters.ChangePushMode }}``.") + $lines.Add('') + + if (Test-Path $aggPath) { + $agg = Get-Content $aggPath -Raw | ConvertFrom-Json + + $lines.Add('## Regeneration summary') + $lines.Add("- Regenerated: **$($agg.regeneration.success)/$($agg.regeneration.total)**") + $lines.Add("- Built OK: **$($agg.build.success)/$($agg.build.total)**") + $lines.Add("- Changelog generated: **$($agg.changelog.generated)/$($agg.changelog.total)**") + $lines.Add("- Packages with breaking changes: **$($agg.changelog.withBreaking)**") + $lines.Add('') + + if ($agg.changelog.breakingPackages -and $agg.changelog.breakingPackages.Count -gt 0) { + $lines.Add("## Packages with breaking changes ($($agg.changelog.breakingPackages.Count))") + $lines.Add("The following $($agg.changelog.breakingPackages.Count) package(s) had breaking changes detected by ``update-changelog``. This may indicate an emitter regression or an intentional spec change — open each CHANGELOG below to triage.") + $lines.Add('') + foreach ($p in $agg.changelog.breakingPackages) { + $pkgShort = ($p -split '/')[-1] + $lines.Add("- [``$pkgShort``]($p/CHANGELOG.md)  $p") + } + $lines.Add('') + } + + if ($agg.skippedNoTspLocation -and $agg.skippedNoTspLocation.Count -gt 0) { + $lines.Add("## Packages skipped — no usable tsp-location.yaml ($($agg.skippedNoTspLocation.Count))") + $lines.Add('These packages were filtered out by matrix gen (`-OnlyTypeSpec true`) and are NOT covered by this PR.') + $lines.Add('**Spec / SDK team action**: please add a `tsp-location.yaml` (or fix the existing one) so these packages participate in the next regeneration run.') + $lines.Add('') + # Normalize reasons into buckets so the summary is meaningful + # even when upstream changes the exact error string (e.g. + # "stale path: ..." carries a different spec path each time). + $reasonBuckets = @{} + foreach ($s in $agg.skippedNoTspLocation) { + $r = $s.reason + if ($r -like 'no tsp-location.yaml*') { $bucket = 'no tsp-location.yaml' } + elseif ($r -like 'stale path*') { $bucket = 'stale path (tspconfig.yaml missing in spec)' } + elseif ($r -like 'relative repo*') { $bucket = 'relative repo (local-only tsp-location)' } + else { $bucket = $r } + if (-not $reasonBuckets.ContainsKey($bucket)) { $reasonBuckets[$bucket] = 0 } + $reasonBuckets[$bucket]++ + } + $lines.Add('**Breakdown by reason:**') + foreach ($k in ($reasonBuckets.Keys | Sort-Object { -$reasonBuckets[$_] })) { + $lines.Add("- $k — **$($reasonBuckets[$k])**") + } + $lines.Add('') + $lines.Add('
Click to expand the full list') + $lines.Add('') + foreach ($s in $agg.skippedNoTspLocation) { $lines.Add("- ``$($s.name)`` — $($s.reason)") } + $lines.Add('') + $lines.Add('
') + $lines.Add('') + } + } else { + $lines.Add('_(aggregated-results.json not found; falling back to minimal description)_') + } + + # Surface files that the CreatePR step had to drop because + # `git apply --3way` couldn't merge them (typically caused by + # an upstream commit landing on the PR target branch after + # the shard had already generated its patch — see the + # 'Apply patches' step for details). Listing them here so + # reviewers / spec teams know which packages are missing + # from this run and will be regenerated on the next one. + $failedLog = Join-Path '$(Pipeline.Workspace)' 'failed-patches.txt' + if (Test-Path $failedLog) { + $failedEntries = Get-Content $failedLog | Where-Object { $_ -and $_.Trim().Length -gt 0 } + if ($failedEntries.Count -gt 0) { + $lines.Add("## Patches skipped due to upstream conflicts ($($failedEntries.Count))") + $lines.Add('These files were dropped because the PR target branch (`${{ parameters.PullRequestTargetBranch }}`) moved underneath the regeneration shard — typically an `[AutoPR @azure-arm-*]` commit landed in the same file between shard checkout and PR creation. They are **not** part of this PR; the next regeneration run will pick them up against the new baseline.') + $lines.Add('') + foreach ($e in $failedEntries) { $lines.Add("- ``$e``") } + $lines.Add('') + } + } + + $bodyText = ($lines -join "`n") + $prTitle = 'TypeSpec regeneration: emitter $(emitterVersion) [${{ parameters.ChangePushMode }}]' + Write-Host '----- PR body preview (first 60 lines) -----' + $lines | Select-Object -First 60 | ForEach-Object { Write-Host $_ } + Write-Host '----- end preview -----' + + # Every pipeline run uses a unique branch name + # (`sdk-regenerate-$(Build.BuildId)`) so Submit-PullRequest + # always creates a fresh PR with the correct title+body, + # rather than reusing an old PR whose body would otherwise + # be stuck on the first run's content. This preserves a + # per-run history of breaking-change reports that mentor + # and reviewers can compare across emitter versions. + & ./eng/common/scripts/Submit-PullRequest.ps1 ` + -RepoOwner '$(RepoOwner)' ` + -RepoName '$(RepoNameOnly)' ` + -BaseBranch '${{ parameters.PullRequestTargetBranch }}' ` + -PROwner '$(RepoOwner)' ` + -PRBranch '$(PRBranchName)' ` + -AuthToken '$(PushToken)' ` + -PRTitle $prTitle ` + -PRBody $bodyText