+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| File | ++ | Statements | ++ | Branches | ++ | Functions | ++ | Lines | ++ |
|---|---|---|---|---|---|---|---|---|---|
| index.ts | +
+
+ |
+ 0% | +0/0 | +0% | +0/0 | +0% | +0/0 | +0% | +0/0 | +
| interfaces.ts | +
+
+ |
+ 100% | +8/8 | +100% | +0/0 | +100% | +2/2 | +100% | +8/8 | +
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 +2 +3 | + + | export * from './interfaces' +export * from './utils' + |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 +33 +34 +35 +36 +37 +38 +39 +40 +41 +42 +43 +44 +45 +46 +47 +48 +49 +50 +51 +52 +53 +54 +55 +56 +57 +58 +59 +60 +61 +62 +63 +64 +65 +66 +67 +68 +69 +70 +71 +72 +73 +74 +75 +76 +77 +78 +79 +80 +81 +82 +83 +84 +85 +86 +87 +88 +89 +90 +91 +92 +93 | 1x +1x +1x +1x +1x + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +1x +1x +1x + + + + + + + + + + + + + + + + + + + | export enum Command {
+ PrepareJob = 'prepare_job',
+ CleanupJob = 'cleanup_job',
+ RunContainerStep = 'run_container_step',
+ RunScriptStep = 'run_script_step'
+}
+
+export interface HookData {
+ command: Command
+ responseFile: string
+ args?: PrepareJobArgs | RunContainerStepArgs | RunScriptStepArgs
+ state?: { [key: string]: any }
+}
+
+export interface PrepareJobArgs {
+ container?: JobContainerInfo
+ services?: ServiceContainerInfo[]
+}
+
+export type RunContainerStepArgs = StepContainerInfo
+
+export interface RunScriptStepArgs {
+ entryPoint: string
+ entryPointArgs: string[]
+ environmentVariables?: { [key: string]: string }
+ prependPath?: string[]
+ workingDirectory: string
+}
+
+export interface ContainerInfo {
+ image?: string
+ entryPoint?: string
+ entryPointArgs?: string[]
+ createOptions?: string
+ environmentVariables?: { [key: string]: string }
+ userMountVolumes?: Mount[]
+ systemMountVolumes?: Mount[]
+ registry?: Registry
+ portMappings?: string[]
+}
+
+export interface ServiceContainerInfo extends ContainerInfo {
+ contextName: string
+ image: string
+}
+
+export interface JobContainerInfo extends ContainerInfo {
+ image: string
+ workingDirectory: string
+ systemMountVolumes: Mount[]
+}
+
+export interface StepContainerInfo extends ContainerInfo {
+ prependPath?: string[]
+ workingDirectory: string
+ dockerfile?: string
+ systemMountVolumes: Mount[]
+}
+
+export interface Mount {
+ sourceVolumePath: string
+ targetVolumePath: string
+ readOnly: boolean
+}
+
+export interface Registry {
+ username?: string
+ password?: string
+ serverUrl: string
+}
+
+export enum Protocol {
+ TCP = 'tcp',
+ UDP = 'udp'
+}
+
+export interface PrepareJobResponse {
+ state?: object
+ context?: ContainerContext
+ services?: { [key: string]: ContainerContext }
+ alpine: boolean
+}
+
+export interface ContainerContext {
+ id?: string
+ network?: string
+ ports?: { [key: string]: string }
+}
+
+export interface ContextPorts {
+ [source: string]: string // source -> target
+}
+ |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| File | ++ | Statements | ++ | Branches | ++ | Functions | ++ | Lines | ++ |
|---|---|---|---|---|---|---|---|---|---|
| index.ts | +
+
+ |
+ 58.82% | +10/17 | +100% | +12/12 | +50% | +2/4 | +58.82% | +10/17 | +
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 +33 +34 +35 +36 +37 +38 +39 +40 +41 +42 +43 | + + + + + + + + + + + + + + + + + + + + + +16x +1x + +15x +1x + + +14x + + + + + +14x +2x +12x +11x + +1x + + | import * as events from 'events'
+import * as fs from 'fs'
+import * as os from 'os'
+import * as readline from 'readline'
+import { HookData } from '../interfaces'
+
+export async function getInputFromStdin(): Promise<HookData> {
+ let input = ''
+
+ const rl = readline.createInterface({
+ input: process.stdin
+ })
+
+ rl.on('line', line => {
+ input = line
+ })
+ await events.default.once(rl, 'close')
+ const inputJson = JSON.parse(input)
+ return inputJson as HookData
+}
+
+export function writeToResponseFile(filePath: string, message: any): void {
+ if (!filePath) {
+ throw new Error(`Expected file path`)
+ }
+ if (!fs.existsSync(filePath)) {
+ throw new Error(`Missing file at path: ${filePath}`)
+ }
+
+ fs.appendFileSync(filePath, `${toCommandValue(message)}${os.EOL}`, {
+ encoding: 'utf8'
+ })
+}
+
+function toCommandValue(input: any): string {
+ if (input === null || input === undefined) {
+ return ''
+ } else if (typeof input === 'string' || input instanceof String) {
+ return input as string
+ }
+ return JSON.stringify(input)
+}
+ |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| File | ++ | Statements | ++ | Branches | ++ | Functions | ++ | Lines | ++ |
|---|---|---|---|---|---|---|---|---|---|
| hooklib/src | +
+
+ |
+ 100% | +8/8 | +100% | +0/0 | +100% | +2/2 | +100% | +8/8 | +
| hooklib/src/utils | +
+
+ |
+ 58.82% | +10/17 | +100% | +12/12 | +50% | +2/4 | +58.82% | +10/17 | +
| k8s/src | +
+
+ |
+ 0% | +0/22 | +0% | +0/7 | +0% | +0/1 | +0% | +0/22 | +
| k8s/src/hooks | +
+
+ |
+ 95.31% | +285/299 | +84.81% | +134/158 | +86.2% | +25/29 | +95.3% | +284/298 | +
| k8s/src/k8s | +
+
+ |
+ 75.37% | +560/743 | +71.71% | +251/350 | +74.59% | +91/122 | +75.68% | +551/728 | +
| k8s/src/k8s/utils | +
+
+ |
+ 98.5% | +394/400 | +92.6% | +263/284 | +98.57% | +69/70 | +98.72% | +387/392 | +
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 | + + + + + + + + + + +4x +3x +3x + +1x + + +4x + + | import * as core from '@actions/core'
+import { formatError } from '../k8s/utils'
+import { prunePods, pruneSecrets } from '../k8s'
+import {
+ collectAndPushNpuMetrics,
+ NPU_COLLECT_TIMEOUT_MS,
+ npuMetricsEnabled,
+ withTimeout
+} from '../k8s/utils/npu-metrics'
+
+export async function cleanupJob(): Promise<void> {
+ if (npuMetricsEnabled()) {
+ try {
+ await withTimeout(collectAndPushNpuMetrics(), NPU_COLLECT_TIMEOUT_MS)
+ } catch (err) {
+ core.debug(`npu-metrics: collection failed: ${formatError(err)}`)
+ }
+ }
+ await Promise.all([prunePods(), pruneSecrets()])
+}
+ |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 +33 +34 +35 +36 +37 +38 +39 +40 +41 +42 +43 +44 +45 +46 +47 +48 +49 +50 +51 +52 +53 +54 +55 +56 +57 +58 +59 +60 +61 +62 +63 +64 +65 +66 | + + +86x +86x + + + + +86x + + + +42x + + + + + + +18x + + + + + + + + + + + + + + +4x + + + + + +7x +7x +7x +7x +7x + + + + +17x + + + +10x + + + +10x + + + +7x + + + | import { v4 as uuidv4 } from 'uuid'
+
+export function getRunnerPodName(): string {
+ const name = process.env.ACTIONS_RUNNER_POD_NAME
+ Iif (!name) {
+ throw new Error(
+ "'ACTIONS_RUNNER_POD_NAME' env is required, please contact your self hosted runner administrator"
+ )
+ }
+ return name
+}
+
+export function getJobPodName(): string {
+ return `${getRunnerPodName().substring(
+ 0,
+ MAX_POD_NAME_LENGTH - '-workflow'.length
+ )}-workflow`
+}
+
+export function getStepPodName(): string {
+ return `${getRunnerPodName().substring(
+ 0,
+ MAX_POD_NAME_LENGTH - ('-step-'.length + STEP_POD_NAME_SUFFIX_LENGTH)
+ )}-step-${uuidv4().substring(0, STEP_POD_NAME_SUFFIX_LENGTH)}`
+}
+
+export function getVolumeClaimName(): string {
+ const name = process.env.ACTIONS_RUNNER_CLAIM_NAME
+ if (!name) {
+ return `${getRunnerPodName()}-work`
+ }
+ return name
+}
+
+export function getSecretName(): string {
+ return `${getRunnerPodName().substring(
+ 0,
+ MAX_POD_NAME_LENGTH - ('-secret-'.length + STEP_POD_NAME_SUFFIX_LENGTH)
+ )}-secret-${uuidv4().substring(0, STEP_POD_NAME_SUFFIX_LENGTH)}`
+}
+
+export const MAX_POD_NAME_LENGTH = 63
+export const STEP_POD_NAME_SUFFIX_LENGTH = 8
+export const CONTAINER_EXTENSION_PREFIX = '$'
+export const JOB_CONTAINER_NAME = 'job'
+export const JOB_CONTAINER_EXTENSION_NAME = '$job'
+
+export class RunnerInstanceLabel {
+ private podName: string
+ constructor() {
+ this.podName = getRunnerPodName()
+ }
+
+ get key(): string {
+ return 'runner-pod'
+ }
+
+ get value(): string {
+ return this.podName
+ }
+
+ toString(): string {
+ return `runner-pod=${this.podName}`
+ }
+}
+ |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| File | ++ | Statements | ++ | Branches | ++ | Functions | ++ | Lines | ++ |
|---|---|---|---|---|---|---|---|---|---|
| cleanup-job.ts | +
+
+ |
+ 100% | +5/5 | +100% | +2/2 | +100% | +1/1 | +100% | +5/5 | +
| constants.ts | +
+
+ |
+ 75% | +15/20 | +25% | +1/4 | +88.88% | +8/9 | +75% | +15/20 | +
| index.ts | +
+
+ |
+ 0% | +0/0 | +0% | +0/0 | +0% | +0/0 | +0% | +0/0 | +
| prepare-job.ts | +
+
+ |
+ 97.63% | +124/127 | +86.66% | +78/90 | +88.88% | +8/9 | +97.63% | +124/127 | +
| run-container-step.ts | +
+
+ |
+ 95.09% | +97/102 | +86% | +43/50 | +71.42% | +5/7 | +95.09% | +97/102 | +
| run-script-step.ts | +
+
+ |
+ 97.77% | +44/45 | +83.33% | +10/12 | +100% | +3/3 | +97.72% | +43/44 | +
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 +2 +3 +4 +5 | + + + + | export * from './cleanup-job' +export * from './prepare-job' +export * from './run-script-step' +export * from './run-container-step' + |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 +33 +34 +35 +36 +37 +38 +39 +40 +41 +42 +43 +44 +45 +46 +47 +48 +49 +50 +51 +52 +53 +54 +55 +56 +57 +58 +59 +60 +61 +62 +63 +64 +65 +66 +67 +68 +69 +70 +71 +72 +73 +74 +75 +76 +77 +78 +79 +80 +81 +82 +83 +84 +85 +86 +87 +88 +89 +90 +91 +92 +93 +94 +95 +96 +97 +98 +99 +100 +101 +102 +103 +104 +105 +106 +107 +108 +109 +110 +111 +112 +113 +114 +115 +116 +117 +118 +119 +120 +121 +122 +123 +124 +125 +126 +127 +128 +129 +130 +131 +132 +133 +134 +135 +136 +137 +138 +139 +140 +141 +142 +143 +144 +145 +146 +147 +148 +149 +150 +151 +152 +153 +154 +155 +156 +157 +158 +159 +160 +161 +162 +163 +164 +165 +166 +167 +168 +169 +170 +171 +172 +173 +174 +175 +176 +177 +178 +179 +180 +181 +182 +183 +184 +185 +186 +187 +188 +189 +190 +191 +192 +193 +194 +195 +196 +197 +198 +199 +200 +201 +202 +203 +204 +205 +206 +207 +208 +209 +210 +211 +212 +213 +214 +215 +216 +217 +218 +219 +220 +221 +222 +223 +224 +225 +226 +227 +228 +229 +230 +231 +232 +233 +234 +235 +236 +237 +238 +239 +240 +241 +242 +243 +244 +245 +246 +247 +248 +249 +250 +251 +252 +253 +254 +255 +256 +257 +258 +259 +260 +261 +262 +263 +264 +265 +266 +267 +268 +269 +270 +271 +272 +273 +274 +275 +276 +277 +278 +279 +280 +281 +282 +283 +284 +285 +286 +287 +288 +289 +290 +291 +292 +293 +294 +295 +296 +297 +298 +299 +300 +301 +302 +303 +304 +305 +306 +307 +308 +309 +310 +311 +312 +313 +314 +315 +316 +317 +318 +319 +320 +321 +322 +323 +324 +325 +326 +327 +328 +329 +330 +331 +332 +333 +334 +335 +336 +337 +338 +339 +340 +341 +342 +343 +344 +345 +346 +347 +348 +349 +350 +351 +352 +353 +354 +355 +356 +357 +358 +359 +360 +361 +362 +363 +364 | + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +20x +1x + + +19x + +19x + +19x +19x +18x + + + + + +18x + + +19x +19x +19x +4x +4x +6x +6x + + +4x +4x +6x +6x +6x + + +6x +4x + +2x + + +6x +6x +6x + + + +19x +1x + + +18x +18x +18x + + + + + + + +6x +6x + + + + + + +6x +6x + + + + + + +6x +6x + + +6x + +6x + + +6x + + +5x +5x + + +5x +5x +5x +2x + + +2x + + + + + +6x + + + + + + + + + +6x + + +12x +1x + +11x + + + +20x +20x +1x + +10x + + +10x +2x + + +10x +10x + + + + + + +1x + + + +1x +1x + + +9x + +9x +2x + + + + + +2x +2x +2x + + + + + + + +2x + + +9x + +9x +9x +9x + + + + +1x +1x +1x + +8x +8x + + + + + + + + + +8x + + +8x + + + + + + + +8x +8x + +8x +8x +8x +8x + + + + + +8x + + + + + +8x + +4x + + + +4x +10x + +6x +6x +1x +1x +1x + + + + +6x + + + + + + +8x + + + + + + + + +35x +1x +1x + + +35x + + + + +35x +1x + + +35x +28x + + +35x +20x + + +35x +35x + + +2x +1x + + + +35x + + + + +35x +34x + + + + + +35x + +35x +34x + + +1x +1x + + +35x +1x + + +1x + + | import * as core from '@actions/core'
+import * as k8s from '@kubernetes/client-node'
+import {
+ JobContainerInfo,
+ ContextPorts,
+ PrepareJobArgs,
+ writeToResponseFile,
+ ServiceContainerInfo
+} from 'hooklib'
+import {
+ containerPorts,
+ createJobPod,
+ isPodContainerAlpine,
+ prunePods,
+ waitForPodPhases,
+ getPrepareJobTimeoutSeconds,
+ execCpToPod,
+ execPodStep
+} from '../k8s'
+import {
+ CONTAINER_VOLUMES,
+ DEFAULT_CONTAINER_ENTRY_POINT,
+ DEFAULT_CONTAINER_ENTRY_POINT_ARGS,
+ formatError,
+ generateContainerName,
+ mergeContainerWithOptions,
+ readExtensionFromFile,
+ PodPhase,
+ fixArgs,
+ prepareJobScript
+} from '../k8s/utils'
+import {
+ CONTAINER_EXTENSION_PREFIX,
+ getJobPodName,
+ JOB_CONTAINER_NAME
+} from './constants'
+import { maybeInjectNpuMetrics } from '../k8s/utils/npu-metrics'
+import { dirname } from 'path'
+
+export async function prepareJob(
+ args: PrepareJobArgs,
+ responseFile
+): Promise<void> {
+ if (!args.container) {
+ throw new Error('Job Container is required.')
+ }
+
+ await prunePods()
+
+ const extension = readExtensionFromFile()
+
+ let container: k8s.V1Container | undefined = undefined
+ if (args.container?.image) {
+ container = createContainerSpec(
+ args.container,
+ JOB_CONTAINER_NAME,
+ true,
+ extension
+ )
+ maybeInjectNpuMetrics(container)
+ }
+
+ let services: k8s.V1Container[] = []
+ let serviceNames: string[] = []
+ if (args.services?.length) {
+ const occurrences = new Map<string, number>()
+ for (const s of args.services) {
+ const base = generateContainerName(s.image)
+ occurrences.set(base, (occurrences.get(base) || 0) + 1)
+ }
+
+ const indices = new Map<string, number>()
+ services = args.services.map(service => {
+ const base = generateContainerName(service.image)
+ const total = occurrences.get(base) || 0
+ const idx = indices.get(base) || 0
+
+ let name: string
+ if (total > 1) {
+ name = `${base}-${idx}`
+ } else {
+ name = base
+ }
+
+ indices.set(base, idx + 1)
+ serviceNames.push(name)
+ return createContainerSpec(service, name, false, extension)
+ })
+ }
+
+ if (!container && !services?.length) {
+ throw new Error('No containers exist, skipping hook invocation')
+ }
+
+ let createdPod: k8s.V1Pod | undefined = undefined
+ try {
+ createdPod = await createJobPod(
+ getJobPodName(),
+ container,
+ services,
+ args.container.registry,
+ extension
+ )
+ } catch (err) {
+ await prunePods()
+ core.debug(`createPod failed: ${JSON.stringify(err)}`)
+ // The k8s client throws HttpException whose message is a multi-line string
+ // containing the raw HTTP dump. Extract the human-readable "message" field
+ // from the embedded JSON body so the log shows something like:
+ // failed to create job pod:
+ // spec.volumes[5].name: Duplicate value: "bad-hostpath"
+ // instead of the full HTTP dump.
+ const raw = err instanceof Error ? err.message : String(err)
+ let detail = raw
+ // The k8s HttpException message is a multi-line dump:
+ // HTTP-Code: 422
+ // Message: Unknown API Status Code!
+ // Body: "{\"kind\":\"Status\",\"message\":\"...\\n\"}"
+ // Headers: {...}
+ // Extract the Body JSON string, unescape it, and pull out "message".
+ try {
+ const bodyStart = raw.indexOf('Body: "')
+ // The boundary may be '"\nHeaders:' (real newline) or the literal
+ // string ends before Headers — use the last '"' before 'Headers:' as fallback
+ const headersIdx = raw.indexOf('Headers:')
+ const bodyEnd =
+ headersIdx !== -1
+ ? raw.lastIndexOf('"', headersIdx) // last " before Headers:
+ : raw.indexOf('"\nHeaders:')
+ if (bodyStart !== -1 && bodyEnd !== -1 && bodyEnd > bodyStart) {
+ // Body content is a JSON string literal (without surrounding quotes).
+ // Wrap it in quotes and JSON.parse to properly unescape \" \\ \n \t etc.
+ const escaped = raw.substring(bodyStart + 7, bodyEnd)
+ const bodyStr = JSON.parse('"' + escaped + '"')
+ // bodyStr may be plain text (e.g. 502 Bad Gateway) instead of JSON.
+ // Parse separately so a non-JSON body still yields a friendly message.
+ try {
+ const parsed = JSON.parse(bodyStr)
+ if (typeof parsed?.message === 'string') {
+ detail = parsed.message
+ }
+ } catch {
+ detail = bodyStr
+ }
+ }
+ } catch {
+ // Parsing failed — fall through and show the raw string
+ }
+ const errorMessage = [
+ 'failed to create job pod:',
+ ` ✗ ${detail}`,
+ '-'.repeat(60),
+ ' → Pod spec was rejected by the k8s API. Check:',
+ ' - resources.requests does not exceed resources.limits',
+ ' - volumeMounts reference a volume defined in spec.volumes',
+ ' - envFrom / valueFrom reference existing Secrets / ConfigMaps',
+ ' - Field types match the k8s schema (kubectl explain pod.spec.containers)'
+ ].join('\n')
+ throw new Error(errorMessage)
+ }
+
+ if (!createdPod?.metadata?.name) {
+ throw new Error('created pod should have metadata.name')
+ }
+ core.debug(
+ `Job pod created, waiting for it to come online ${createdPod?.metadata?.name}`
+ )
+
+ const runnerWorkspaceEnv = process.env.RUNNER_WORKSPACE
+ if (!runnerWorkspaceEnv) {
+ throw new Error('RUNNER_WORKSPACE environment variable is not set')
+ }
+ const runnerWorkspace = dirname(runnerWorkspaceEnv)
+
+ let prepareScript: { containerPath: string; runnerPath: string } | undefined
+ if (args.container?.userMountVolumes?.length) {
+ prepareScript = prepareJobScript(args.container.userMountVolumes || [])
+ }
+
+ try {
+ await waitForPodPhases(
+ createdPod.metadata.name,
+ new Set([PodPhase.RUNNING]),
+ new Set([PodPhase.PENDING]),
+ getPrepareJobTimeoutSeconds()
+ )
+ } catch (err) {
+ await prunePods()
+ // Unwrap nested "Error: " prefix so the message renders as:
+ // pod failed to come online:
+ // <detail from waitForPodPhases, already formatted with sections>
+ const detail = err instanceof Error ? err.message : String(err)
+ throw new Error(`pod failed to come online:\n${detail}`)
+ }
+
+ await execCpToPod(createdPod.metadata.name, runnerWorkspace, '/__w')
+
+ if (prepareScript) {
+ await execPodStep(
+ ['sh', '-e', prepareScript.containerPath],
+ createdPod.metadata.name,
+ JOB_CONTAINER_NAME
+ )
+
+ const promises: Promise<void>[] = []
+ for (const vol of args?.container?.userMountVolumes || []) {
+ promises.push(
+ execCpToPod(
+ createdPod.metadata.name,
+ vol.sourceVolumePath,
+ vol.targetVolumePath
+ )
+ )
+ }
+ await Promise.all(promises)
+ }
+
+ core.debug('Job pod is ready for traffic')
+
+ let isAlpine = false
+ try {
+ isAlpine = await isPodContainerAlpine(
+ createdPod.metadata.name,
+ JOB_CONTAINER_NAME
+ )
+ } catch (err) {
+ const message = formatError(err)
+ core.debug(`Failed to determine if the pod is alpine: ${message}`)
+ throw new Error(`failed to determine if the pod is alpine: ${message}`)
+ }
+ core.debug(`Setting isAlpine to ${isAlpine}`)
+ generateResponseFile(responseFile, args, createdPod, isAlpine, serviceNames)
+}
+
+function generateResponseFile(
+ responseFile: string,
+ args: PrepareJobArgs,
+ appPod: k8s.V1Pod,
+ isAlpine: boolean,
+ serviceNames?: string[]
+): void {
+ Iif (!appPod.metadata?.name) {
+ throw new Error('app pod must have metadata.name specified')
+ }
+ const response = {
+ state: {
+ jobPod: appPod.metadata.name
+ },
+ context: {},
+ isAlpine
+ }
+
+ const mainContainer = appPod.spec?.containers?.find(
+ c => c.name === JOB_CONTAINER_NAME
+ )
+ Eif (mainContainer) {
+ const mainContainerContextPorts: ContextPorts = {}
+ Eif (mainContainer?.ports) {
+ for (const port of mainContainer.ports) {
+ mainContainerContextPorts[port.containerPort] =
+ mainContainerContextPorts.hostPort
+ }
+ }
+
+ response.context['container'] = {
+ image: mainContainer.image,
+ ports: mainContainerContextPorts
+ }
+ }
+
+ if (args.services?.length) {
+ const serviceContainerNames =
+ serviceNames && serviceNames.length
+ ? serviceNames
+ : args.services?.map(s => generateContainerName(s.image)) || []
+
+ response.context['services'] = appPod?.spec?.containers
+ ?.filter(c => serviceContainerNames.includes(c.name))
+ .map(c => {
+ const ctxPorts: ContextPorts = {}
+ if (c.ports?.length) {
+ for (const port of c.ports) {
+ Eif (port.containerPort && port.hostPort) {
+ ctxPorts[port.containerPort.toString()] = port.hostPort.toString()
+ }
+ }
+ }
+
+ return {
+ image: c.image,
+ ports: ctxPorts
+ }
+ })
+ }
+
+ writeToResponseFile(responseFile, JSON.stringify(response))
+}
+
+export function createContainerSpec(
+ container: JobContainerInfo | ServiceContainerInfo,
+ name: string,
+ jobContainer = false,
+ extension?: k8s.V1PodTemplateSpec
+): k8s.V1Container {
+ if (!container.entryPoint && jobContainer) {
+ container.entryPoint = DEFAULT_CONTAINER_ENTRY_POINT
+ container.entryPointArgs = DEFAULT_CONTAINER_ENTRY_POINT_ARGS
+ }
+
+ const podContainer = {
+ name,
+ image: container.image,
+ ports: containerPorts(container)
+ } as k8s.V1Container
+ if (container['workingDirectory']) {
+ podContainer.workingDir = container['workingDirectory']
+ }
+
+ if (container.entryPoint) {
+ podContainer.command = [container.entryPoint]
+ }
+
+ if (container.entryPointArgs && container.entryPointArgs.length > 0) {
+ podContainer.args = fixArgs(container.entryPointArgs)
+ }
+
+ podContainer.env = []
+ for (const [key, value] of Object.entries(
+ container['environmentVariables'] || {}
+ )) {
+ if (value && key !== 'HOME') {
+ podContainer.env.push({ name: key, value })
+ }
+ }
+
+ podContainer.env.push({
+ name: 'GITHUB_ACTIONS',
+ value: 'true'
+ })
+
+ if (!('CI' in (container['environmentVariables'] || {}))) {
+ podContainer.env.push({
+ name: 'CI',
+ value: 'true'
+ })
+ }
+
+ podContainer.volumeMounts = CONTAINER_VOLUMES
+
+ if (!extension) {
+ return podContainer
+ }
+
+ const from = extension.spec?.containers?.find(
+ c => c.name === CONTAINER_EXTENSION_PREFIX + name
+ )
+
+ if (from) {
+ mergeContainerWithOptions(podContainer, from)
+ }
+
+ return podContainer
+}
+ |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 +33 +34 +35 +36 +37 +38 +39 +40 +41 +42 +43 +44 +45 +46 +47 +48 +49 +50 +51 +52 +53 +54 +55 +56 +57 +58 +59 +60 +61 +62 +63 +64 +65 +66 +67 +68 +69 +70 +71 +72 +73 +74 +75 +76 +77 +78 +79 +80 +81 +82 +83 +84 +85 +86 +87 +88 +89 +90 +91 +92 +93 +94 +95 +96 +97 +98 +99 +100 +101 +102 +103 +104 +105 +106 +107 +108 +109 +110 +111 +112 +113 +114 +115 +116 +117 +118 +119 +120 +121 +122 +123 +124 +125 +126 +127 +128 +129 +130 +131 +132 +133 +134 +135 +136 +137 +138 +139 +140 +141 +142 +143 +144 +145 +146 +147 +148 +149 +150 +151 +152 +153 +154 +155 +156 +157 +158 +159 +160 +161 +162 +163 +164 +165 +166 +167 +168 +169 +170 +171 +172 +173 +174 +175 +176 +177 +178 +179 +180 +181 +182 +183 +184 +185 +186 +187 +188 +189 +190 +191 +192 +193 +194 +195 +196 +197 +198 +199 +200 +201 +202 +203 +204 +205 +206 +207 +208 +209 +210 +211 +212 +213 +214 +215 +216 +217 +218 +219 +220 +221 +222 +223 +224 +225 +226 +227 +228 +229 +230 +231 +232 +233 +234 +235 +236 +237 +238 +239 +240 +241 +242 +243 +244 +245 +246 +247 +248 +249 +250 +251 +252 +253 +254 +255 +256 +257 +258 +259 +260 +261 +262 +263 +264 +265 +266 +267 +268 +269 +270 +271 +272 | + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +20x +1x + + +19x +1x + + + + +18x +20x +20x +17x + + +18x + +18x +18x + + +18x +18x + +2x +2x +2x + + +16x +1x + + + + + +15x + +15x +15x + + + + + + +14x +14x +14x +1x + + + +13x +13x +13x +13x +1x + +12x + +12x + + +12x + +12x + + + + + + + +12x + +12x + +12x +12x +12x + + + + +10x +3x + + + + + +7x + + + + +7x + +9x + +9x + + + + +7x + +2x +2x + +12x + + +12x +12x +11x +11x +1x +1x + + + + + + +12x +12x + +15x + + + + + + + + + + + + + + + + +7x +7x +7x + +7x +7x +6x +5x + +7x +7x +4x + +4x + + + +4x +1x +1x +1x + + + +3x + + +3x + + + + + +2x + +6x +1x + + + + + +1x + + +7x +1x + +2x + +1x + + +7x +7x +3x + +7x + + + + + + +18x +18x +18x +18x +18x +18x + +18x + +18x +18x + + + + + +18x + + + + + + | import * as core from '@actions/core'
+import * as fs from 'fs'
+import * as k8s from '@kubernetes/client-node'
+import { RunContainerStepArgs } from 'hooklib'
+import { dirname } from 'path'
+import {
+ createContainerStepPod,
+ deletePod,
+ describePodFailure,
+ execCpFromPod,
+ execCpToPod,
+ execPodStepWithOutput,
+ getContainerTerminatedErrors,
+ getPodByName,
+ getPrepareJobTimeoutSeconds,
+ getTerminatedReasonHint,
+ waitForPodPhases
+} from '../k8s'
+import {
+ CONTAINER_VOLUMES,
+ formatError,
+ mergeContainerWithOptions,
+ PodPhase,
+ readExtensionFromFile,
+ DEFAULT_CONTAINER_ENTRY_POINT_ARGS,
+ writeContainerStepScript
+} from '../k8s/utils'
+import {
+ getJobPodName,
+ getStepPodName,
+ JOB_CONTAINER_EXTENSION_NAME,
+ JOB_CONTAINER_NAME
+} from './constants'
+import { maybeInjectNpuMetrics } from '../k8s/utils/npu-metrics'
+
+export async function runContainerStep(
+ stepContainer: RunContainerStepArgs
+): Promise<number> {
+ if (stepContainer.dockerfile) {
+ throw new Error('Building container actions is not currently supported')
+ }
+
+ if (!stepContainer.entryPoint) {
+ throw new Error(
+ 'failed to start the container since the entrypoint is overwritten'
+ )
+ }
+
+ const envs = stepContainer.environmentVariables || {}
+ envs['GITHUB_ACTIONS'] = 'true'
+ if (!('CI' in envs)) {
+ envs.CI = 'true'
+ }
+
+ const extension = readExtensionFromFile()
+
+ const container = createContainerSpec(stepContainer, extension)
+ maybeInjectNpuMetrics(container)
+
+ let pod: k8s.V1Pod
+ try {
+ pod = await createContainerStepPod(getStepPodName(), container, extension)
+ } catch (err) {
+ const message = formatError(err)
+ core.debug(`createContainerStepPod failed: ${message}`)
+ throw new Error(`failed to run container step: ${message}`)
+ }
+
+ if (!pod.metadata?.name) {
+ throw new Error(
+ `Expected job ${JSON.stringify(
+ pod
+ )} to have correctly set the metadata.name`
+ )
+ }
+ const podName = pod.metadata.name
+
+ try {
+ await waitForPodPhases(
+ podName,
+ new Set([PodPhase.RUNNING]),
+ new Set([PodPhase.PENDING, PodPhase.UNKNOWN]),
+ getPrepareJobTimeoutSeconds()
+ )
+
+ const runnerWorkspaceEnv = process.env.RUNNER_WORKSPACE
+ const githubWorkspaceEnv = process.env.GITHUB_WORKSPACE
+ if (!runnerWorkspaceEnv || !githubWorkspaceEnv) {
+ throw new Error(
+ 'RUNNER_WORKSPACE or GITHUB_WORKSPACE environment variable is not set'
+ )
+ }
+ const runnerWorkspace = dirname(runnerWorkspaceEnv)
+ const githubWorkspace = githubWorkspaceEnv
+ const parts = githubWorkspace.split('/').slice(-2)
+ if (parts.length !== 2) {
+ throw new Error(`Invalid github workspace directory: ${githubWorkspace}`)
+ }
+ const relativeWorkspace = parts.join('/')
+
+ core.debug(
+ `Copying files from pod ${getJobPodName()} to ${runnerWorkspace}/${relativeWorkspace}`
+ )
+ await execCpFromPod(getJobPodName(), `/__w`, `${runnerWorkspace}`)
+
+ const { containerPath, runnerPath } = writeContainerStepScript(
+ `${runnerWorkspace}/__w/_temp`,
+ githubWorkspace,
+ stepContainer.entryPoint,
+ stepContainer.entryPointArgs,
+ envs
+ )
+
+ await execCpToPod(podName, `${runnerWorkspace}/__w`, '/__w')
+
+ fs.rmSync(`${runnerWorkspace}/__w`, { recursive: true, force: true })
+
+ try {
+ core.debug(`Executing container step script in pod ${podName}`)
+ const { code, output } = await execPodStepWithOutput(
+ ['sh', '-e', containerPath],
+ pod.metadata.name,
+ JOB_CONTAINER_NAME
+ )
+ if (code === 0) {
+ return 0
+ }
+ // Non-zero exit: surface a structured error so the user can tell whether
+ // it was their script or the container that failed. Read container
+ // status BEFORE deletePod runs (in the outer finally) to inspect the
+ // terminated reason, if it is already available.
+ const classification = await classifyScriptError(
+ pod.metadata.name,
+ code,
+ output
+ )
+ throw new Error(classification)
+ } catch (err) {
+ core.debug(`execPodStep failed: ${formatError(err)}`)
+ // Re-throw our classified errors verbatim; wrap anything else.
+ if (
+ err instanceof Error &&
+ (err.message.startsWith('Step failed:') ||
+ err.message.startsWith('failed to run script step'))
+ ) {
+ throw err
+ }
+ const message = formatError(err)
+ throw new Error(`failed to run container step: ${message}`)
+ } finally {
+ fs.rmSync(runnerPath, { force: true })
+ }
+ } catch (error) {
+ try {
+ const errorPod = await getPodByName(podName)
+ const terminatedErrors = getContainerTerminatedErrors(errorPod)
+ if (terminatedErrors.length > 0) {
+ const details = await describePodFailure(podName)
+ core.error(
+ `Pod ${podName} has unrecoverable container errors:\n${terminatedErrors.join('\n')}\n${details}`
+ )
+ }
+ } catch {
+ // Best-effort: pod may already be deleted or unreachable
+ }
+ core.error(`Failed to run container step: ${error}`)
+ throw error
+ } finally {
+ await deletePod(podName).catch(err => {
+ core.error(`Failed to delete step pod ${podName}: ${err}`)
+ })
+ }
+}
+
+// Inspect the pod's container status to determine whether a non-zero exit
+// code came from the user's script (container terminated cleanly with
+// reason=Completed) or from a container-level failure (OOMKilled, etc.).
+// The container state may not yet be 'terminated' when called (k8s updates
+// it asynchronously), so we default to treating unknown state as a script
+// issue and let the user check their script first.
+async function classifyScriptError(
+ podName: string,
+ exitCode: number,
+ tailOutput: string
+): Promise<string> {
+ const sep = '-'.repeat(60)
+ const errors: string[] = [` ✗ exit code: ${exitCode}`]
+ const sections: string[] = []
+
+ try {
+ const pod = await getPodByName(podName)
+ const cs = pod.status?.containerStatuses?.find(
+ s => s.name === JOB_CONTAINER_NAME
+ )
+ const term = cs?.state?.terminated
+ if (term) {
+ const reason = term.reason ?? 'Completed'
+ const isContainerFault =
+ reason === 'OOMKilled' ||
+ reason === 'Error' ||
+ reason === 'FailedPostStartHookError' ||
+ (term.exitCode === 137 && reason !== 'Completed')
+ if (isContainerFault) {
+ const detail = term.message ? `\n ${term.message}` : ''
+ const hint = `\n${getTerminatedReasonHint(reason, term.exitCode)}`
+ errors.push(
+ ` ✗ container "${JOB_CONTAINER_NAME}": ${reason} (exit code ${term.exitCode})${detail}${hint}`
+ )
+ } else {
+ errors.push(
+ ` → your script exited with a non-zero code; please check your script for errors`
+ )
+ sections.push(
+ `Container status: ${reason} (exit code ${term.exitCode})`
+ )
+ }
+ } else {
+ // Container state unavailable — treat as script issue by default
+ errors.push(` → please check your script for errors`)
+ }
+ if (cs?.state?.waiting) {
+ errors.push(
+ ` ✗ container "${JOB_CONTAINER_NAME}" waiting: ${cs.state.waiting.reason ?? 'unknown'}`
+ )
+ }
+ } catch {
+ // pod already gone or API error — default hint
+ errors.push(` → please check your script for errors`)
+ }
+
+ if (tailOutput) {
+ const outputLines = tailOutput
+ .split('\n')
+ .map(l => ` ${l}`)
+ .join('\n')
+ sections.push(`Last output:\n${outputLines}`)
+ }
+
+ let result = `failed to run script step:\n${errors.join('\n')}`
+ if (sections.length) {
+ result += `\n${sep}\n${sections.join('\n')}`
+ }
+ return result
+}
+
+function createContainerSpec(
+ container: RunContainerStepArgs,
+ extension?: k8s.V1PodTemplateSpec
+): k8s.V1Container {
+ const podContainer = new k8s.V1Container()
+ podContainer.name = JOB_CONTAINER_NAME
+ podContainer.image = container.image
+ podContainer.workingDir = '/__w'
+ podContainer.command = ['tail']
+ podContainer.args = DEFAULT_CONTAINER_ENTRY_POINT_ARGS
+
+ podContainer.volumeMounts = CONTAINER_VOLUMES
+
+ Eif (!extension) {
+ return podContainer
+ }
+
+ const from = extension.spec?.containers?.find(
+ c => c.name === JOB_CONTAINER_EXTENSION_NAME
+ )
+ Iif (from) {
+ mergeContainerWithOptions(podContainer, from)
+ }
+
+ return podContainer
+}
+ |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 +33 +34 +35 +36 +37 +38 +39 +40 +41 +42 +43 +44 +45 +46 +47 +48 +49 +50 +51 +52 +53 +54 +55 +56 +57 +58 +59 +60 +61 +62 +63 +64 +65 +66 +67 +68 +69 +70 +71 +72 +73 +74 +75 +76 +77 +78 +79 +80 +81 +82 +83 +84 +85 +86 +87 +88 +89 +90 +91 +92 +93 +94 +95 +96 +97 +98 +99 +100 +101 +102 +103 +104 +105 +106 +107 +108 +109 +110 +111 +112 +113 +114 +115 +116 +117 +118 +119 +120 +121 +122 +123 +124 +125 +126 +127 +128 +129 +130 +131 +132 +133 +134 +135 +136 +137 +138 +139 +140 +141 +142 +143 +144 +145 +146 +147 +148 | + + + + + + + + + + + + + + + +2x +2x + + + +2x +2x +2x + +2x + +2x + +2x +2x +2x + + + + + + + +10x +2x + + + + + +8x +8x + + + + + + + +8x +8x +8x +8x + +8x + + + + + + + + +8x + + + + + +8x + + + + + + + + + + + + + + + + +8x +8x + + + + + +1x +1x +1x + + + +7x +7x +7x +7x + + + + +5x +2x + + +4x +4x + + + +3x + +1x +1x + +7x +7x + + + + + +3x +3x + + +3x + + + + + +1x + + + | /* eslint-disable @typescript-eslint/no-unused-vars */
+import * as fs from 'fs'
+import * as core from '@actions/core'
+import { RunScriptStepArgs } from 'hooklib'
+import {
+ execCpFromPod,
+ execCpToPod,
+ execPodStep,
+ execPodStepWithOutput
+} from '../k8s'
+import { formatError, writeRunScript } from '../k8s/utils'
+import { JOB_CONTAINER_NAME } from './constants'
+import { dirname } from 'path'
+import * as shlex from 'shlex'
+
+function formatScriptError(exitCode: number, tailOutput: string): string {
+ const sep = '-'.repeat(60)
+ const errors = [
+ ` ✗ exit code: ${exitCode}`,
+ ` → your script exited with a non-zero code; please check your script for errors`
+ ]
+ const sections: string[] = []
+ Eif (tailOutput) {
+ const outputLines = tailOutput
+ .split('\n')
+ .map(l => ` ${l}`)
+ .join('\n')
+ sections.push(`Last output:\n${outputLines}`)
+ }
+ let result = `failed to run script step:\n${errors.join('\n')}`
+ Eif (sections.length) result += `\n${sep}\n${sections.join('\n')}`
+ return result
+}
+
+export async function runScriptStep(
+ args: RunScriptStepArgs,
+ state
+): Promise<void> {
+ // Validate that pod was created successfully (prepareJob succeeded)
+ if (!state?.jobPod) {
+ throw new Error(
+ 'jobPod must be set - ensure prepareJob completed successfully before running script steps'
+ )
+ }
+
+ // Write the entrypoint first. This will be later coppied to the workflow pod
+ const { entryPoint, entryPointArgs, environmentVariables } = args
+ const { containerPath, runnerPath } = writeRunScript(
+ args.workingDirectory,
+ entryPoint,
+ entryPointArgs,
+ args.prependPath,
+ environmentVariables
+ )
+
+ const workdir = dirname(process.env.RUNNER_WORKSPACE as string)
+ const runnerTemp = `${workdir}/_temp`
+ const containerTemp = '/__w/_temp'
+ const containerTempSrc = '/__w/_temp_pre'
+ // Ensure base and staging dirs exist before copying
+ await execPodStep(
+ [
+ 'sh',
+ '-c',
+ 'mkdir -p /__w && mkdir -p /__w/_temp && mkdir -p /__w/_temp_pre'
+ ],
+ state.jobPod,
+ JOB_CONTAINER_NAME
+ )
+ await execCpToPod(state.jobPod, runnerTemp, containerTempSrc)
+
+ // Copy GitHub directories from temp to /github
+ // Merge strategy:
+ // - Overwrite files in _runner_file_commands
+ // - Append files not already present elsewhere
+ const mergeCommands = [
+ 'set -e',
+ 'mkdir -p /__w/_temp /__w/_temp_pre',
+ 'SRC=/__w/_temp_pre',
+ 'DST=/__w/_temp',
+ // Overwrite _runner_file_commands
+ 'cp -a "$SRC/_runner_file_commands/." "$DST/_runner_file_commands"',
+ `find "$SRC" -type f ! -path "*/_runner_file_commands/*" -exec sh -c '
+ rel="\${1#$2/}"
+ target="$3/$rel"
+ mkdir -p "$(dirname "$target")"
+ cp -a "$1" "$target"
+ ' _ {} "$SRC" "$DST" \\;`,
+ // Remove _temp_pre after merging
+ 'rm -rf /__w/_temp_pre'
+ ]
+
+ try {
+ await execPodStep(
+ ['sh', '-c', mergeCommands.join(' && ')],
+ state.jobPod,
+ JOB_CONTAINER_NAME
+ )
+ } catch (err) {
+ const message = formatError(err)
+ core.debug(`Failed to merge temp directories: ${message}`)
+ throw new Error(`failed to merge temp dirs: ${message}`)
+ }
+
+ // Execute the entrypoint script
+ args.entryPoint = 'sh'
+ args.entryPointArgs = ['-e', containerPath]
+ try {
+ const { code, output } = await execPodStepWithOutput(
+ [args.entryPoint, ...args.entryPointArgs],
+ state.jobPod,
+ JOB_CONTAINER_NAME
+ )
+ if (code !== 0) {
+ throw new Error(formatScriptError(code, output))
+ }
+ } catch (err) {
+ core.debug(`execPodStep failed: ${formatError(err)}`)
+ if (
+ err instanceof Error &&
+ err.message.startsWith('failed to run script step')
+ ) {
+ throw err
+ }
+ const message = formatError(err)
+ throw new Error(`failed to run script step: ${message}`)
+ } finally {
+ try {
+ fs.rmSync(runnerPath, { force: true })
+ } catch (removeErr) {
+ core.debug(`Failed to remove file ${runnerPath}: ${removeErr}`)
+ }
+ }
+
+ try {
+ core.debug(
+ `Copying from job pod '${state.jobPod}' ${containerTemp} to ${runnerTemp}`
+ )
+ await execCpFromPod(
+ state.jobPod,
+ `${containerTemp}/_runner_file_commands`,
+ `${workdir}/_temp`
+ )
+ } catch (error) {
+ core.warning('Failed to copy _temp from pod')
+ }
+}
+ |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| File | ++ | Statements | ++ | Branches | ++ | Functions | ++ | Lines | ++ |
|---|---|---|---|---|---|---|---|---|---|
| index.ts | +
+
+ |
+ 0% | +0/22 | +0% | +0/7 | +0% | +0/1 | +0% | +0/22 | +
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 +33 +34 +35 +36 +37 +38 +39 +40 +41 +42 +43 +44 +45 +46 +47 +48 +49 +50 +51 +52 +53 +54 +55 +56 +57 | + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + | import * as core from '@actions/core'
+import {
+ Command,
+ getInputFromStdin,
+ PrepareJobArgs,
+ RunContainerStepArgs,
+ RunScriptStepArgs
+} from 'hooklib'
+import {
+ cleanupJob,
+ prepareJob,
+ runContainerStep,
+ runScriptStep
+} from './hooks'
+import { isAuthPermissionsOK, namespace, requiredPermissions } from './k8s'
+
+async function run(): Promise<void> {
+ try {
+ const input = await getInputFromStdin()
+
+ const args = input['args']
+ const command = input['command']
+ const responseFile = input['responseFile']
+ const state = input['state']
+ if (!(await isAuthPermissionsOK())) {
+ throw new Error(
+ `The Service account needs the following permissions ${JSON.stringify(
+ requiredPermissions
+ )} on the pod resource in the '${namespace()}' namespace. Please contact your self hosted runner administrator.`
+ )
+ }
+
+ let exitCode = 0
+ switch (command) {
+ case Command.PrepareJob:
+ await prepareJob(args as PrepareJobArgs, responseFile)
+ return process.exit(0)
+ case Command.CleanupJob:
+ await cleanupJob()
+ return process.exit(0)
+ case Command.RunScriptStep:
+ await runScriptStep(args as RunScriptStepArgs, state)
+ return process.exit(0)
+ case Command.RunContainerStep:
+ exitCode = await runContainerStep(args as RunContainerStepArgs)
+ return process.exit(exitCode)
+ default:
+ throw new Error(`Command not recognized: ${command}`)
+ }
+ } catch (error) {
+ core.error(error as Error)
+ process.exit(1)
+ }
+}
+
+void run()
+ |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 +33 +34 +35 +36 +37 +38 +39 +40 +41 +42 +43 +44 +45 +46 +47 +48 +49 +50 +51 +52 +53 +54 +55 +56 +57 +58 +59 +60 +61 +62 +63 +64 +65 +66 +67 +68 +69 +70 +71 +72 +73 +74 +75 +76 +77 +78 +79 +80 +81 +82 +83 +84 +85 +86 +87 +88 +89 +90 +91 +92 +93 +94 +95 +96 +97 +98 +99 +100 +101 +102 +103 +104 +105 +106 +107 +108 +109 +110 +111 +112 +113 +114 +115 +116 +117 +118 +119 +120 +121 +122 +123 +124 +125 +126 +127 +128 +129 +130 +131 +132 +133 +134 +135 +136 +137 | + + + + + + + + + + + + + +22x +22x + + + +20x +20x +20x +2x + + +20x +20x + + + +38x +38x +8x +8x + +30x + + + +21x +8x + +21x +9x +9x + +21x +6x +6x + + + + +16x + + +16x + +16x +9x +1x +1x + +9x +3x + + +3x +3x +3x + + + +3x + + + + + + + + + +16x + +16x +1x + + +1x +1x + + + +16x +1x +1x + + +16x +1x +1x + + + +16x +21x + +4x + +17x +16x +16x +16x +8x +8x + +15x + + + +1x +1x + + + +1x + + + + +1x + + + + + | import * as core from '@actions/core'
+
+export interface HeartbeatWebSocket {
+ readyState: number
+ ping(): void
+ close(): void
+ on(event: string, listener: (...args: any[]) => void): this
+ once(event: string, listener: (...args: any[]) => void): this
+}
+
+export function parsePositiveMsEnv(
+ value: string | undefined,
+ fallback: number
+): number {
+ const parsed = Number.parseInt(value ?? '', 10)
+ return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback
+}
+
+export class WebSocketHeartbeat {
+ private pingInterval: ReturnType<typeof setInterval> | null = null
+ private pongTimeout: ReturnType<typeof setTimeout> | null = null
+ private lastHeartbeatLog = 0
+ private static readonly LOG_INTERVAL_MS = 2 * 60 * 1000
+
+ constructor(
+ private readonly pingPeriodMs: number,
+ private readonly pongDeadlineMs: number
+ ) {}
+
+ private shouldLog(): boolean {
+ const now = Date.now()
+ if (now - this.lastHeartbeatLog >= WebSocketHeartbeat.LOG_INTERVAL_MS) {
+ this.lastHeartbeatLog = now
+ return true
+ }
+ return false
+ }
+
+ stop(): void {
+ if (this.shouldLog()) {
+ core.debug('[Heartbeat] stopping heartbeat')
+ }
+ if (this.pingInterval) {
+ clearInterval(this.pingInterval)
+ this.pingInterval = null
+ }
+ if (this.pongTimeout) {
+ clearTimeout(this.pongTimeout)
+ this.pongTimeout = null
+ }
+ }
+
+ start(ws: HeartbeatWebSocket, reject: (err: Error) => void): void {
+ core.debug(
+ `[Heartbeat] Starting with period=${this.pingPeriodMs}ms, deadline=${this.pongDeadlineMs}ms`
+ )
+ this.lastHeartbeatLog = Date.now()
+
+ const resetPongTimeout = (): void => {
+ if (this.pongTimeout) {
+ clearTimeout(this.pongTimeout)
+ this.pongTimeout = null
+ }
+ this.pongTimeout = setTimeout(() => {
+ core.warning(
+ `[Heartbeat] No pong received in ${this.pongDeadlineMs}ms, closing stale connection`
+ )
+ this.stop()
+ try {
+ ws.close()
+ } catch {
+ // ignore errors closing an already-closing socket
+ }
+ reject(
+ new Error(
+ `WebSocket heartbeat timeout: no pong within ${this.pongDeadlineMs}ms`
+ )
+ )
+ }, this.pongDeadlineMs)
+ }
+
+ // Arm the deadline only after the first ping is sent, not immediately on
+ // start, so a slow CONNECTING socket cannot time out before any exchange.
+ let deadlineArmed = false
+
+ ws.on('pong', () => {
+ Iif (this.shouldLog()) {
+ core.debug('[Heartbeat] Pong received')
+ }
+ Eif (deadlineArmed) {
+ resetPongTimeout()
+ }
+ })
+
+ ws.on('error', (err: Error) => {
+ core.error(`[Heartbeat] WebSocket error: ${err.message}`)
+ this.stop()
+ })
+
+ ws.on('close', () => {
+ core.debug('[Heartbeat] WebSocket closed, stopping heartbeat')
+ this.stop()
+ })
+
+ // WebSocket readyState: 0 = CONNECTING, 1 = OPEN, 2 = CLOSING, 3 = CLOSED
+ this.pingInterval = setInterval(() => {
+ if (ws.readyState === 0) {
+ // Still connecting — skip this tick but keep the interval alive
+ return
+ }
+ if (ws.readyState === 1) {
+ try {
+ ws.ping()
+ if (!deadlineArmed) {
+ deadlineArmed = true
+ resetPongTimeout()
+ }
+ Iif (this.shouldLog()) {
+ core.debug('[Heartbeat] Ping sent')
+ }
+ } catch (err) {
+ core.error(`[Heartbeat] Ping failed: ${err}`)
+ this.stop()
+ }
+ } else {
+ // CLOSING (2) or CLOSED (3)
+ Iif (this.shouldLog()) {
+ core.debug(
+ `[Heartbeat] WebSocket closing/closed (readyState=${ws.readyState}), stopping heartbeat`
+ )
+ }
+ this.stop()
+ }
+ }, this.pingPeriodMs)
+ }
+}
+ |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| File | ++ | Statements | ++ | Branches | ++ | Functions | ++ | Lines | ++ |
|---|---|---|---|---|---|---|---|---|---|
| heartbeat.ts | +
+
+ |
+ 95.08% | +58/61 | +86.66% | +26/30 | +100% | +11/11 | +95.08% | +58/61 | +
| index.ts | +
+
+ |
+ 73.6% | +502/682 | +70.31% | +225/320 | +72.07% | +80/111 | +73.91% | +493/667 | +
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 +33 +34 +35 +36 +37 +38 +39 +40 +41 +42 +43 +44 +45 +46 +47 +48 +49 +50 +51 +52 +53 +54 +55 +56 +57 +58 +59 +60 +61 +62 +63 +64 +65 +66 +67 +68 +69 +70 +71 +72 +73 +74 +75 +76 +77 +78 +79 +80 +81 +82 +83 +84 +85 +86 +87 +88 +89 +90 +91 +92 +93 +94 +95 +96 +97 +98 +99 +100 +101 +102 +103 +104 +105 +106 +107 +108 +109 +110 +111 +112 +113 +114 +115 +116 +117 +118 +119 +120 +121 +122 +123 +124 +125 +126 +127 +128 +129 +130 +131 +132 +133 +134 +135 +136 +137 +138 +139 +140 +141 +142 +143 +144 +145 +146 +147 +148 +149 +150 +151 +152 +153 +154 +155 +156 +157 +158 +159 +160 +161 +162 +163 +164 +165 +166 +167 +168 +169 +170 +171 +172 +173 +174 +175 +176 +177 +178 +179 +180 +181 +182 +183 +184 +185 +186 +187 +188 +189 +190 +191 +192 +193 +194 +195 +196 +197 +198 +199 +200 +201 +202 +203 +204 +205 +206 +207 +208 +209 +210 +211 +212 +213 +214 +215 +216 +217 +218 +219 +220 +221 +222 +223 +224 +225 +226 +227 +228 +229 +230 +231 +232 +233 +234 +235 +236 +237 +238 +239 +240 +241 +242 +243 +244 +245 +246 +247 +248 +249 +250 +251 +252 +253 +254 +255 +256 +257 +258 +259 +260 +261 +262 +263 +264 +265 +266 +267 +268 +269 +270 +271 +272 +273 +274 +275 +276 +277 +278 +279 +280 +281 +282 +283 +284 +285 +286 +287 +288 +289 +290 +291 +292 +293 +294 +295 +296 +297 +298 +299 +300 +301 +302 +303 +304 +305 +306 +307 +308 +309 +310 +311 +312 +313 +314 +315 +316 +317 +318 +319 +320 +321 +322 +323 +324 +325 +326 +327 +328 +329 +330 +331 +332 +333 +334 +335 +336 +337 +338 +339 +340 +341 +342 +343 +344 +345 +346 +347 +348 +349 +350 +351 +352 +353 +354 +355 +356 +357 +358 +359 +360 +361 +362 +363 +364 +365 +366 +367 +368 +369 +370 +371 +372 +373 +374 +375 +376 +377 +378 +379 +380 +381 +382 +383 +384 +385 +386 +387 +388 +389 +390 +391 +392 +393 +394 +395 +396 +397 +398 +399 +400 +401 +402 +403 +404 +405 +406 +407 +408 +409 +410 +411 +412 +413 +414 +415 +416 +417 +418 +419 +420 +421 +422 +423 +424 +425 +426 +427 +428 +429 +430 +431 +432 +433 +434 +435 +436 +437 +438 +439 +440 +441 +442 +443 +444 +445 +446 +447 +448 +449 +450 +451 +452 +453 +454 +455 +456 +457 +458 +459 +460 +461 +462 +463 +464 +465 +466 +467 +468 +469 +470 +471 +472 +473 +474 +475 +476 +477 +478 +479 +480 +481 +482 +483 +484 +485 +486 +487 +488 +489 +490 +491 +492 +493 +494 +495 +496 +497 +498 +499 +500 +501 +502 +503 +504 +505 +506 +507 +508 +509 +510 +511 +512 +513 +514 +515 +516 +517 +518 +519 +520 +521 +522 +523 +524 +525 +526 +527 +528 +529 +530 +531 +532 +533 +534 +535 +536 +537 +538 +539 +540 +541 +542 +543 +544 +545 +546 +547 +548 +549 +550 +551 +552 +553 +554 +555 +556 +557 +558 +559 +560 +561 +562 +563 +564 +565 +566 +567 +568 +569 +570 +571 +572 +573 +574 +575 +576 +577 +578 +579 +580 +581 +582 +583 +584 +585 +586 +587 +588 +589 +590 +591 +592 +593 +594 +595 +596 +597 +598 +599 +600 +601 +602 +603 +604 +605 +606 +607 +608 +609 +610 +611 +612 +613 +614 +615 +616 +617 +618 +619 +620 +621 +622 +623 +624 +625 +626 +627 +628 +629 +630 +631 +632 +633 +634 +635 +636 +637 +638 +639 +640 +641 +642 +643 +644 +645 +646 +647 +648 +649 +650 +651 +652 +653 +654 +655 +656 +657 +658 +659 +660 +661 +662 +663 +664 +665 +666 +667 +668 +669 +670 +671 +672 +673 +674 +675 +676 +677 +678 +679 +680 +681 +682 +683 +684 +685 +686 +687 +688 +689 +690 +691 +692 +693 +694 +695 +696 +697 +698 +699 +700 +701 +702 +703 +704 +705 +706 +707 +708 +709 +710 +711 +712 +713 +714 +715 +716 +717 +718 +719 +720 +721 +722 +723 +724 +725 +726 +727 +728 +729 +730 +731 +732 +733 +734 +735 +736 +737 +738 +739 +740 +741 +742 +743 +744 +745 +746 +747 +748 +749 +750 +751 +752 +753 +754 +755 +756 +757 +758 +759 +760 +761 +762 +763 +764 +765 +766 +767 +768 +769 +770 +771 +772 +773 +774 +775 +776 +777 +778 +779 +780 +781 +782 +783 +784 +785 +786 +787 +788 +789 +790 +791 +792 +793 +794 +795 +796 +797 +798 +799 +800 +801 +802 +803 +804 +805 +806 +807 +808 +809 +810 +811 +812 +813 +814 +815 +816 +817 +818 +819 +820 +821 +822 +823 +824 +825 +826 +827 +828 +829 +830 +831 +832 +833 +834 +835 +836 +837 +838 +839 +840 +841 +842 +843 +844 +845 +846 +847 +848 +849 +850 +851 +852 +853 +854 +855 +856 +857 +858 +859 +860 +861 +862 +863 +864 +865 +866 +867 +868 +869 +870 +871 +872 +873 +874 +875 +876 +877 +878 +879 +880 +881 +882 +883 +884 +885 +886 +887 +888 +889 +890 +891 +892 +893 +894 +895 +896 +897 +898 +899 +900 +901 +902 +903 +904 +905 +906 +907 +908 +909 +910 +911 +912 +913 +914 +915 +916 +917 +918 +919 +920 +921 +922 +923 +924 +925 +926 +927 +928 +929 +930 +931 +932 +933 +934 +935 +936 +937 +938 +939 +940 +941 +942 +943 +944 +945 +946 +947 +948 +949 +950 +951 +952 +953 +954 +955 +956 +957 +958 +959 +960 +961 +962 +963 +964 +965 +966 +967 +968 +969 +970 +971 +972 +973 +974 +975 +976 +977 +978 +979 +980 +981 +982 +983 +984 +985 +986 +987 +988 +989 +990 +991 +992 +993 +994 +995 +996 +997 +998 +999 +1000 +1001 +1002 +1003 +1004 +1005 +1006 +1007 +1008 +1009 +1010 +1011 +1012 +1013 +1014 +1015 +1016 +1017 +1018 +1019 +1020 +1021 +1022 +1023 +1024 +1025 +1026 +1027 +1028 +1029 +1030 +1031 +1032 +1033 +1034 +1035 +1036 +1037 +1038 +1039 +1040 +1041 +1042 +1043 +1044 +1045 +1046 +1047 +1048 +1049 +1050 +1051 +1052 +1053 +1054 +1055 +1056 +1057 +1058 +1059 +1060 +1061 +1062 +1063 +1064 +1065 +1066 +1067 +1068 +1069 +1070 +1071 +1072 +1073 +1074 +1075 +1076 +1077 +1078 +1079 +1080 +1081 +1082 +1083 +1084 +1085 +1086 +1087 +1088 +1089 +1090 +1091 +1092 +1093 +1094 +1095 +1096 +1097 +1098 +1099 +1100 +1101 +1102 +1103 +1104 +1105 +1106 +1107 +1108 +1109 +1110 +1111 +1112 +1113 +1114 +1115 +1116 +1117 +1118 +1119 +1120 +1121 +1122 +1123 +1124 +1125 +1126 +1127 +1128 +1129 +1130 +1131 +1132 +1133 +1134 +1135 +1136 +1137 +1138 +1139 +1140 +1141 +1142 +1143 +1144 +1145 +1146 +1147 +1148 +1149 +1150 +1151 +1152 +1153 +1154 +1155 +1156 +1157 +1158 +1159 +1160 +1161 +1162 +1163 +1164 +1165 +1166 +1167 +1168 +1169 +1170 +1171 +1172 +1173 +1174 +1175 +1176 +1177 +1178 +1179 +1180 +1181 +1182 +1183 +1184 +1185 +1186 +1187 +1188 +1189 +1190 +1191 +1192 +1193 +1194 +1195 +1196 +1197 +1198 +1199 +1200 +1201 +1202 +1203 +1204 +1205 +1206 +1207 +1208 +1209 +1210 +1211 +1212 +1213 +1214 +1215 +1216 +1217 +1218 +1219 +1220 +1221 +1222 +1223 +1224 +1225 +1226 +1227 +1228 +1229 +1230 +1231 +1232 +1233 +1234 +1235 +1236 +1237 +1238 +1239 +1240 +1241 +1242 +1243 +1244 +1245 +1246 +1247 +1248 +1249 +1250 +1251 +1252 +1253 +1254 +1255 +1256 +1257 +1258 +1259 +1260 +1261 +1262 +1263 +1264 +1265 +1266 +1267 +1268 +1269 +1270 +1271 +1272 +1273 +1274 +1275 +1276 +1277 +1278 +1279 +1280 +1281 +1282 +1283 +1284 +1285 +1286 +1287 +1288 +1289 +1290 +1291 +1292 +1293 +1294 +1295 +1296 +1297 +1298 +1299 +1300 +1301 +1302 +1303 +1304 +1305 +1306 +1307 +1308 +1309 +1310 +1311 +1312 +1313 +1314 +1315 +1316 +1317 +1318 +1319 +1320 +1321 +1322 +1323 +1324 +1325 +1326 +1327 +1328 +1329 +1330 +1331 +1332 +1333 +1334 +1335 +1336 +1337 +1338 +1339 +1340 +1341 +1342 +1343 +1344 +1345 +1346 +1347 +1348 +1349 +1350 +1351 +1352 +1353 +1354 +1355 +1356 +1357 +1358 +1359 +1360 +1361 +1362 +1363 +1364 +1365 +1366 +1367 +1368 +1369 +1370 +1371 +1372 +1373 +1374 +1375 +1376 +1377 +1378 +1379 +1380 +1381 +1382 +1383 +1384 +1385 +1386 +1387 +1388 +1389 +1390 +1391 +1392 +1393 +1394 +1395 +1396 +1397 +1398 +1399 +1400 +1401 +1402 +1403 +1404 +1405 +1406 +1407 +1408 +1409 +1410 +1411 +1412 +1413 +1414 +1415 +1416 +1417 +1418 +1419 +1420 +1421 +1422 +1423 +1424 +1425 +1426 +1427 +1428 +1429 +1430 +1431 +1432 +1433 +1434 +1435 +1436 +1437 +1438 +1439 +1440 +1441 +1442 +1443 +1444 +1445 +1446 +1447 +1448 +1449 +1450 +1451 +1452 +1453 +1454 +1455 +1456 +1457 +1458 +1459 +1460 +1461 +1462 +1463 +1464 +1465 +1466 +1467 +1468 +1469 +1470 +1471 +1472 +1473 +1474 +1475 +1476 +1477 +1478 +1479 +1480 +1481 +1482 +1483 +1484 +1485 +1486 +1487 +1488 +1489 +1490 +1491 +1492 +1493 +1494 +1495 +1496 +1497 +1498 +1499 +1500 +1501 +1502 +1503 +1504 +1505 +1506 +1507 +1508 +1509 +1510 +1511 +1512 +1513 +1514 +1515 +1516 +1517 +1518 +1519 +1520 +1521 +1522 +1523 +1524 +1525 +1526 +1527 +1528 +1529 +1530 +1531 +1532 +1533 +1534 +1535 +1536 +1537 +1538 +1539 +1540 +1541 +1542 +1543 +1544 +1545 +1546 +1547 +1548 +1549 +1550 +1551 +1552 +1553 +1554 +1555 +1556 +1557 +1558 +1559 +1560 +1561 +1562 +1563 +1564 +1565 +1566 +1567 +1568 +1569 +1570 +1571 +1572 +1573 +1574 +1575 +1576 +1577 +1578 +1579 +1580 +1581 +1582 +1583 +1584 +1585 +1586 +1587 +1588 +1589 +1590 +1591 +1592 +1593 +1594 +1595 +1596 +1597 +1598 +1599 +1600 +1601 +1602 +1603 +1604 +1605 +1606 +1607 +1608 +1609 +1610 +1611 +1612 +1613 +1614 +1615 +1616 +1617 +1618 +1619 +1620 +1621 +1622 +1623 +1624 +1625 +1626 +1627 +1628 +1629 +1630 +1631 +1632 +1633 +1634 +1635 +1636 +1637 +1638 +1639 +1640 +1641 +1642 +1643 +1644 +1645 +1646 +1647 +1648 +1649 +1650 +1651 +1652 +1653 +1654 +1655 +1656 +1657 +1658 +1659 +1660 +1661 +1662 +1663 +1664 +1665 +1666 +1667 +1668 +1669 +1670 +1671 +1672 +1673 +1674 +1675 +1676 +1677 +1678 +1679 +1680 +1681 +1682 +1683 +1684 +1685 +1686 +1687 +1688 +1689 +1690 +1691 +1692 +1693 +1694 +1695 +1696 +1697 +1698 +1699 +1700 +1701 +1702 +1703 +1704 +1705 +1706 +1707 +1708 +1709 +1710 +1711 +1712 +1713 +1714 +1715 +1716 +1717 +1718 +1719 +1720 +1721 +1722 +1723 +1724 +1725 +1726 +1727 +1728 +1729 +1730 +1731 +1732 +1733 +1734 +1735 +1736 +1737 +1738 +1739 +1740 +1741 +1742 +1743 +1744 +1745 +1746 +1747 +1748 +1749 +1750 +1751 +1752 +1753 +1754 +1755 +1756 +1757 +1758 +1759 +1760 +1761 +1762 +1763 +1764 +1765 +1766 +1767 +1768 +1769 +1770 +1771 +1772 +1773 +1774 +1775 +1776 +1777 +1778 +1779 +1780 +1781 +1782 +1783 +1784 +1785 +1786 +1787 +1788 +1789 +1790 +1791 +1792 +1793 +1794 +1795 +1796 +1797 +1798 +1799 +1800 +1801 +1802 +1803 +1804 +1805 +1806 +1807 +1808 +1809 +1810 +1811 +1812 +1813 +1814 +1815 +1816 +1817 +1818 +1819 +1820 +1821 +1822 +1823 +1824 +1825 +1826 +1827 +1828 +1829 +1830 +1831 +1832 +1833 +1834 +1835 +1836 +1837 +1838 +1839 | + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +1x + +1x + +1x +1x +1x + +1x + +1x + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +5x +5x +4x + +5x +1x + + +5x + +5x +5x + +5x +5x + +5x +5x + + +5x + +5x +5x +5x + + + + + +5x +5x + +5x + + + + + + +5x +5x + + +5x + + + + + + + + + + + + + + + + + + + + + + + + + + + +5x + +5x + + + + + + + + + + + + + + +5x + + + + + + + + + +5x +1x + + +5x +1x + + +5x + + + + + + + + + + +1x + +1x +1x + +1x +1x + +1x +1x + + +1x + +1x +1x + +1x + +1x + + + + + + + + + + + + + + +1x + + + +1x + + + +1x + + + + + + +3x + + + + + + + + + + + + +7x +7x + + + +7x + +7x +7x + + + +7x + + + +7x + + + +7x + +7x +7x +7x + +7x + + + + + + + + + + +6x + + + +6x + + +6x +6x +6x + + + + + + + + + + + + + + + + + + + + + +6x +3x +3x +3x + +3x + + +3x +3x + + + + +6x +6x +6x +4x + +2x + + + +5x +5x + + +5x +5x +2x +2x + + + + + + +2x +1x +1x + +2x + + + +4x + + + + + + + + + + + + + + + + +5x +5x + + +5x +5x +2x +2x +2x + + + +5x +5x +1x +1x +1x +2x + + +5x +5x + + + + + + +5x +5x + + + +5x + + + + + +5x + +1x +1x +1x +1x + + + + + +5x + + + + + + + + + + + + + + +5x +4x +4x + + +5x +5x + + + + + + + + + + +3x +3x +3x +1x + + + + +2x +2x +1x + +1x + + + + + + + + + + +2x +2x +2x +2x +1x + +1x + + + + + + + + + + +3x + +3x +3x + +2x +2x +2x + + + + + + +3x +3x + + + + + + + + + + +2x +2x +1x + +1x + + + + + +1x + + + +1x + + +1x + + +1x + +4x + +1x + +1x +1x +1x + + + + + +2x + +2x + + + +2x +2x +1x + +2x +2x +2x + +1x + +2x + +1x + +1x +1x +1x + +1x + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +2x +2x +2x +2x +1x + + +1x + + + + + + + + +2x + + + + + + + + + + + +2x + +2x +2x +2x +2x +2x +2x +2x +2x + + +2x +2x +2x + + + + + +2x + + + + + + + + +2x + +2x +2x +2x +2x +2x +2x + +2x + + +2x +2x +2x +2x + + +2x + + + +2x + + + +3x + + + + + + +2x + + + +2x +1x + + +1x + + +3x + + + + +1x + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +1x + + + + + + + + + + + + + + + + + + + + + + +1x + + + + + + + + + + + + + +1x + + + + + + + +1x + + + + + + +28x +28x +26x + +2x +2x +2x +2x +2x + + +2x + + + + + + +21x +21x +20x + +1x +1x +1x +1x +1x + + +1x + + + + +20x +20x +19x + +1x +1x +3x +3x +1x + + +1x + + + + + + + + + + + +11x +26x + + + + + + + + + + + + +15x +15x +5x +1x + +4x +4x +8x +8x +5x +5x + +1x + + + + +4x +4x +4x + + + +10x + +5x + + + + + + + +2x + +2x + +1x + + + + +20x +20x +20x + + + +20x +13x +13x +10x +10x +10x +10x + + +20x + + + + + + +14x +5x + +9x +2x + +7x +6x + +1x + + + +1x + +1x + +3x + + +1x + + + +18x +18x +18x + + + +18x +11x +11x +7x +7x +7x +7x + + +18x + + + +10x + +7x + + + + + + +1x + + + + + + +2x + + + + + + + + + + + + + + + + + + + +19x +19x + + + + +19x +19x + + + +18x + +1x + + + + +1x + + +18x +18x +18x +14x + + + + +1x + + + + +13x + + + +2x + + + + +2x + +11x +1x + +10x +10x +14x +14x +14x +14x + +18x + + + + + + + + + + +7x +7x + +1x + + + + +6x +6x + + +6x +6x + + +7x +7x + + + + +6x + + + + + + +6x + + +6x +6x + + + +7x +7x +2x +2x + + +1x + + + + + + +2x + +2x +1x +1x +1x + + + + +6x +1x + + + +6x +6x +3x + + +6x + + +6x + + + + + + + +6x +6x + + + +6x + + + + + + + + + +6x +6x +3x + + +3x + + + +3x + +3x +3x +3x +3x + +3x + + + + +3x + + + + + + + + + +9x + + + + + + + + + + + + + +8x + +8x + +8x + + + +8x + + + + + +8x + + + + + + + + + + + + + +6x +6x +6x + +8x +8x + + + + +2x + + + + +2x +2x + + + + + + + +2x + +6x +6x +2x + + + + + + + +4x +1x +1x +1x + + + + +1x + + + + + + + +3x +3x +3x +3x + + + + + + + + + + + + + + + + + + + + +4x + +4x +1x + + +3x +3x +2x + + +2x + + +1x + + + +15x + + + + + +1x + + + + + + + + +13x +3x + +10x + + + +2x + + + +2x +1x + +1x + + + + + + +2x +2x +2x + + + + +2x + + + + +2x +2x +2x +1x +1x + + + + + +2x + + + +2x +1x + + +1x + +2x + + + + + + + +1x + + + +1x + + + +2x +2x +2x +8x +26x +26x +26x +26x +26x +26x +26x +26x + + + + +2x +14x + + + + + + +2x +2x +2x + + + + + + + + + +1x + + +2x + + + +117x +116x + + +1x +1x +1x + + + + + + + + + + + + + + + + + + + + + +8x +8x +8x +8x +2x + + + + +2x +2x + +2x +2x + + +2x +2x + +2x + + + + + + + + +12x +12x +2x + +10x +11x +11x +1x + + +10x +10x + + +11x +11x +1x + + +9x +13x +13x +3x + +10x + + +9x +5x + +4x +4x + + +6x + +5x + + + +2x + + + + + + +2x + + + + + + +3x + + + +2x + + | import * as core from '@actions/core'
+import * as fs from 'fs'
+import * as path from 'path'
+import { spawn } from 'child_process'
+import * as k8s from '@kubernetes/client-node'
+import tar from 'tar-fs'
+import * as stream from 'stream'
+import { WritableStreamBuffer } from 'stream-buffers'
+import { createHash } from 'crypto'
+import type { ContainerInfo, Registry } from 'hooklib'
+import {
+ getSecretName,
+ JOB_CONTAINER_NAME,
+ RunnerInstanceLabel
+} from '../hooks/constants'
+import {
+ PodPhase,
+ formatError,
+ mergePodSpecWithOptions,
+ mergeObjectMeta,
+ fixArgs,
+ listDirAllCommand,
+ sleep,
+ EXTERNALS_VOLUME_NAME,
+ GITHUB_VOLUME_NAME,
+ WORK_VOLUME
+} from './utils'
+import * as shlex from 'shlex'
+import { parsePositiveMsEnv, WebSocketHeartbeat } from './heartbeat'
+import type { HeartbeatWebSocket } from './heartbeat'
+
+const kc = new k8s.KubeConfig()
+
+kc.loadFromDefault()
+
+const k8sApi = kc.makeApiClient(k8s.CoreV1Api)
+const k8sBatchV1Api = kc.makeApiClient(k8s.BatchV1Api)
+const k8sAuthorizationV1Api = kc.makeApiClient(k8s.AuthorizationV1Api)
+
+const DEFAULT_WAIT_FOR_POD_TIME_SECONDS = 10 * 60 // 10 min
+
+export const requiredPermissions = [
+ {
+ group: '',
+ verbs: ['get', 'list', 'create', 'delete'],
+ resource: 'pods',
+ subresource: ''
+ },
+ {
+ group: '',
+ verbs: ['get', 'create'],
+ resource: 'pods',
+ subresource: 'exec'
+ },
+ {
+ group: '',
+ verbs: ['get', 'list', 'watch'],
+ resource: 'pods',
+ subresource: 'log'
+ },
+ {
+ group: '',
+ verbs: ['create', 'delete', 'get', 'list'],
+ resource: 'secrets',
+ subresource: ''
+ }
+]
+
+export async function createJobPod(
+ name: string,
+ jobContainer?: k8s.V1Container,
+ services?: k8s.V1Container[],
+ registry?: Registry,
+ extension?: k8s.V1PodTemplateSpec
+): Promise<k8s.V1Pod> {
+ const containers: k8s.V1Container[] = []
+ if (jobContainer) {
+ containers.push(jobContainer)
+ }
+ if (services?.length) {
+ containers.push(...services)
+ }
+
+ const appPod = new k8s.V1Pod()
+
+ appPod.apiVersion = 'v1'
+ appPod.kind = 'Pod'
+
+ appPod.metadata = new k8s.V1ObjectMeta()
+ appPod.metadata.name = name
+
+ const instanceLabel = new RunnerInstanceLabel()
+ appPod.metadata.labels = {
+ [instanceLabel.key]: instanceLabel.value
+ }
+ appPod.metadata.annotations = {}
+
+ appPod.spec = new k8s.V1PodSpec()
+ appPod.spec.containers = containers
+ appPod.spec.securityContext = {
+ fsGroup: 1001
+ }
+
+ // Extract working directory from GITHUB_WORKSPACE
+ // GITHUB_WORKSPACE is like /__w/repo-name/repo-name
+ const githubWorkspace = process.env.GITHUB_WORKSPACE
+ const workingDirPath = githubWorkspace?.split('/').slice(-2).join('/') ?? ''
+
+ const initCommands = [
+ 'mkdir -p /mnt/externals',
+ 'mkdir -p /mnt/work',
+ 'mkdir -p /mnt/github',
+ 'mv /home/runner/externals/* /mnt/externals/'
+ ]
+
+ Eif (workingDirPath) {
+ initCommands.push(`mkdir -p /mnt/work/${workingDirPath}`)
+ }
+
+ appPod.spec.initContainers = [
+ {
+ name: 'fs-init',
+ image:
+ process.env.ACTIONS_RUNNER_IMAGE ||
+ 'ghcr.io/actions/actions-runner:latest',
+ command: ['sh', '-c', initCommands.join(' && ')],
+ securityContext: {
+ runAsGroup: 1001,
+ runAsUser: 1001
+ },
+ volumeMounts: [
+ {
+ name: EXTERNALS_VOLUME_NAME,
+ mountPath: '/mnt/externals'
+ },
+ {
+ name: WORK_VOLUME,
+ mountPath: '/mnt/work'
+ },
+ {
+ name: GITHUB_VOLUME_NAME,
+ mountPath: '/mnt/github'
+ }
+ ]
+ }
+ ]
+
+ appPod.spec.restartPolicy = 'Never'
+
+ appPod.spec.volumes = [
+ {
+ name: EXTERNALS_VOLUME_NAME,
+ emptyDir: {}
+ },
+ {
+ name: GITHUB_VOLUME_NAME,
+ emptyDir: {}
+ },
+ {
+ name: WORK_VOLUME,
+ emptyDir: {}
+ }
+ ]
+
+ Iif (registry) {
+ const secret = await createDockerSecret(registry)
+ if (!secret?.metadata?.name) {
+ throw new Error(`created secret does not have secret.metadata.name`)
+ }
+ const secretReference = new k8s.V1LocalObjectReference()
+ secretReference.name = secret.metadata.name
+ appPod.spec.imagePullSecrets = [secretReference]
+ }
+
+ if (extension?.metadata) {
+ mergeObjectMeta(appPod, extension.metadata)
+ }
+
+ if (extension?.spec) {
+ mergePodSpecWithOptions(appPod.spec, extension.spec)
+ }
+
+ return await k8sApi.createNamespacedPod({
+ namespace: namespace(),
+ body: appPod
+ })
+}
+
+export async function createContainerStepPod(
+ name: string,
+ container: k8s.V1Container,
+ extension?: k8s.V1PodTemplateSpec
+): Promise<k8s.V1Pod> {
+ const appPod = new k8s.V1Pod()
+
+ appPod.apiVersion = 'v1'
+ appPod.kind = 'Pod'
+
+ appPod.metadata = new k8s.V1ObjectMeta()
+ appPod.metadata.name = name
+
+ const instanceLabel = new RunnerInstanceLabel()
+ appPod.metadata.labels = {
+ [instanceLabel.key]: instanceLabel.value
+ }
+ appPod.metadata.annotations = {}
+
+ appPod.spec = new k8s.V1PodSpec()
+ appPod.spec.containers = [container]
+
+ appPod.spec.restartPolicy = 'Never'
+
+ appPod.spec.volumes = [
+ {
+ name: EXTERNALS_VOLUME_NAME,
+ emptyDir: {}
+ },
+ {
+ name: GITHUB_VOLUME_NAME,
+ emptyDir: {}
+ },
+ {
+ name: WORK_VOLUME,
+ emptyDir: {}
+ }
+ ]
+
+ Iif (extension?.metadata) {
+ mergeObjectMeta(appPod, extension.metadata)
+ }
+
+ Iif (extension?.spec) {
+ mergePodSpecWithOptions(appPod.spec, extension.spec)
+ }
+
+ return await k8sApi.createNamespacedPod({
+ namespace: namespace(),
+ body: appPod
+ })
+}
+
+export async function deletePod(name: string): Promise<void> {
+ await k8sApi.deleteNamespacedPod({
+ name,
+ namespace: namespace(),
+ gracePeriodSeconds: 0
+ })
+}
+
+export async function execPodStep(
+ command: string[],
+ podName: string,
+ containerName: string,
+ stdin?: stream.Readable
+): Promise<number> {
+ const exec = new k8s.Exec(kc)
+ core.debug(
+ `[execPodStep] Starting: cmd="${command[0]}" (${command.length} args), pod=${podName}, container=${containerName}`
+ )
+
+ command = fixArgs(command)
+
+ const DEFAULT_PING_PERIOD_MS = 5000
+ const pingPeriodMs = parsePositiveMsEnv(
+ process.env.ACTIONS_RUNNER_HEARTBEAT_PERIOD_MS,
+ DEFAULT_PING_PERIOD_MS
+ )
+ const pongDeadlineMs = parsePositiveMsEnv(
+ process.env.ACTIONS_RUNNER_HEARTBEAT_DEADLINE_MS,
+ pingPeriodMs * 12 + 1000
+ )
+ core.debug(
+ `[execPodStep] Heartbeat config: pingPeriodMs=${pingPeriodMs}, pongDeadlineMs=${pongDeadlineMs}`
+ )
+
+ const heartbeat = new WebSocketHeartbeat(pingPeriodMs, pongDeadlineMs)
+
+ return new Promise<number>((resolve, reject) => {
+ core.debug('[execPodStep] About to call exec.exec')
+ let ws: HeartbeatWebSocket | null = null
+
+ exec
+ .exec(
+ namespace(),
+ podName,
+ containerName,
+ command,
+ process.stdout,
+ process.stderr,
+ stdin ?? null,
+ false /* tty */,
+ async resp => {
+ core.debug(
+ `[execPodStep] execPodStep response: ${JSON.stringify(resp)}`
+ )
+
+ heartbeat.stop()
+
+ // Close WebSocket and wait for it before resolving/rejecting
+ const closeWebSocket = async (): Promise<void> => {
+ const socket = ws
+ Iif (
+ socket &&
+ (socket.readyState === 1 || socket.readyState === 0)
+ ) {
+ return new Promise<void>(closeResolve => {
+ const closeTimeout = setTimeout(() => {
+ core.warning(
+ '[execPodStep] WebSocket close timeout, forcing cleanup'
+ )
+ closeResolve()
+ }, 5000)
+
+ socket.once('close', () => {
+ clearTimeout(closeTimeout)
+ core.debug('[execPodStep] WebSocket closed cleanly')
+ closeResolve()
+ })
+ socket.close()
+ })
+ }
+ }
+
+ if (resp.status === 'Success') {
+ core.debug(`[execPodStep] Success, code: ${resp.code}`)
+ await closeWebSocket()
+ resolve(resp.code || 0)
+ } else {
+ core.debug(
+ `[execPodStep] Failure: ${JSON.stringify({ message: resp?.message, details: resp?.details })}`
+ )
+ await closeWebSocket()
+ reject(new Error(resp?.message || 'execPodStep failed'))
+ }
+ }
+ )
+ .then(websocket => {
+ core.debug('[execPodStep] exec.exec resolved, ws object received')
+ ws = websocket
+ if (ws) {
+ heartbeat.start(ws, reject)
+ } else {
+ core.warning('[Heartbeat] WebSocket is null, heartbeat not started')
+ }
+ })
+ .catch(async e => {
+ heartbeat.stop()
+ core.error(`[execPodStep] exec.exec threw error: ${e}`)
+
+ // Close WebSocket before rejecting with timeout protection
+ const socket = ws
+ if (socket && (socket.readyState === 1 || socket.readyState === 0)) {
+ await new Promise<void>(closeResolve => {
+ const closeTimeout = setTimeout(() => {
+ core.warning(
+ '[execPodStep] WebSocket close timeout in error handler'
+ )
+ closeResolve()
+ }, 5000)
+
+ socket.once('close', () => {
+ clearTimeout(closeTimeout)
+ closeResolve()
+ })
+ socket.close()
+ })
+ }
+
+ reject(e)
+ })
+ })
+}
+
+// Variant of execPodStep that also captures the last `tailLines` lines of
+// combined stdout/stderr so callers can surface the script's own output on
+// failure. Returns the exit code plus the tail (empty string on success or if
+// nothing was emitted). The original streams are still tee'd to
+// process.stdout/process.stderr so GitHub Actions logs are unchanged.
+export async function execPodStepWithOutput(
+ command: string[],
+ podName: string,
+ containerName: string,
+ tailLines = 20,
+ stdin?: stream.Readable
+): Promise<{ code: number; output: string }> {
+ const exec = new k8s.Exec(kc)
+ command = fixArgs(command)
+
+ // Ring buffer of the last N non-empty lines (cap to keep memory bounded).
+ const buffer: string[] = []
+ const push = (line: string): void => {
+ Iif (line.length === 0) return
+ buffer.push(line)
+ Iif (buffer.length > tailLines) buffer.shift()
+ }
+
+ // Separate pending buffers per stream to prevent stdout/stderr interleaving.
+ let pendingOut = ''
+ const ingestOut = (chunk: Buffer | string): void => {
+ pendingOut += chunk.toString('utf8')
+ const lines = pendingOut.split(/\r?\n/)
+ pendingOut = lines.pop() ?? ''
+ for (const line of lines) push(line)
+ }
+
+ let pendingErr = ''
+ const ingestErr = (chunk: Buffer | string): void => {
+ pendingErr += chunk.toString('utf8')
+ const lines = pendingErr.split(/\r?\n/)
+ pendingErr = lines.pop() ?? ''
+ for (const line of lines) push(line)
+ }
+
+ const flushPending = (): void => {
+ Iif (pendingOut) {
+ push(pendingOut)
+ pendingOut = ''
+ }
+ Iif (pendingErr) {
+ push(pendingErr)
+ pendingErr = ''
+ }
+ }
+
+ const capture = new stream.Writable({
+ write(chunk: Buffer | string, _enc, cb) {
+ try {
+ ingestOut(chunk)
+ process.stdout.write(chunk)
+ cb()
+ } catch (e) {
+ cb(e as Error)
+ }
+ }
+ })
+ const captureErr = new stream.Writable({
+ write(chunk: Buffer | string, _enc, cb) {
+ try {
+ ingestErr(chunk)
+ process.stderr.write(chunk)
+ cb()
+ } catch (e) {
+ cb(e as Error)
+ }
+ }
+ })
+
+ // Parse an exit code from a k8s exec error/status message.
+ // Handles both "command terminated with exit code N" and
+ // "command terminated with non-zero exit code: command terminated with exit code N".
+ const parseExitCode = (msg: string | undefined): number | null => {
+ const m = msg?.match(/exit code[:\s]+(\d+)/i)
+ return m ? parseInt(m[1], 10) : null
+ }
+
+ return await new Promise(function (resolve, reject) {
+ exec
+ .exec(
+ namespace(),
+ podName,
+ containerName,
+ command,
+ capture,
+ captureErr,
+ stdin ?? null,
+ false /* tty */,
+ resp => {
+ core.debug(`execPodStepWithOutput response: ${JSON.stringify(resp)}`)
+ flushPending()
+ if (resp.status === 'Success') {
+ resolve({ code: resp.code || 0, output: buffer.join('\n') })
+ } else {
+ // k8s exec returns status='Failure' for non-zero script exit.
+ // resp.code may be undefined depending on k8s version; fall back
+ // to parsing the exit code from the message string.
+ const code = parseExitCode(resp?.message)
+ if (code !== null) {
+ resolve({ code, output: buffer.join('\n') })
+ } else {
+ reject(new Error(resp?.message || 'execPodStepWithOutput failed'))
+ }
+ }
+ }
+ )
+ .catch(e => {
+ // In @kubernetes/client-node v1.x the promise returned by exec()
+ // itself rejects for non-zero exits (the status callback may or may
+ // not fire first). Detect this case via the error message so the
+ // caller can classify the error instead of treating it as a hook
+ // failure. Genuine failures (connection drop, etc.) still reject.
+ flushPending()
+ const errMsg = e instanceof Error ? e.message : String(e)
+ const code = parseExitCode(errMsg)
+ if (code !== null) {
+ resolve({ code, output: buffer.join('\n') })
+ } else {
+ reject(e)
+ }
+ })
+ })
+}
+
+export async function execCalculateOutputHashSorted(
+ podName: string,
+ containerName: string,
+ command: string[]
+): Promise<{ hash: string; lines: string[] }> {
+ const exec = new k8s.Exec(kc)
+
+ let output = ''
+ const outputWriter = new stream.Writable({
+ write(chunk, _enc, cb) {
+ try {
+ output += chunk.toString('utf8')
+ cb()
+ } catch (e) {
+ cb(e as Error)
+ }
+ }
+ })
+
+ await new Promise<void>((resolve, reject) => {
+ exec
+ .exec(
+ namespace(),
+ podName,
+ containerName,
+ command,
+ outputWriter, // capture stdout
+ process.stderr,
+ null,
+ false /* tty */,
+ resp => {
+ core.debug(`internalExecOutput response: ${JSON.stringify(resp)}`)
+ if (resp.status === 'Success') {
+ resolve()
+ } else {
+ core.debug(
+ JSON.stringify({
+ message: resp?.message,
+ details: resp?.details
+ })
+ )
+ reject(new Error(resp?.message || 'internalExecOutput failed'))
+ }
+ }
+ )
+ .catch(e => reject(e))
+ })
+
+ outputWriter.end()
+
+ // Sort lines for consistent ordering across platforms
+ const lines = output
+ .split('\n')
+ .filter(line => line.length > 0)
+ .sort()
+ const sortedOutput = lines.join('\n') + '\n'
+
+ const hash = createHash('sha256')
+ hash.update(sortedOutput)
+ return { hash: hash.digest('hex'), lines }
+}
+
+export async function localCalculateOutputHashSorted(
+ commands: string[]
+): Promise<{ hash: string; lines: string[] }> {
+ return await new Promise<{ hash: string; lines: string[] }>(
+ (resolve, reject) => {
+ const child = spawn(commands[0], commands.slice(1), {
+ stdio: ['ignore', 'pipe', 'ignore']
+ })
+
+ let output = ''
+ child.stdout.on('data', chunk => {
+ output += chunk.toString('utf8')
+ })
+ child.on('error', reject)
+ child.on('close', (code: number) => {
+ if (code === 0) {
+ // Sort lines for consistent ordering across distributions/platforms
+ const lines = output
+ .split('\n')
+ .filter(line => line.length > 0)
+ .sort()
+ const sortedOutput = lines.join('\n') + '\n'
+
+ const hash = createHash('sha256')
+ hash.update(sortedOutput)
+ resolve({ hash: hash.digest('hex'), lines })
+ } else {
+ reject(new Error(`child process exited with code ${code}`))
+ }
+ })
+ }
+ )
+}
+
+export async function execCpToPod(
+ podName: string,
+ runnerPath: string,
+ containerPath: string
+): Promise<void> {
+ core.debug(`Copying ${runnerPath} to pod ${podName} at ${containerPath}`)
+
+ let attempt = 0
+ while (true) {
+ try {
+ const exec = new k8s.Exec(kc)
+ // Use tar to extract with --no-same-owner to avoid ownership issues.
+ // Then use find to fix permissions. The -m flag helps but we also need to fix permissions after.
+ const command = [
+ 'sh',
+ '-c',
+ `tar xf - --no-same-owner -C ${shlex.quote(containerPath)} 2>/dev/null; ` +
+ `find ${shlex.quote(containerPath)} -type f -exec chmod u+rw {} \\; 2>/dev/null; ` +
+ `find ${shlex.quote(containerPath)} -type d -exec chmod u+rwx {} \\; 2>/dev/null; ` +
+ `sync 2>/dev/null || true`
+ ]
+ const readStream = tar.pack(runnerPath)
+ const errStream = new WritableStreamBuffer()
+ await new Promise((resolve, reject) => {
+ exec
+ .exec(
+ namespace(),
+ podName,
+ JOB_CONTAINER_NAME,
+ command,
+ null,
+ errStream,
+ readStream,
+ false,
+ async status => {
+ if (errStream.size()) {
+ return reject(
+ new Error(
+ `Error from execCpToPod - status: ${status.status}, details: \n ${errStream.getContentsAsString()}`
+ )
+ )
+ }
+ resolve(status)
+ }
+ )
+ .catch(e => reject(e))
+ })
+ break
+ } catch (error) {
+ core.debug(`cpToPod: Attempt ${attempt + 1} failed: ${error}`)
+ attempt++
+ if (attempt >= 30) {
+ throw new Error(
+ `cpToPod failed after ${attempt} attempts: ${formatError(error)}`
+ )
+ }
+ await sleep(1000)
+ }
+ }
+
+ let attempts = 15
+ const delay = 1000
+ for (let i = 0; i < attempts; i++) {
+ try {
+ const { hash: want, lines: wantLines } =
+ await localCalculateOutputHashSorted([
+ 'sh',
+ '-c',
+ listDirAllCommand(runnerPath)
+ ])
+
+ const { hash: got, lines: gotLines } =
+ await execCalculateOutputHashSorted(podName, JOB_CONTAINER_NAME, [
+ 'sh',
+ '-c',
+ listDirAllCommand(containerPath)
+ ])
+
+ if (got !== want) {
+ core.debug(
+ `[cpToPod hash mismatch attempt ${i + 1}/${attempts}] want='${want}' got='${got}'`
+ )
+ core.debug(
+ `[cpToPod] runner file count=${wantLines.length} pod file count=${gotLines.length}`
+ )
+ const wantMap = new Map(
+ wantLines.map(l => [l.replace(/^\d+ /, ''), l.split(' ')[0]])
+ )
+ const gotMap = new Map(
+ gotLines.map(l => [l.replace(/^\d+ /, ''), l.split(' ')[0]])
+ )
+ const onlyInWant = wantLines.filter(
+ l => !gotMap.has(l.replace(/^\d+ /, ''))
+ )
+ const onlyInGot = gotLines.filter(
+ l => !wantMap.has(l.replace(/^\d+ /, ''))
+ )
+ const sizeDiff = wantLines
+ .filter(l => {
+ const name = l.replace(/^\d+ /, '')
+ return gotMap.has(name) && gotMap.get(name) !== wantMap.get(name)
+ })
+ .map(l => {
+ const name = l.replace(/^\d+ /, '')
+ return `${name}: runner=${wantMap.get(name)} pod=${gotMap.get(name)}`
+ })
+ core.debug(`[cpToPod] only in runner: ${JSON.stringify(onlyInWant)}`)
+ core.debug(`[cpToPod] only in pod: ${JSON.stringify(onlyInGot)}`)
+ core.debug(`[cpToPod] size mismatch: ${JSON.stringify(sizeDiff)}`)
+ await sleep(delay)
+ continue
+ }
+
+ break
+ } catch (error) {
+ core.debug(`Attempt ${i + 1} failed: ${error}`)
+ await sleep(delay)
+ }
+ }
+}
+
+export async function execCpFromPod(
+ podName: string,
+ containerPath: string,
+ parentRunnerPath: string
+): Promise<void> {
+ const targetRunnerPath = `${parentRunnerPath}/${path.basename(containerPath)}`
+ core.debug(
+ `Copying from pod ${podName} ${containerPath} to ${targetRunnerPath}`
+ )
+
+ // Clear target before extracting so deleted-on-pod files don't linger locally.
+ // tar.extract() appends into the destination; without this, stale files (e.g.
+ // git-credentials removed by checkout cleanup inside the pod) cause a permanent
+ // hash mismatch that no amount of retrying can resolve.
+ if (fs.existsSync(targetRunnerPath)) {
+ fs.rmSync(targetRunnerPath, { recursive: true, force: true })
+ }
+
+ let attempt = 0
+ while (true) {
+ try {
+ // make temporary directory
+ const exec = new k8s.Exec(kc)
+ const containerPaths = containerPath.split('/')
+ const dirname = containerPaths.pop() as string
+ const command = [
+ 'tar',
+ 'cf',
+ '-',
+ '-C',
+ containerPaths.join('/') || '/',
+ dirname
+ ]
+ const writerStream = tar.extract(parentRunnerPath)
+ const errStream = new WritableStreamBuffer()
+
+ await new Promise((resolve, reject) => {
+ // Resolve only after writerStream finishes flushing to disk.
+ // The k8s status callback fires when the pod-side tar process exits,
+ // but tar-fs may still be writing buffered data to the local filesystem.
+ // Waiting for 'finish' ensures all files are on disk before hash check.
+ writerStream.on('finish', resolve)
+ writerStream.on('error', reject)
+ exec
+ .exec(
+ namespace(),
+ podName,
+ JOB_CONTAINER_NAME,
+ command,
+ writerStream,
+ errStream,
+ null,
+ false,
+ async (_s: k8s.V1Status) => {
+ if (errStream.size()) {
+ return reject(
+ new Error(
+ `Error from cpFromPod - details: \n ${errStream.getContentsAsString()}`
+ )
+ )
+ }
+ }
+ )
+ .catch(e => reject(e))
+ })
+ break
+ } catch (error) {
+ core.debug(`Attempt ${attempt + 1} failed: ${error}`)
+ attempt++
+ if (attempt >= 30) {
+ throw new Error(
+ `execCpFromPod failed after ${attempt} attempts: ${formatError(error)}`
+ )
+ }
+ await sleep(1000)
+ }
+ }
+
+ let attempts = 15
+ const delay = 1000
+ for (let i = 0; i < attempts; i++) {
+ try {
+ const { hash: want, lines: wantLines } =
+ await execCalculateOutputHashSorted(podName, JOB_CONTAINER_NAME, [
+ 'sh',
+ '-c',
+ listDirAllCommand(containerPath)
+ ])
+
+ const { hash: got, lines: gotLines } =
+ await localCalculateOutputHashSorted([
+ 'sh',
+ '-c',
+ listDirAllCommand(targetRunnerPath)
+ ])
+
+ if (got !== want) {
+ core.debug(
+ `[cpFromPod hash mismatch attempt ${i + 1}/${attempts}] want='${want}' got='${got}'`
+ )
+ core.debug(
+ `[cpFromPod] pod file count=${wantLines.length} runner file count=${gotLines.length}`
+ )
+ const wantMap = new Map(
+ wantLines.map(l => [l.replace(/^\d+ /, ''), l.split(' ')[0]])
+ )
+ const gotMap = new Map(
+ gotLines.map(l => [l.replace(/^\d+ /, ''), l.split(' ')[0]])
+ )
+ const onlyInWant = wantLines.filter(
+ l => !gotMap.has(l.replace(/^\d+ /, ''))
+ )
+ const onlyInGot = gotLines.filter(
+ l => !wantMap.has(l.replace(/^\d+ /, ''))
+ )
+ const sizeDiff = wantLines
+ .filter(l => {
+ const name = l.replace(/^\d+ /, '')
+ return gotMap.has(name) && gotMap.get(name) !== wantMap.get(name)
+ })
+ .map(l => {
+ const name = l.replace(/^\d+ /, '')
+ return `${name}: pod=${wantMap.get(name)} runner=${gotMap.get(name)}`
+ })
+ core.debug(`[cpFromPod] only in pod: ${JSON.stringify(onlyInWant)}`)
+ core.debug(`[cpFromPod] only in runner: ${JSON.stringify(onlyInGot)}`)
+ core.debug(`[cpFromPod] size mismatch: ${JSON.stringify(sizeDiff)}`)
+ await sleep(delay)
+ continue
+ }
+
+ break
+ } catch (error) {
+ core.debug(`Attempt ${i + 1} failed: ${error}`)
+ await sleep(delay)
+ }
+ }
+}
+
+export async function waitForJobToComplete(jobName: string): Promise<void> {
+ const backOffManager = new BackOffManager()
+ while (true) {
+ try {
+ if (await isJobSucceeded(jobName)) {
+ return
+ }
+ } catch (error) {
+ throw new Error(`job ${jobName} has failed: ${formatError(error)}`)
+ }
+ await backOffManager.backOff()
+ }
+}
+
+export async function createDockerSecret(
+ registry: Registry
+): Promise<k8s.V1Secret> {
+ const authContent = {
+ auths: {
+ [registry.serverUrl || 'https://index.docker.io/v1/']: {
+ username: registry.username,
+ password: registry.password,
+ auth: Buffer.from(`${registry.username}:${registry.password}`).toString(
+ 'base64'
+ )
+ }
+ }
+ }
+
+ const runnerInstanceLabel = new RunnerInstanceLabel()
+
+ const secretName = getSecretName()
+ const secret = new k8s.V1Secret()
+ secret.immutable = true
+ secret.apiVersion = 'v1'
+ secret.metadata = new k8s.V1ObjectMeta()
+ secret.metadata.name = secretName
+ secret.metadata.namespace = namespace()
+ secret.metadata.labels = {
+ [runnerInstanceLabel.key]: runnerInstanceLabel.value
+ }
+ secret.type = 'kubernetes.io/dockerconfigjson'
+ secret.kind = 'Secret'
+ secret.data = {
+ '.dockerconfigjson': Buffer.from(JSON.stringify(authContent)).toString(
+ 'base64'
+ )
+ }
+
+ return await k8sApi.createNamespacedSecret({
+ namespace: namespace(),
+ body: secret
+ })
+}
+
+export async function createSecretForEnvs(envs: {
+ [key: string]: string
+}): Promise<string> {
+ const runnerInstanceLabel = new RunnerInstanceLabel()
+
+ const secret = new k8s.V1Secret()
+ const secretName = getSecretName()
+ secret.immutable = true
+ secret.apiVersion = 'v1'
+ secret.metadata = new k8s.V1ObjectMeta()
+ secret.metadata.name = secretName
+
+ secret.metadata.labels = {
+ [runnerInstanceLabel.key]: runnerInstanceLabel.value
+ }
+ secret.kind = 'Secret'
+ secret.data = {}
+ for (const [key, value] of Object.entries(envs)) {
+ secret.data[key] = Buffer.from(value).toString('base64')
+ }
+
+ await k8sApi.createNamespacedSecret({
+ namespace: namespace(),
+ body: secret
+ })
+ return secretName
+}
+
+export async function deleteSecret(name: string): Promise<void> {
+ await k8sApi.deleteNamespacedSecret({
+ name,
+ namespace: namespace()
+ })
+}
+
+export async function pruneSecrets(): Promise<void> {
+ const secretList = await k8sApi.listNamespacedSecret({
+ namespace: namespace(),
+ labelSelector: new RunnerInstanceLabel().toString()
+ })
+ if (!secretList.items.length) {
+ return
+ }
+
+ await Promise.all(
+ secretList.items.map(
+ async secret =>
+ secret.metadata?.name && (await deleteSecret(secret.metadata.name))
+ )
+ )
+}
+
+export const UNRECOVERABLE_WAITING_REASONS = new Set([
+ // k8s has already retried image pull multiple times with exponential backoff.
+ // ErrImagePull is excluded: it fires on the first failure (could be a transient
+ // TLS timeout or network blip) and k8s will naturally promote it to
+ // ImagePullBackOff after a moment. Fast-failing on ErrImagePull would kill
+ // jobs that would have succeeded on the next pull attempt.
+ 'ImagePullBackOff',
+ // Image name is syntactically invalid — cannot self-heal without a config fix.
+ 'InvalidImageName',
+ // Container spec is invalid (bad env vars, resource limits, securityContext) —
+ // cannot self-heal without a config fix.
+ 'CreateContainerConfigError'
+ // CreateContainerError is excluded: it is sometimes emitted transiently by the
+ // container runtime (e.g. during a node-level runtime restart). It can
+ // self-resolve on the next kubelet retry cycle.
+])
+
+// Pod *event* reasons (from the event stream, not container status) that
+// indicate a permanent failure the pod will never recover from on its own.
+// These surface as Warning events on the pod (visible via `kubectl describe`)
+// rather than in container.status.state.waiting.reason, so they need a separate
+// list + a separate (async) detection path.
+//
+// - FailedMount: a volume cannot be mounted (e.g. hostPath `type: Directory`
+// pointing at a path that does not exist on the node, missing PVC, missing
+// Secret/ConfigMap volume). The container stays in `ContainerCreating`
+// waiting state, which is NOT in UNRECOVERABLE_WAITING_REASONS, so without
+// this check the hook polls until the 3600s timeout.
+// - FailedBinding: a PVC could not be bound (no matching PV, storage class
+// misconfiguration). Usually paired with FailedMount once the pod retries.
+//
+// FailedScheduling is included but gated by PERMANENT_SCHEDULING_PATTERNS:
+// only messages that positively identify a permanent configuration error
+// trigger fast-fail. Resource shortages ("Insufficient cpu/memory/gpu")
+// are silently skipped so the pod keeps queuing until the timeout.
+export const UNRECOVERABLE_EVENT_REASONS = new Set([
+ 'FailedMount',
+ 'FailedBinding',
+ 'FailedScheduling'
+])
+
+// Patterns that POSITIVELY IDENTIFY a permanent, unrecoverable scheduling
+// failure in a FailedScheduling event message. Fast-fail fires ONLY when
+// one of these matches. Anything else (resource shortages, unknown format,
+// absent message) is treated as transient — the pod keeps queuing.
+//
+// Permanent examples (WILL fast-fail):
+// "0/3 nodes: 3 node(s) didn't match Pod's node affinity/selector."
+// "0/3 nodes: 3 node(s) had untolerated taint {gpu: true}."
+// "0/3 nodes: persistentvolumeclaim "my-pvc" not found."
+//
+// Transient examples (will NOT fast-fail, keep queuing):
+// "0/3 nodes: 3 Insufficient nvidia.com/gpu."
+// "0/3 nodes: 3 Insufficient memory."
+// "0/1 nodes are available" ← ambiguous, unknown
+//
+// Extend at runtime via env var ACTIONS_RUNNER_K8S_PERMANENT_SCHEDULING_PATTERNS
+// (comma-separated regex strings; cannot remove built-in entries).
+export const PERMANENT_SCHEDULING_PATTERNS: readonly RegExp[] = [
+ // Node selector / affinity label mismatch — no node carries the required
+ // labels. Will not self-resolve without a pod-spec or node-label change.
+ /node\(s\) didn't match Pod's node affinity\/selector/i,
+ /node\(s\) didn't match node affinity/i,
+ // Untolerated taint — every node carries a taint the pod does not tolerate.
+ // Will not self-resolve without adding a toleration or removing the taint.
+ /node\(s\) had untolerated taint/i,
+ // PVC referenced in the pod spec does not exist in the namespace. The
+ // scheduler refuses to place the pod until the PVC is created. The workflow
+ // job spec is wrong — it won't self-resolve.
+ /persistentvolumeclaim "[^"]+" not found/i
+]
+
+export const UNRECOVERABLE_TERMINATED_REASONS = new Set([
+ 'OOMKilled',
+ 'Error',
+ 'FailedPostStartHookError'
+])
+
+// Maximum number of Warning events to include in a pod's failure description so
+// that the GitHub Actions log is not flooded.
+const MAX_DIAGNOSTIC_EVENTS = 10
+
+// The fast-fail whitelist can be extended (not narrowed) from the environment so
+// operators can add deterministic terminal reasons without a code change. This
+// reuses the same env-var configuration pattern as the prepare-job timeout. When
+// the variable is unset, behaviour is identical to the built-in defaults.
+export function getUnrecoverableWaitingReasons(): Set<string> {
+ const extra = process.env['ACTIONS_RUNNER_K8S_UNRECOVERABLE_WAITING_REASONS']
+ if (!extra) {
+ return UNRECOVERABLE_WAITING_REASONS
+ }
+ const reasons = new Set(UNRECOVERABLE_WAITING_REASONS)
+ for (const reason of extra.split(',')) {
+ const trimmed = reason.trim()
+ Eif (trimmed) {
+ reasons.add(trimmed)
+ }
+ }
+ return reasons
+}
+
+// Mirrors getUnrecoverableWaitingReasons() but for pod *event* reasons. The
+// env var ACTIONS_RUNNER_K8S_UNRECOVERABLE_EVENT_REASONS adds to (never removes
+// from) the built-in UNRECOVERABLE_EVENT_REASONS set.
+export function getUnrecoverableEventReasons(): Set<string> {
+ const extra = process.env['ACTIONS_RUNNER_K8S_UNRECOVERABLE_EVENT_REASONS']
+ if (!extra) {
+ return UNRECOVERABLE_EVENT_REASONS
+ }
+ const reasons = new Set(UNRECOVERABLE_EVENT_REASONS)
+ for (const reason of extra.split(',')) {
+ const trimmed = reason.trim()
+ Eif (trimmed) {
+ reasons.add(trimmed)
+ }
+ }
+ return reasons
+}
+
+export function getUnrecoverableTerminatedReasons(): Set<string> {
+ const extra =
+ process.env['ACTIONS_RUNNER_K8S_UNRECOVERABLE_TERMINATED_REASONS']
+ if (!extra) {
+ return UNRECOVERABLE_TERMINATED_REASONS
+ }
+ const reasons = new Set(UNRECOVERABLE_TERMINATED_REASONS)
+ for (const reason of extra.split(',')) {
+ const trimmed = reason.trim()
+ if (trimmed) {
+ reasons.add(trimmed)
+ }
+ }
+ return reasons
+}
+
+// Returns true when a FailedScheduling event message positively identifies a
+// permanent configuration error that will not self-resolve. Returns false
+// (treat as transient, keep queuing) for:
+// - absent/empty messages → unknown, assume transient
+// - resource shortages → Insufficient cpu/memory/gpu/etc.
+// - any unrecognised format → unknown, assume transient
+export function isPermanentSchedulingFailure(
+ message: string | undefined
+): boolean {
+ if (!message) return false
+ return getPermanentSchedulingPatterns().some(p => p.test(message))
+}
+
+// Returns PERMANENT_SCHEDULING_PATTERNS extended by any extra patterns from
+// ACTIONS_RUNNER_K8S_PERMANENT_SCHEDULING_PATTERNS (comma-separated regexes).
+// Invalid regex strings are skipped with a warning.
+// The compiled result is cached: env var changes during a single process are
+// rare (effectively never in a runner job), but this avoids repeated split +
+// RegExp construction on every FailedScheduling poll iteration.
+let _cachedSchedulingPatterns: readonly RegExp[] | undefined
+let _lastSchedulingPatternsEnv: string | undefined
+
+export function getPermanentSchedulingPatterns(): readonly RegExp[] {
+ const extra = process.env['ACTIONS_RUNNER_K8S_PERMANENT_SCHEDULING_PATTERNS']
+ if (!extra) return PERMANENT_SCHEDULING_PATTERNS
+ if (_cachedSchedulingPatterns && _lastSchedulingPatternsEnv === extra) {
+ return _cachedSchedulingPatterns
+ }
+ const patterns: RegExp[] = [...PERMANENT_SCHEDULING_PATTERNS]
+ for (const raw of extra.split(',')) {
+ const trimmed = raw.trim()
+ if (!trimmed) continue
+ try {
+ patterns.push(new RegExp(trimmed, 'i'))
+ } catch {
+ core.warning(
+ `ACTIONS_RUNNER_K8S_PERMANENT_SCHEDULING_PATTERNS: invalid regex "${trimmed}", skipped`
+ )
+ }
+ }
+ _lastSchedulingPatternsEnv = extra
+ _cachedSchedulingPatterns = patterns
+ return patterns
+}
+
+function getWaitingReasonHint(reason: string): string {
+ switch (reason) {
+ case 'ImagePullBackOff':
+ return [
+ ` → Image pull has failed repeatedly (k8s retried with exponential backoff). Check:`,
+ ` - Image name and tag are correct and exist in the registry`,
+ ` - If private registry: imagePullSecret is configured and credentials are valid`,
+ ` - Network connectivity from the node to the registry (DNS, firewall, proxy, TLS)`,
+ ` Run: kubectl describe pod <pod> | grep -A10 "Events"`
+ ].join('\n')
+ case 'InvalidImageName':
+ return ` → Image name is malformed. Check the workflow/job container image configuration.`
+ case 'CreateContainerConfigError':
+ return ` → Container config is invalid. Check env vars, resource limits, and securityContext in the job spec.`
+ default:
+ return ` → Check pod events with: kubectl describe pod <pod>`
+ }
+}
+
+export function getContainerErrors(pod: k8s.V1Pod): string[] {
+ const errors: string[] = []
+ const unrecoverableReasons = getUnrecoverableWaitingReasons()
+ const allStatuses = [
+ ...(pod.status?.initContainerStatuses ?? []),
+ ...(pod.status?.containerStatuses ?? [])
+ ]
+ for (const cs of allStatuses) {
+ const waiting = cs.state?.waiting
+ if (waiting?.reason && unrecoverableReasons.has(waiting.reason)) {
+ const reason = ` ✗ container "${cs.name}": ${waiting.reason}`
+ const detail = waiting.message ? `\n ${waiting.message}` : ''
+ const hint = `\n${getWaitingReasonHint(waiting.reason)}`
+ errors.push(`${reason}${detail}${hint}`)
+ }
+ }
+ return errors
+}
+
+export function getTerminatedReasonHint(
+ reason: string,
+ exitCode: number | undefined
+): string {
+ if (reason === 'OOMKilled') {
+ return ` → Container exceeded its memory limit and was killed by the OOM killer.\n Increase the memory limit in the job spec or reduce memory usage in the script.`
+ }
+ if (reason === 'FailedPostStartHookError') {
+ return ` → The postStart lifecycle hook failed. Check the hook command and its exit code.`
+ }
+ if (reason === 'Error') {
+ switch (exitCode) {
+ case 137:
+ return ` → Exit code 137: process was killed (SIGKILL). Likely OOM or forceful termination.\n Check memory usage and resource limits.`
+ case 139:
+ return ` → Exit code 139: segmentation fault (SIGSEGV). The process crashed due to a memory access error.`
+ case 126:
+ return ` → Exit code 126: permission denied. The script or binary is not executable.\n Check file permissions inside the container image.`
+ case 127:
+ return ` → Exit code 127: command not found. The script or binary does not exist in the container.\n Check the image contents and the command/entrypoint configuration.`
+ default:
+ return ` → Script or process exited with a non-zero code (${exitCode ?? 'unknown'}).\n Check the step output above for error messages.\n Common causes: script logic errors, missing dependencies, unhandled exceptions.`
+ }
+ }
+ return ` → Check pod logs with: kubectl logs <pod> -c ${reason}`
+}
+
+export function getContainerTerminatedErrors(pod: k8s.V1Pod): string[] {
+ const errors: string[] = []
+ const unrecoverableReasons = getUnrecoverableTerminatedReasons()
+ const allStatuses = [
+ ...(pod.status?.initContainerStatuses ?? []),
+ ...(pod.status?.containerStatuses ?? [])
+ ]
+ for (const cs of allStatuses) {
+ const terminated = cs.state?.terminated
+ if (terminated?.reason && unrecoverableReasons.has(terminated.reason)) {
+ const reason = ` ✗ container "${cs.name}": ${terminated.reason} (exit code ${terminated.exitCode})`
+ const detail = terminated.message ? `\n ${terminated.message}` : ''
+ const hint = `\n${getTerminatedReasonHint(terminated.reason, terminated.exitCode)}`
+ errors.push(`${reason}${detail}${hint}`)
+ }
+ }
+ return errors
+}
+
+function getEventReasonHint(reason: string): string {
+ switch (reason) {
+ case 'FailedMount':
+ return [
+ ` → A volume could not be mounted. Check:`,
+ ` - PVC is bound (kubectl get pvc)`,
+ ` - Secret/ConfigMap referenced in the volume exists`,
+ ` - hostPath directories exist on the scheduled node`
+ ].join('\n')
+ case 'FailedBinding':
+ return [
+ ` → A PVC could not be bound to a PV. Check:`,
+ ` - StorageClass exists and has a provisioner`,
+ ` - Sufficient capacity is available`,
+ ` - Access mode (ReadWriteOnce/ReadWriteMany) matches available PVs`
+ ].join('\n')
+ case 'FailedScheduling':
+ return [
+ ` → Pod cannot be scheduled due to a permanent configuration error. Check:`,
+ ` - nodeSelector / nodeAffinity labels match at least one node`,
+ ` - All tolerations are present for node taints`,
+ ` - PVCs referenced in the pod spec exist in the namespace`
+ ].join('\n')
+ default:
+ return ` → Check pod events with: kubectl describe pod <pod>`
+ }
+}
+
+// Inspects the pod's Warning events for reasons in UNRECOVERABLE_EVENT_REASONS.
+// For FailedScheduling, only fast-fails when the message positively matches
+// a known-permanent configuration error (see PERMANENT_SCHEDULING_PATTERNS).
+// Resource shortages ("Insufficient cpu/memory/gpu") are silently skipped so
+// the pod keeps queuing until the timeout. Best-effort: if events cannot be
+// listed (e.g. the optional 'events' RBAC permission is missing) it returns []
+// instead of blocking container-level fast-fail detection. Deduplicates by
+// reason so a repeatedly-retried event is reported once.
+export async function getPodEventErrors(podName: string): Promise<string[]> {
+ const unrecoverableReasons = getUnrecoverableEventReasons()
+ Iif (unrecoverableReasons.size === 0) {
+ return []
+ }
+
+ let items: k8s.CoreV1Event[]
+ try {
+ const result = await k8sApi.listNamespacedEvent({
+ namespace: namespace(),
+ fieldSelector: `involvedObject.name=${podName}`
+ })
+ items = result.items ?? []
+ } catch (err) {
+ core.debug(
+ `Could not list events for pod ${podName} during fast-fail check: ${
+ err instanceof Error ? err.message : String(err)
+ }`
+ )
+ return []
+ }
+
+ const errors: string[] = []
+ const seenReasons = new Set<string>()
+ for (const e of items) {
+ if (
+ e.type !== 'Warning' ||
+ !e.reason ||
+ !unrecoverableReasons.has(e.reason)
+ ) {
+ continue
+ }
+ // FailedScheduling: only fast-fail when the message positively matches a
+ // known-permanent config error. Resource shortages and unknown messages
+ // are treated as transient — let the pod keep queuing.
+ if (
+ e.reason === 'FailedScheduling' &&
+ !isPermanentSchedulingFailure(e.message)
+ ) {
+ core.debug(
+ `[fast-fail] Skipping FailedScheduling (not a recognised permanent error): ${
+ e.message ?? '(no message)'
+ }`
+ )
+ continue
+ }
+ if (seenReasons.has(e.reason)) {
+ continue
+ }
+ seenReasons.add(e.reason)
+ const count = e.count && e.count > 1 ? ` (x${e.count})` : ''
+ const reason = ` ✗ event: ${e.reason}${count}`
+ const detail = e.message ? `\n ${e.message}` : ''
+ const hint = getEventReasonHint(e.reason)
+ errors.push(`${reason}${detail}\n${hint}`)
+ }
+ return errors
+}
+
+// describePodFailure aggregates everything that might explain why a pod is not
+// healthy into a single human-readable string. It makes NO success/failure
+// judgement of its own (a terminated exitCode=0 container is just reported as
+// such) -- callers decide what is "bad". It never throws: if it cannot read the
+// pod or list events it returns/embeds a best-effort note instead, so it is safe
+// to call from any error path.
+export async function describePodFailure(podName: string): Promise<string> {
+ let pod: k8s.V1Pod
+ try {
+ pod = await readPod(podName)
+ } catch (err) {
+ return `Could not read pod ${podName} for diagnostics: ${
+ err instanceof Error ? err.message : String(err)
+ }`
+ }
+
+ const sections: string[] = []
+ const status = pod.status
+
+ // --- Pod-level status ---
+ const podLines: string[] = []
+ const phaseLabel = status?.phase
+ ? `${status.phase}${status.reason ? ` (${status.reason})` : ''}`
+ : 'Unknown'
+ podLines.push(`Pod status: ${phaseLabel}`)
+ Iif (status?.message) {
+ podLines.push(` ${status.message}`)
+ }
+
+ // Surface conditions that are False (e.g. PodScheduled=False from FailedScheduling)
+ for (const cond of status?.conditions ?? []) {
+ if (cond.status === 'False') {
+ const reason = cond.reason ? ` (${cond.reason})` : ''
+ const msg = cond.message ? `: ${cond.message}` : ''
+ podLines.push(` ✗ ${cond.type}=False${reason}${msg}`)
+ }
+ }
+ sections.push(podLines.join('\n'))
+
+ // --- Container-level status (non-zero exits and non-unrecoverable waits) ---
+ const unrecoverableReasons = getUnrecoverableWaitingReasons()
+ const allStatuses = [
+ ...(status?.initContainerStatuses ?? []),
+ ...(status?.containerStatuses ?? [])
+ ]
+ const containerLines: string[] = []
+ for (const cs of allStatuses) {
+ const waiting = cs.state?.waiting
+ if (waiting?.reason) {
+ // Skip reasons already surfaced by getContainerErrors() in the caller's
+ // first line to avoid printing the same error twice.
+ Iif (!unrecoverableReasons.has(waiting.reason)) {
+ const msg = waiting.message ? `\n ${waiting.message}` : ''
+ containerLines.push(
+ ` ✗ container "${cs.name}" waiting: ${waiting.reason}${msg}`
+ )
+ }
+ }
+ const terminated = cs.state?.terminated
+ // Only surface non-zero exits; exit 0 (e.g. fs-init Completed) is noise.
+ if (terminated && terminated.exitCode !== 0) {
+ const reason = terminated.reason ?? 'Unknown'
+ const msg = terminated.message ? `\n ${terminated.message}` : ''
+ containerLines.push(
+ ` ✗ container "${cs.name}" terminated: ${reason} (exit code ${terminated.exitCode})${msg}`
+ )
+ }
+ }
+ if (containerLines.length) {
+ sections.push(`Container details:\n${containerLines.join('\n')}`)
+ }
+
+ // --- Warning events ---
+ const eventLines = await describePodWarningEvents(podName)
+ if (eventLines.length) {
+ sections.push(`Recent warning events:\n${eventLines.join('\n')}`)
+ }
+
+ Iif (!sections.length) {
+ return `No additional diagnostic information available for pod ${podName}`
+ }
+ return sections.join('\n\n')
+}
+
+// Reads the most recent Warning events for a pod. Best-effort: listing events
+// requires the optional "events" get/list permission, so any error (including
+// RBAC Forbidden) is swallowed and an empty list is returned.
+async function describePodWarningEvents(podName: string): Promise<string[]> {
+ let items: k8s.CoreV1Event[]
+ try {
+ const result = await k8sApi.listNamespacedEvent({
+ namespace: namespace(),
+ fieldSelector: `involvedObject.name=${podName}`
+ })
+ items = result.items ?? []
+ } catch (err) {
+ core.debug(
+ `Could not list events for pod ${podName} (the 'events' permission may be missing): ${
+ err instanceof Error ? err.message : String(err)
+ }`
+ )
+ return []
+ }
+
+ const warnings = items.filter(e => e.type === 'Warning')
+ if (!warnings.length) {
+ return []
+ }
+
+ const eventTime = (e: k8s.CoreV1Event): number => {
+ const t = e.lastTimestamp ?? e.eventTime ?? e.firstTimestamp
+ return t ? new Date(t).getTime() : 0
+ }
+ warnings.sort((a, b) => eventTime(a) - eventTime(b))
+
+ const recent = warnings.slice(-MAX_DIAGNOSTIC_EVENTS)
+ const lines = recent.map(e => {
+ const count = e.count && e.count > 1 ? ` (x${e.count})` : ''
+ return ` [${e.reason ?? 'Warning'}]${count} ${e.message ?? ''}`.trimEnd()
+ })
+ Iif (warnings.length > recent.length) {
+ lines.unshift(
+ ` (showing ${recent.length} of ${warnings.length} warning events)`
+ )
+ }
+ return lines
+}
+
+// Inspects the pod's status.conditions for scheduling failures. FailedScheduling
+// is intentionally excluded from fast-fail detection (see UNRECOVERABLE_EVENT_REASONS
+// comment) — scheduling failures are transient resource waits that should queue
+// until the timeout, not terminate early. This function always returns [] as a
+// result, but is kept so checkUnrecoverableErrors compiles and the dedup logic
+// remains intact for future use.
+export function getPodConditionErrors(_p: k8s.V1Pod): string[] {
+ return []
+}
+
+// Aggregates the three independent error-detection sources (container waiting
+// reasons, pod Warning events, and pod conditions) into a single list, applying
+// the cross-source deduplication rule (FailedScheduling event + Unschedulable
+// condition = same failure, report once). Returns [] when no unrecoverable
+// error is detected. The caller decides what to do with the list -- typically
+// attach describePodFailure diagnostics and throw.
+export async function checkUnrecoverableErrors(
+ pod: k8s.V1Pod,
+ podName: string
+): Promise<string[]> {
+ // Deterministic terminal errors on a container (e.g. ImagePullBackOff).
+ const containerErrors = getContainerErrors(pod)
+ // Terminated container errors (e.g. OOMKilled, non-zero exit) with hints.
+ const terminatedErrors = getContainerTerminatedErrors(pod)
+ // Best-effort: returns [] when the optional events RBAC permission is absent.
+ const eventErrors = await getPodEventErrors(podName)
+ // Conditions are set near-instantly by the scheduler while events may take
+ // a few extra seconds to propagate, so always run the check. Deduplicate
+ // against events so the same FailedScheduling isn't printed twice.
+ const conditionErrors = getPodConditionErrors(pod).filter(
+ c =>
+ !eventErrors.some(
+ e => e.includes('FailedScheduling') && c.includes('Unschedulable')
+ )
+ )
+ return [
+ ...containerErrors,
+ ...terminatedErrors,
+ ...eventErrors,
+ ...conditionErrors
+ ]
+}
+
+export async function waitForPodPhases(
+ podName: string,
+ awaitingPhases: Set<PodPhase>,
+ backOffPhases: Set<PodPhase>,
+ maxTimeSeconds = DEFAULT_WAIT_FOR_POD_TIME_SECONDS
+): Promise<void> {
+ const backOffManager = new BackOffManager(maxTimeSeconds)
+ let phase: PodPhase = PodPhase.UNKNOWN
+ while (true) {
+ let pod: k8s.V1Pod
+ try {
+ pod = await readPod(podName)
+ } catch (err) {
+ // Transient API error (network blip, API server busy): log and back off
+ // rather than crashing the loop. The timeout will eventually fire if the
+ // pod stays permanently unreadable (e.g. RBAC issue).
+ core.warning(
+ `[waitForPodPhases] Could not read pod ${podName}, will retry after backoff: ${
+ err instanceof Error ? err.message : String(err)
+ }`
+ )
+ try {
+ await backOffManager.backOff()
+ } catch {
+ throw new Error(
+ `Pod ${podName} timed out after ${maxTimeSeconds}s (pod read failed: ${
+ err instanceof Error ? err.message : String(err)
+ })\n${'-'.repeat(60)}\n(pod was unreadable; no further diagnostics available)`
+ )
+ }
+ continue
+ }
+ phase = parsePodPhase(pod)
+ if (awaitingPhases.has(phase)) {
+ return
+ }
+
+ // The pod reached a phase we are not willing to keep waiting on
+ // (a terminal/unhealthy phase). First check for unrecoverable
+ // container-level errors (e.g. OOMKilled, non-zero exit) so they are
+ // reported with actionable hints. Fall back to the generic "is unhealthy"
+ // message when no specific error is found.
+ if (!backOffPhases.has(phase)) {
+ const errors = await checkUnrecoverableErrors(pod, podName)
+ const details = await describePodFailure(podName)
+ Iif (errors.length > 0) {
+ throw new Error(
+ `Pod ${podName} has unrecoverable errors:\n${errors.join('\n')}\n${'-'.repeat(60)}\n${details}`
+ )
+ }
+ throw new Error(
+ `Pod ${podName} is unhealthy (phase: ${phase})\n${'-'.repeat(60)}\n${details}`
+ )
+ }
+
+ // Still in a back-off phase, but a deterministic unrecoverable error
+ // was detected (container / event / condition). Fail fast with
+ // diagnostics instead of waiting out the full timeout.
+ const errors = await checkUnrecoverableErrors(pod, podName)
+ Eif (errors.length > 0) {
+ const details = await describePodFailure(podName)
+ throw new Error(
+ `Pod ${podName} has unrecoverable errors:\n${errors.join('\n')}\n${'-'.repeat(60)}\n${details}`
+ )
+ }
+
+ try {
+ await backOffManager.backOff()
+ } catch (_err) {
+ // BackOffManager throws "backoff timeout" when maxTimeSeconds is exceeded.
+ // Don't surface that bare message: collect diagnostics first so the user
+ // can see WHY the pod never became ready.
+ const details = await describePodFailure(podName)
+ throw new Error(
+ `Pod ${podName} timed out after ${maxTimeSeconds}s (phase: ${phase})\n${'-'.repeat(60)}\n${details}`
+ )
+ }
+ }
+}
+
+export function getPrepareJobTimeoutSeconds(): number {
+ const envTimeoutSeconds =
+ process.env['ACTIONS_RUNNER_PREPARE_JOB_TIMEOUT_SECONDS']
+
+ if (!envTimeoutSeconds) {
+ return DEFAULT_WAIT_FOR_POD_TIME_SECONDS
+ }
+
+ const timeoutSeconds = parseInt(envTimeoutSeconds, 10)
+ if (!timeoutSeconds || timeoutSeconds <= 0) {
+ core.warning(
+ `Prepare job timeout is invalid ("${timeoutSeconds}"): use an int > 0`
+ )
+ return DEFAULT_WAIT_FOR_POD_TIME_SECONDS
+ }
+
+ return timeoutSeconds
+}
+
+async function readPod(podName: string): Promise<k8s.V1Pod> {
+ return await k8sApi.readNamespacedPod({
+ name: podName,
+ namespace: namespace()
+ })
+}
+
+const podPhaseLookup = new Set<string>([
+ PodPhase.PENDING,
+ PodPhase.RUNNING,
+ PodPhase.SUCCEEDED,
+ PodPhase.FAILED,
+ PodPhase.UNKNOWN
+])
+
+export function parsePodPhase(pod: k8s.V1Pod): PodPhase {
+ if (!pod.status?.phase || !podPhaseLookup.has(pod.status.phase)) {
+ return PodPhase.UNKNOWN
+ }
+ return pod.status.phase as PodPhase
+}
+
+async function isJobSucceeded(name: string): Promise<boolean> {
+ const job = await k8sBatchV1Api.readNamespacedJob({
+ name,
+ namespace: namespace()
+ })
+ if (job.status?.failed) {
+ throw new Error(`job ${name} has failed`)
+ }
+ return !!job.status?.succeeded
+}
+
+export async function getPodLogs(
+ podName: string,
+ containerName: string
+): Promise<void> {
+ const log = new k8s.Log(kc)
+ const logStream = new stream.PassThrough()
+ logStream.on('data', chunk => {
+ // use write rather than console.log to prevent double line feed
+ process.stdout.write(chunk)
+ })
+
+ await log.log(namespace(), podName, containerName, logStream, {
+ follow: true,
+ pretty: false,
+ timestamps: false
+ })
+ await new Promise((resolve, reject) => {
+ logStream.on('end', () => resolve(null))
+ logStream.on('error', err => {
+ process.stderr.write(err.message)
+ reject(err)
+ })
+ })
+}
+
+export async function prunePods(): Promise<void> {
+ const podList = await k8sApi.listNamespacedPod({
+ namespace: namespace(),
+ labelSelector: new RunnerInstanceLabel().toString()
+ })
+ if (!podList.items.length) {
+ return
+ }
+
+ await Promise.all(
+ podList.items.map(
+ async pod => pod.metadata?.name && (await deletePod(pod.metadata.name))
+ )
+ )
+}
+
+export async function getPodStatus(
+ name: string
+): Promise<k8s.V1PodStatus | undefined> {
+ const pod = await k8sApi.readNamespacedPod({
+ name,
+ namespace: namespace()
+ })
+ return pod.status
+}
+
+export async function isAuthPermissionsOK(): Promise<boolean> {
+ const sar = new k8s.V1SelfSubjectAccessReview()
+ const asyncs: Promise<k8s.V1SelfSubjectAccessReview>[] = []
+ for (const resource of requiredPermissions) {
+ for (const verb of resource.verbs) {
+ sar.spec = new k8s.V1SelfSubjectAccessReviewSpec()
+ sar.spec.resourceAttributes = new k8s.V1ResourceAttributes()
+ sar.spec.resourceAttributes.verb = verb
+ sar.spec.resourceAttributes.namespace = namespace()
+ sar.spec.resourceAttributes.group = resource.group
+ sar.spec.resourceAttributes.resource = resource.resource
+ sar.spec.resourceAttributes.subresource = resource.subresource
+ asyncs.push(
+ k8sAuthorizationV1Api.createSelfSubjectAccessReview({ body: sar })
+ )
+ }
+ }
+ const responses = await Promise.all(asyncs)
+ return responses.every(resp => resp.status?.allowed)
+}
+
+export async function isPodContainerAlpine(
+ podName: string,
+ containerName: string
+): Promise<boolean> {
+ let isAlpine = true
+ try {
+ await execPodStep(
+ [
+ 'sh',
+ '-c',
+ `[ $(cat /etc/*release* | grep -i -e "^ID=*alpine*" -c) != 0 ] || exit 1`
+ ],
+ podName,
+ containerName
+ )
+ } catch {
+ isAlpine = false
+ }
+
+ return isAlpine
+}
+
+export function namespace(): string {
+ if (process.env['ACTIONS_RUNNER_KUBERNETES_NAMESPACE']) {
+ return process.env['ACTIONS_RUNNER_KUBERNETES_NAMESPACE']
+ }
+
+ const context = kc.getContexts().find(ctx => ctx.namespace)
+ Eif (context?.namespace) {
+ return context.namespace
+ }
+
+ // When running in-cluster the kubeconfig context has no namespace field;
+ // read it from the mounted ServiceAccount file instead.
+ const saNamespaceFile =
+ '/var/run/secrets/kubernetes.io/serviceaccount/namespace'
+ try {
+ const ns = fs.readFileSync(saNamespaceFile, 'utf8').trim()
+ if (ns) {
+ return ns
+ }
+ } catch {
+ // not running in-cluster, fall through to error
+ }
+
+ throw new Error(
+ 'Failed to determine namespace. Set the ACTIONS_RUNNER_KUBERNETES_NAMESPACE environment variable or ensure the kubeconfig context includes a namespace.'
+ )
+}
+
+class BackOffManager {
+ private backOffSeconds = 1
+ totalTime = 0
+ constructor(private throwAfterSeconds?: number) {
+ if (!throwAfterSeconds || throwAfterSeconds < 0) {
+ this.throwAfterSeconds = undefined
+ }
+ }
+
+ async backOff(): Promise<void> {
+ await new Promise(resolve =>
+ setTimeout(resolve, this.backOffSeconds * 1000)
+ )
+ this.totalTime += this.backOffSeconds
+ Iif (this.throwAfterSeconds && this.throwAfterSeconds < this.totalTime) {
+ throw new Error('backoff timeout')
+ }
+ Eif (this.backOffSeconds < 20) {
+ this.backOffSeconds *= 2
+ }
+ Iif (this.backOffSeconds > 20) {
+ this.backOffSeconds = 20
+ }
+ }
+}
+
+export function containerPorts(
+ container: ContainerInfo
+): k8s.V1ContainerPort[] {
+ const ports: k8s.V1ContainerPort[] = []
+ if (!container.portMappings?.length) {
+ return ports
+ }
+ for (const portDefinition of container.portMappings) {
+ const portProtoSplit = portDefinition.split('/')
+ if (portProtoSplit.length > 2) {
+ throw new Error(`Unexpected port format: ${portDefinition}`)
+ }
+
+ const port = new k8s.V1ContainerPort()
+ port.protocol =
+ portProtoSplit.length === 2 ? portProtoSplit[1].toUpperCase() : 'TCP'
+
+ const portSplit = portProtoSplit[0].split(':')
+ if (portSplit.length > 2) {
+ throw new Error('ports should have at most one ":" separator')
+ }
+
+ const parsePort = (p: string): number => {
+ const num = Number(p)
+ if (!Number.isInteger(num) || num < 1 || num > 65535) {
+ throw new Error(`invalid container port: ${p}`)
+ }
+ return num
+ }
+
+ if (portSplit.length === 1) {
+ port.containerPort = parsePort(portSplit[0])
+ } else {
+ port.hostPort = parsePort(portSplit[0])
+ port.containerPort = parsePort(portSplit[1])
+ }
+
+ ports.push(port)
+ }
+ return ports
+}
+
+export async function getPodByName(name): Promise<k8s.V1Pod> {
+ return await k8sApi.readNamespacedPod({
+ name,
+ namespace: namespace()
+ })
+}
+
+export async function getSecretByName(name: string): Promise<k8s.V1Secret> {
+ return await k8sApi.readNamespacedSecret({
+ name,
+ namespace: namespace()
+ })
+}
+
+export async function listPodsByRunnerInstance(): Promise<k8s.V1Pod[]> {
+ const podList = await k8sApi.listNamespacedPod({
+ namespace: namespace(),
+ labelSelector: new RunnerInstanceLabel().toString()
+ })
+ return podList.items ?? []
+}
+ |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| File | ++ | Statements | ++ | Branches | ++ | Functions | ++ | Lines | ++ |
|---|---|---|---|---|---|---|---|---|---|
| index.ts | +
+
+ |
+ 98.48% | +130/132 | +95.78% | +91/95 | +100% | +19/19 | +98.47% | +129/131 | +
| npu-metrics.ts | +
+
+ |
+ 98.5% | +264/268 | +91% | +172/189 | +98.03% | +50/51 | +98.85% | +258/261 | +
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 +33 +34 +35 +36 +37 +38 +39 +40 +41 +42 +43 +44 +45 +46 +47 +48 +49 +50 +51 +52 +53 +54 +55 +56 +57 +58 +59 +60 +61 +62 +63 +64 +65 +66 +67 +68 +69 +70 +71 +72 +73 +74 +75 +76 +77 +78 +79 +80 +81 +82 +83 +84 +85 +86 +87 +88 +89 +90 +91 +92 +93 +94 +95 +96 +97 +98 +99 +100 +101 +102 +103 +104 +105 +106 +107 +108 +109 +110 +111 +112 +113 +114 +115 +116 +117 +118 +119 +120 +121 +122 +123 +124 +125 +126 +127 +128 +129 +130 +131 +132 +133 +134 +135 +136 +137 +138 +139 +140 +141 +142 +143 +144 +145 +146 +147 +148 +149 +150 +151 +152 +153 +154 +155 +156 +157 +158 +159 +160 +161 +162 +163 +164 +165 +166 +167 +168 +169 +170 +171 +172 +173 +174 +175 +176 +177 +178 +179 +180 +181 +182 +183 +184 +185 +186 +187 +188 +189 +190 +191 +192 +193 +194 +195 +196 +197 +198 +199 +200 +201 +202 +203 +204 +205 +206 +207 +208 +209 +210 +211 +212 +213 +214 +215 +216 +217 +218 +219 +220 +221 +222 +223 +224 +225 +226 +227 +228 +229 +230 +231 +232 +233 +234 +235 +236 +237 +238 +239 +240 +241 +242 +243 +244 +245 +246 +247 +248 +249 +250 +251 +252 +253 +254 +255 +256 +257 +258 +259 +260 +261 +262 +263 +264 +265 +266 +267 +268 +269 +270 +271 +272 +273 +274 +275 +276 +277 +278 +279 +280 +281 +282 +283 +284 +285 +286 +287 +288 +289 +290 +291 +292 +293 +294 +295 +296 +297 +298 +299 +300 +301 +302 +303 +304 +305 +306 +307 +308 +309 +310 +311 +312 +313 +314 +315 +316 +317 +318 +319 +320 +321 +322 +323 +324 +325 +326 +327 +328 +329 +330 +331 +332 +333 +334 +335 +336 +337 +338 +339 +340 +341 +342 +343 +344 +345 +346 +347 +348 +349 +350 +351 +352 +353 +354 +355 +356 +357 +358 +359 | + + + + + + + + +7x +7x + +7x +7x + +7x +7x +7x + +7x + + + + + + + + + + + + + + + + + + +5x +5x + + +5x + + + + + + +5x +5x +5x +5x + + + + + + + + + + + + +16x +16x + + +2x +2x + + +16x + +16x + + + + + + + + +16x +16x +16x +16x + + + + + + + + + + + + +3x + +3x +3x +1x + + +1x + + + + + + + + + + +3x +3x +3x +3x +3x + + + + + + +19x +7x + +12x +12x +14x + + + + + +3x + + + +11x + + + + + + + + +9x + + + +9x + + + +18x +18x + +18x +2x + + +16x + + + + + + + + + + + + + + +5x +9x +2x + + +2x +7x +1x +1x +6x +2x +2x +4x +1x +1x +3x +2x +2x + +1x + + + + + + + + +2x +5x +2x + +1x + + +3x +1x +1x + +2x + + + + + + + + +6x +2x + + + +4x +3x +3x +1x + +3x + + + +4x +3x +3x +1x + + + +3x + + + + + +42x +42x +38x + +4x +4x +1x + +1x + + + +3x + + +7x +7x +7x +7x +7x +7x +7x + + + +6x +6x +1x + +5x +5x + + + + + +35x +8x + +27x + + + +1x + + + +2x + + + + + + + + + +39x +2x + + + + + +37x + +39x +5x +5x +5x +5x + + + + + +32x +27x + + +5x + + + + +3x +3x +1x + +2x +2x + +1x + + + + + +2x + + | import * as k8s from '@kubernetes/client-node'
+import * as fs from 'fs'
+import * as yaml from 'js-yaml'
+import * as core from '@actions/core'
+import { v1 as uuidv4 } from 'uuid'
+import { CONTAINER_EXTENSION_PREFIX } from '../../hooks/constants'
+import * as shlex from 'shlex'
+import { Mount } from 'hooklib'
+
+export const DEFAULT_CONTAINER_ENTRY_POINT_ARGS = [`-f`, `/dev/null`]
+export const DEFAULT_CONTAINER_ENTRY_POINT = 'tail'
+
+export const ENV_HOOK_TEMPLATE_PATH = 'ACTIONS_RUNNER_CONTAINER_HOOK_TEMPLATE'
+export const ENV_USE_KUBE_SCHEDULER = 'ACTIONS_RUNNER_USE_KUBE_SCHEDULER'
+
+export const EXTERNALS_VOLUME_NAME = 'externals'
+export const GITHUB_VOLUME_NAME = 'github'
+export const WORK_VOLUME = 'work'
+
+export const CONTAINER_VOLUMES: k8s.V1VolumeMount[] = [
+ {
+ name: EXTERNALS_VOLUME_NAME,
+ mountPath: '/__e'
+ },
+ {
+ name: WORK_VOLUME,
+ mountPath: '/__w'
+ },
+ {
+ name: GITHUB_VOLUME_NAME,
+ mountPath: '/github'
+ }
+]
+
+export function prepareJobScript(userVolumeMounts: Mount[]): {
+ containerPath: string
+ runnerPath: string
+} {
+ let mountDirs = userVolumeMounts
+ .map(m => shlex.quote(m.targetVolumePath))
+ .join(' ')
+
+ const content = `#!/bin/sh -l
+set -e
+cp -R /__w/_temp/_github_home /github/home
+cp -R /__w/_temp/_github_workflow /github/workflow
+mkdir -p ${mountDirs}
+`
+
+ const filename = `${uuidv4()}.sh`
+ const entryPointPath = `${process.env.RUNNER_TEMP}/${filename}`
+ fs.writeFileSync(entryPointPath, content)
+ return {
+ containerPath: `/__w/_temp/${filename}`,
+ runnerPath: entryPointPath
+ }
+}
+
+export function writeRunScript(
+ workingDirectory: string,
+ entryPoint: string,
+ entryPointArgs?: string[],
+ prependPath?: string[],
+ environmentVariables?: { [key: string]: string }
+): { containerPath: string; runnerPath: string } {
+ let exportPath = ''
+ if (prependPath?.length) {
+ // TODO: remove compatibility with typeof prependPath === 'string' as we bump to next major version, the hooks will lose PrependPath compat with runners 2.293.0 and older
+ const prepend =
+ typeof prependPath === 'string' ? prependPath : prependPath.join(':')
+ exportPath = `export PATH=${prepend}:$PATH`
+ }
+
+ let environmentPrefix = scriptEnv(environmentVariables)
+
+ const content = `#!/bin/sh -l
+set -e
+rm "$0" # remove script after running
+${exportPath}
+cd ${workingDirectory} && \
+exec ${environmentPrefix} ${entryPoint} ${
+ entryPointArgs?.length ? entryPointArgs.join(' ') : ''
+ }
+`
+ const filename = `${uuidv4()}.sh`
+ const entryPointPath = `${process.env.RUNNER_TEMP}/${filename}`
+ fs.writeFileSync(entryPointPath, content)
+ return {
+ containerPath: `/__w/_temp/${filename}`,
+ runnerPath: entryPointPath
+ }
+}
+
+export function writeContainerStepScript(
+ dst: string,
+ workingDirectory: string,
+ entryPoint: string,
+ entryPointArgs?: string[],
+ environmentVariables?: { [key: string]: string }
+): { containerPath: string; runnerPath: string } {
+ let environmentPrefix = scriptEnv(environmentVariables)
+
+ const parts = workingDirectory.split('/').slice(-2)
+ if (parts.length !== 2) {
+ throw new Error(`Invalid working directory: ${workingDirectory}`)
+ }
+
+ const content = `#!/bin/sh -l
+rm "$0" # remove script after running
+mv /__w/_temp/_github_home /github/home && \
+mv /__w/_temp/_github_workflow /github/workflow && \
+mv /__w/_temp/_runner_file_commands /github/file_commands || true && \
+mv ${shlex.quote('/__w/' + parts.join('/') + '/')} /github/workspace && \
+cd /github/workspace && \
+exec ${environmentPrefix} ${shlex.quote(entryPoint)} ${
+ entryPointArgs?.length ? entryPointArgs.map(shlex.quote).join(' ') : ''
+ }
+`
+ const filename = `${uuidv4()}.sh`
+ const entryPointPath = `${dst}/${filename}`
+ core.debug(`Writing container step script to ${entryPointPath}`)
+ fs.writeFileSync(entryPointPath, content)
+ return {
+ containerPath: `/__w/_temp/${filename}`,
+ runnerPath: entryPointPath
+ }
+}
+
+function scriptEnv(envs?: { [key: string]: string }): string {
+ if (!envs || !Object.entries(envs).length) {
+ return ''
+ }
+ const envBuffer: string[] = []
+ for (const [key, value] of Object.entries(envs)) {
+ if (
+ key.includes(`=`) ||
+ key.includes(`'`) ||
+ key.includes(`"`) ||
+ key.includes(`$`)
+ ) {
+ throw new Error(
+ `environment key ${key} is invalid - the key must not contain =, $, ', or "`
+ )
+ }
+ envBuffer.push(
+ `"${key}=${value
+ .replace(/\\/g, '\\\\')
+ .replace(/"/g, '\\"')
+ .replace(/\$/g, '\\$')
+ .replace(/`/g, '\\`')}"`
+ )
+ }
+
+ Iif (!envBuffer?.length) {
+ return ''
+ }
+
+ return `env ${envBuffer.join(' ')} `
+}
+
+export function generateContainerName(image: string): string {
+ const nameWithTag = image.split('/').pop()
+ const name = nameWithTag?.split(':')[0]
+
+ if (!name) {
+ throw new Error(`Image definition '${image}' is invalid`)
+ }
+
+ return name
+}
+
+// Overwrite or append based on container options
+//
+// Keep in mind, envs and volumes could be passed as fields in container definition
+// so default volume mounts and envs are appended first, and then create options are used
+// to append more values
+//
+// Rest of the fields are just applied
+// For example, container.createOptions.container.image is going to overwrite container.image field
+export function mergeContainerWithOptions(
+ base: k8s.V1Container,
+ from: k8s.V1Container
+): void {
+ for (const [key, value] of Object.entries(from)) {
+ if (key === 'name') {
+ Iif (value !== CONTAINER_EXTENSION_PREFIX + base.name) {
+ core.warning("Skipping name override: name can't be overwritten")
+ }
+ continue
+ } else if (key === 'image') {
+ core.warning("Skipping image override: image can't be overwritten")
+ continue
+ } else if (key === 'env') {
+ const envs = value as k8s.V1EnvVar[]
+ base.env = mergeLists(base.env, envs)
+ } else if (key === 'volumeMounts' && value) {
+ const volumeMounts = value as k8s.V1VolumeMount[]
+ base.volumeMounts = mergeLists(base.volumeMounts, volumeMounts)
+ } else if (key === 'ports' && value) {
+ const ports = value as k8s.V1ContainerPort[]
+ base.ports = mergeLists(base.ports, ports)
+ } else {
+ base[key] = value
+ }
+ }
+}
+
+export function mergePodSpecWithOptions(
+ base: k8s.V1PodSpec,
+ from: k8s.V1PodSpec
+): void {
+ for (const [key, value] of Object.entries(from)) {
+ if (key === 'containers') {
+ base.containers.push(
+ ...from.containers.filter(
+ e => !e.name?.startsWith(CONTAINER_EXTENSION_PREFIX)
+ )
+ )
+ } else if (key === 'volumes' && value) {
+ const volumes = value as k8s.V1Volume[]
+ base.volumes = mergeLists(base.volumes, volumes)
+ } else {
+ base[key] = value
+ }
+ }
+}
+
+export function mergeObjectMeta(
+ base: { metadata?: k8s.V1ObjectMeta },
+ from: k8s.V1ObjectMeta
+): void {
+ if (!base.metadata?.labels || !base.metadata?.annotations) {
+ throw new Error(
+ "Can't merge metadata: base.metadata or base.annotations field is undefined"
+ )
+ }
+ if (from?.labels) {
+ for (const [key, value] of Object.entries(from.labels)) {
+ if (base.metadata?.labels?.[key]) {
+ core.warning(`Label ${key} is already defined and will be overwritten`)
+ }
+ base.metadata.labels[key] = value
+ }
+ }
+
+ if (from?.annotations) {
+ for (const [key, value] of Object.entries(from.annotations)) {
+ if (base.metadata?.annotations?.[key]) {
+ core.warning(
+ `Annotation ${key} is already defined and will be overwritten`
+ )
+ }
+ base.metadata.annotations[key] = value
+ }
+ }
+}
+
+export function readExtensionFromFile(): k8s.V1PodTemplateSpec | undefined {
+ const filePath = process.env[ENV_HOOK_TEMPLATE_PATH]
+ if (!filePath) {
+ return undefined
+ }
+ const doc = yaml.load(fs.readFileSync(filePath, 'utf8'))
+ if (!doc || typeof doc !== 'object') {
+ throw new Error(`Failed to parse ${filePath}`)
+ }
+ return doc as k8s.V1PodTemplateSpec
+}
+
+export function useKubeScheduler(): boolean {
+ return process.env[ENV_USE_KUBE_SCHEDULER] === 'true'
+}
+
+export enum PodPhase {
+ PENDING = 'Pending',
+ RUNNING = 'Running',
+ SUCCEEDED = 'Succeeded',
+ FAILED = 'Failed',
+ UNKNOWN = 'Unknown',
+ COMPLETED = 'Completed'
+}
+
+function mergeLists<T>(base?: T[], from?: T[]): T[] {
+ const b: T[] = base || []
+ if (!from?.length) {
+ return b
+ }
+ b.push(...from)
+ return b
+}
+
+export function fixArgs(args: string[]): string[] {
+ // Preserve shell command strings passed via `sh -c` without re-tokenizing.
+ // Retokenizing would split the script into multiple args, breaking `sh -c`.
+ if (args.length >= 2 && args[0] === 'sh' && args[1] === '-c') {
+ return args
+ }
+ return shlex.split(args.join(' '))
+}
+
+export async function sleep(ms: number): Promise<void> {
+ return new Promise(resolve => setTimeout(resolve, ms))
+}
+
+export function listDirAllCommand(dir: string): string {
+ return `cd ${shlex.quote(dir)} && find . -type f -not -path '*/_runner_hook_responses*' -exec stat -c '%s %n' {} \\;`
+}
+
+// Safely turn an unknown thrown value into a diagnostic string without
+// throwing. The previous `JSON.stringify(err)` pattern crashed with
+// `TypeError: Converting circular structure to JSON` when err was a
+// @kubernetes/client-node HTTP error (its response embeds a
+// TLSSocket <-> HTTPParser cycle). The thrown TypeError shadowed the
+// original failure in every catch block that used it (issue #329).
+export function formatError(err: unknown): string {
+ if (err === null || err === undefined) {
+ return String(err)
+ }
+
+ // @kubernetes/client-node API errors expose the actual server message
+ // under response.body — prefer that when available.
+ const body =
+ (err as { response?: { body?: unknown } })?.response?.body ??
+ (err as { body?: unknown })?.body
+ if (body && typeof body === 'object') {
+ const msg = (body as { message?: unknown }).message
+ const reason = (body as { reason?: unknown }).reason
+ Eif (typeof msg === 'string') {
+ return typeof reason === 'string' && reason.length > 0
+ ? `${msg} (reason: ${reason})`
+ : msg
+ }
+ }
+
+ if (err instanceof Error) {
+ return err.message
+ }
+
+ if (typeof err === 'object') {
+ // Non-Error objects sometimes carry a top-level message (axios-style
+ // errors, hand-rolled error-likes). Extract before the JSON.stringify
+ // branch so a circular ref doesn't reduce the diagnostic to
+ // "[object Object]".
+ const msg = (err as { message?: unknown }).message
+ if (typeof msg === 'string') {
+ return msg
+ }
+ try {
+ return JSON.stringify(err)
+ } catch {
+ return String(err)
+ }
+ }
+
+ // Primitives serialise more readably via String() than JSON.stringify
+ // (which would quote strings and refuse to handle symbols).
+ return String(err)
+}
+ |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 +33 +34 +35 +36 +37 +38 +39 +40 +41 +42 +43 +44 +45 +46 +47 +48 +49 +50 +51 +52 +53 +54 +55 +56 +57 +58 +59 +60 +61 +62 +63 +64 +65 +66 +67 +68 +69 +70 +71 +72 +73 +74 +75 +76 +77 +78 +79 +80 +81 +82 +83 +84 +85 +86 +87 +88 +89 +90 +91 +92 +93 +94 +95 +96 +97 +98 +99 +100 +101 +102 +103 +104 +105 +106 +107 +108 +109 +110 +111 +112 +113 +114 +115 +116 +117 +118 +119 +120 +121 +122 +123 +124 +125 +126 +127 +128 +129 +130 +131 +132 +133 +134 +135 +136 +137 +138 +139 +140 +141 +142 +143 +144 +145 +146 +147 +148 +149 +150 +151 +152 +153 +154 +155 +156 +157 +158 +159 +160 +161 +162 +163 +164 +165 +166 +167 +168 +169 +170 +171 +172 +173 +174 +175 +176 +177 +178 +179 +180 +181 +182 +183 +184 +185 +186 +187 +188 +189 +190 +191 +192 +193 +194 +195 +196 +197 +198 +199 +200 +201 +202 +203 +204 +205 +206 +207 +208 +209 +210 +211 +212 +213 +214 +215 +216 +217 +218 +219 +220 +221 +222 +223 +224 +225 +226 +227 +228 +229 +230 +231 +232 +233 +234 +235 +236 +237 +238 +239 +240 +241 +242 +243 +244 +245 +246 +247 +248 +249 +250 +251 +252 +253 +254 +255 +256 +257 +258 +259 +260 +261 +262 +263 +264 +265 +266 +267 +268 +269 +270 +271 +272 +273 +274 +275 +276 +277 +278 +279 +280 +281 +282 +283 +284 +285 +286 +287 +288 +289 +290 +291 +292 +293 +294 +295 +296 +297 +298 +299 +300 +301 +302 +303 +304 +305 +306 +307 +308 +309 +310 +311 +312 +313 +314 +315 +316 +317 +318 +319 +320 +321 +322 +323 +324 +325 +326 +327 +328 +329 +330 +331 +332 +333 +334 +335 +336 +337 +338 +339 +340 +341 +342 +343 +344 +345 +346 +347 +348 +349 +350 +351 +352 +353 +354 +355 +356 +357 +358 +359 +360 +361 +362 +363 +364 +365 +366 +367 +368 +369 +370 +371 +372 +373 +374 +375 +376 +377 +378 +379 +380 +381 +382 +383 +384 +385 +386 +387 +388 +389 +390 +391 +392 +393 +394 +395 +396 +397 +398 +399 +400 +401 +402 +403 +404 +405 +406 +407 +408 +409 +410 +411 +412 +413 +414 +415 +416 +417 +418 +419 +420 +421 +422 +423 +424 +425 +426 +427 +428 +429 +430 +431 +432 +433 +434 +435 +436 +437 +438 +439 +440 +441 +442 +443 +444 +445 +446 +447 +448 +449 +450 +451 +452 +453 +454 +455 +456 +457 +458 +459 +460 +461 +462 +463 +464 +465 +466 +467 +468 +469 +470 +471 +472 +473 +474 +475 +476 +477 +478 +479 +480 +481 +482 +483 +484 +485 +486 +487 +488 +489 +490 +491 +492 +493 +494 +495 +496 +497 +498 +499 +500 +501 +502 +503 +504 +505 +506 +507 +508 +509 +510 +511 +512 +513 +514 +515 +516 +517 +518 +519 +520 +521 +522 +523 +524 +525 +526 +527 +528 +529 +530 +531 +532 +533 +534 +535 +536 +537 +538 +539 +540 +541 +542 +543 +544 +545 +546 +547 +548 +549 +550 +551 +552 +553 +554 +555 +556 +557 +558 +559 +560 +561 +562 +563 +564 +565 +566 +567 +568 +569 +570 +571 +572 +573 +574 +575 +576 +577 +578 +579 +580 +581 +582 +583 +584 +585 +586 +587 +588 +589 +590 +591 +592 +593 +594 +595 +596 +597 +598 +599 +600 +601 +602 +603 +604 +605 +606 +607 +608 +609 +610 +611 +612 +613 +614 +615 +616 +617 +618 +619 +620 +621 +622 +623 +624 +625 +626 +627 +628 +629 +630 +631 +632 +633 +634 +635 +636 +637 +638 +639 +640 +641 +642 +643 +644 +645 +646 +647 +648 +649 +650 +651 +652 +653 +654 +655 +656 +657 +658 +659 +660 +661 +662 +663 +664 +665 +666 +667 +668 +669 +670 +671 +672 +673 +674 +675 +676 +677 +678 +679 +680 +681 +682 +683 +684 +685 +686 +687 +688 +689 +690 +691 +692 +693 +694 +695 +696 +697 +698 +699 +700 +701 +702 +703 +704 +705 +706 +707 +708 +709 +710 +711 +712 +713 +714 +715 +716 +717 +718 +719 +720 +721 +722 +723 +724 +725 +726 +727 +728 +729 +730 +731 +732 +733 +734 +735 +736 +737 +738 +739 +740 +741 +742 +743 +744 +745 +746 +747 +748 +749 +750 +751 +752 +753 +754 +755 +756 +757 +758 +759 | + + + + + + + + + + + + + + +3x +3x +3x +3x + +3x +3x +3x + +3x +3x +3x +3x + +3x +3x +3x + +3x +3x + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +51x + + + + + + + +5x +5x +1x + +4x +4x +2x + + +2x + +2x + + + + + +9x +9x +7x + +2x +2x +1x + + +1x + +1x + + + +60x +60x +107x +16x +14x + + + +46x + + + +18x + + + +19x + + + + + +8x + + + + + + + + + + + + + + + + + + + + + +3x + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +11x + + + + + + + + + + +13x + + + + + + +10x + + + + + +1x + + + + + +9x +2x + +10x + + + + + + +7x +7x + + + +1x + + +6x + + + +1x + + +1x + + +5x +5x + +5x + + + + + + + +7x + + + + + + + +7x +7x + + + +39x +1x + +38x +37x + +1x + + + + + +17x +2x + +15x +15x +2x + +13x +13x +13x +12x + +13x + +15x +15x + +2x + +13x +2x + +11x +11x +11x +2x + +9x + + + + + + + + + +11x + +2x + + + + + + +14x +1x + +15x + + + +111x + + + +8x +88x +88x + + + +3x + + + + + + + + + + + + + + + + + + +8x + + + + + + + + + + +8x +8x +8x +8x +8x +8x +8x +8x +8x + +88x + + +8x +8x +8x +8x +8x + +8x +8x + + +8x + + + +3x + + + + + + + + + + +7x + + + + + + + + +3x +3x +3x + +2x +2x + + +1x +1x + + + + + + + + + + + + + + + + + + + +18x +18x +18x + + +18x +5x + + +5x + +17x +17x + + + + + + + + +5x +5x +5x + + + +17x +17x +17x + + + + + + + + + +13x +13x +13x +13x + +13x +25x +12x + +25x +25x + + + + + + +9x +8x + +1x + +16x + + +17x + + + + + +8x +8x +18x +18x +5x + +13x +13x +13x + + + + +8x +8x +3x + +5x + + + + + + + +5x +5x +4x + +1x + + +1x + + + + +11x + + +11x +11x + + + +10x +10x +10x + + + +10x +8x +6x +6x +6x + + + +4x +2x +2x +1x + + +3x + + + + + + + + + + + +9x +9x +9x +9x +9x + + + +18x +8x +8x +8x +8x + + + + +9x +7x + +9x +9x + + +9x + + + + + + + + + + +9x +7x + +2x +9x +9x +1x + +1x + + + + + +12x +12x +1x + +11x +11x + +12x +12x +5x + +6x +1x + +5x + + + + + +4x + + + +8x +1x + +7x +7x +7x + +1x +1x + +6x +16x + +6x +1x + + +5x +5x +5x +5x + + + + +5x +5x +1x + + +1x + + +4x +8x +8x +8x + +1x + + + + + + + + + + + + +8x +7x +1x +1x + + +6x +6x + + + + + + + + + + + +8x + + + + +8x +8x +8x +4x + +2x +2x +2x + + + + + + | import * as core from '@actions/core'
+import * as k8s from '@kubernetes/client-node'
+import * as https from 'https'
+import * as http from 'http'
+import { URL } from 'url'
+import { getRunnerPodName, JOB_CONTAINER_NAME } from '../../hooks/constants'
+import { formatError } from './index'
+import {
+ execPodStepWithOutput,
+ getPodByName,
+ getSecretByName,
+ listPodsByRunnerInstance,
+ namespace
+} from '../index'
+
+export const NPU_METRICS_MARKER = 'NPU_METRICS_V1'
+export const ENV_NPU_METRICS_ENABLED = 'ACTIONS_RUNNER_NPU_METRICS'
+export const ENV_NPU_SAMPLE_INTERVAL = 'NPU_SAMPLE_INTERVAL'
+export const ENV_NPU_IDLE_THRESHOLD = 'NPU_IDLE_THRESHOLD'
+
+export const DEFAULT_NPU_SAMPLE_INTERVAL_SECONDS = 1
+export const MIN_NPU_SAMPLE_INTERVAL_SECONDS = 0.2
+export const DEFAULT_NPU_IDLE_THRESHOLD_PERCENT = 5
+
+export const NPU_SAMPLES_FILE = '/tmp/npu-samples.jsonl'
+export const NPU_TERMINATION_LOG = '/dev/termination-log'
+export const NPU_SAMPLES_MAX_BYTES = 4 * 1024 * 1024
+export const NPU_AGGREGATE_MAX_BYTES = 4096
+
+export const NPU_METRICS_SECRET_NAME = 'npu-metrics-pushgw'
+export const NPU_METRICS_PUSH_JOB = 'npu-job-record'
+export const NPU_COLLECT_TIMEOUT_MS = 10_000
+
+const NPU_RESOURCE_REGEX = /^huawei\.com\/(ascend\S*)$/i
+const SCALE_SET_NAME_REGEX = /^linux-[a-z0-9]+-[a-z0-9]+-\d+-([a-z][a-z0-9-]*)$/
+
+export interface NpuCardAggregate {
+ card: string
+ samples: number
+ peakUtil: number
+ avgUtil: number
+ peakHbm: number
+ firstTs: number
+ lastTs: number
+}
+
+export interface NpuAggregatePayload {
+ version: string
+ ts: number
+ cards: NpuCardAggregate[]
+}
+
+export interface NpuPushConfig {
+ url: string
+ username?: string
+ password?: string
+}
+
+export interface NpuJobLabels {
+ cluster: string
+ namespace: string
+ pod: string
+ podType: string
+ repo: string
+ runId: string
+ npuType: string
+ cardsRequested: string
+ result: string
+}
+
+export function npuMetricsEnabled(
+ env: Record<string, string | undefined> = process.env
+): boolean {
+ return (
+ (env[ENV_NPU_METRICS_ENABLED] ?? '').trim().toLowerCase() !== 'disabled'
+ )
+}
+
+export function getNpuSampleIntervalSeconds(
+ env: Record<string, string | undefined> = process.env
+): number {
+ const raw = env[ENV_NPU_SAMPLE_INTERVAL]
+ if (!raw) {
+ return DEFAULT_NPU_SAMPLE_INTERVAL_SECONDS
+ }
+ const parsed = Number(raw)
+ if (!Number.isFinite(parsed) || parsed <= 0) {
+ core.warning(
+ `NPU_SAMPLE_INTERVAL is invalid ("${raw}"): use ${DEFAULT_NPU_SAMPLE_INTERVAL_SECONDS}`
+ )
+ return DEFAULT_NPU_SAMPLE_INTERVAL_SECONDS
+ }
+ return Math.max(parsed, MIN_NPU_SAMPLE_INTERVAL_SECONDS)
+}
+
+export function getIdleThresholdPercent(
+ env: Record<string, string | undefined> = process.env
+): number {
+ const raw = env[ENV_NPU_IDLE_THRESHOLD]
+ if (!raw) {
+ return DEFAULT_NPU_IDLE_THRESHOLD_PERCENT
+ }
+ const parsed = Number(raw)
+ if (!Number.isFinite(parsed) || parsed < 0) {
+ core.warning(
+ `NPU_IDLE_THRESHOLD is invalid ("${raw}"): use ${DEFAULT_NPU_IDLE_THRESHOLD_PERCENT}`
+ )
+ return DEFAULT_NPU_IDLE_THRESHOLD_PERCENT
+ }
+ return parsed
+}
+
+export function containerHasNpuRequest(container: k8s.V1Container): boolean {
+ const resources = [container.resources?.requests, container.resources?.limits]
+ for (const entry of resources) {
+ for (const key of Object.keys(entry ?? {})) {
+ if (NPU_RESOURCE_REGEX.test(key)) {
+ return true
+ }
+ }
+ }
+ return false
+}
+
+export function hasNpuRequest(spec: k8s.V1PodSpec | undefined): boolean {
+ return (spec?.containers ?? []).some(c => containerHasNpuRequest(c))
+}
+
+function listNpuCardsCommand(): string {
+ return `ls /dev/davinci[0-9]* 2>/dev/null | sed 's|.*/davinci||' | sort -n | uniq | tr '\\n' ' '`
+}
+
+export function buildNpuSamplerScript(
+ intervalSeconds = DEFAULT_NPU_SAMPLE_INTERVAL_SECONDS
+): string {
+ return [
+ `# ${NPU_METRICS_MARKER} sampler (detached, best-effort)`,
+ `F=${NPU_SAMPLES_FILE}`,
+ `CARDS=$(${listNpuCardsCommand()})`,
+ `[ -z "$CARDS" ] && exit 0`,
+ `while :; do`,
+ ` NOW=$(date +%s)`,
+ ` for C in $CARDS; do`,
+ ` OUT=$(npu-smi info -t usages -i "$C" 2>/dev/null) || continue`,
+ ` U=$(printf '%s\\n' "$OUT" | awk '{l=$0; gsub(/[ \\t\\r]/, "", l)} l ~ /AICore/ { if (match(l, /[0-9.]+%?$/)) { v = substr(l, RSTART, RLENGTH); sub(/%$/, "", v); print v; exit } }')`,
+ ` H=$(printf '%s\\n' "$OUT" | awk '{l=$0; gsub(/[ \\t\\r]/, "", l)} l ~ /HBM/ { if (match(l, /[0-9.]+%?$/)) { v = substr(l, RSTART, RLENGTH); sub(/%$/, "", v); print v; exit } }')`,
+ ` printf '{"ts":%s,"card":%s,"util":%s,"hbm":%s}\\n' "$NOW" "$C" "\${U:-0}" "\${H:-0}" >> "$F" 2>/dev/null || true`,
+ ` done`,
+ ` SZ=$(wc -c < "$F" 2>/dev/null || echo 0)`,
+ ` if [ "$SZ" -gt ${NPU_SAMPLES_MAX_BYTES} ]; then`,
+ ` tail -n 2000 "$F" > "$F.r" 2>/dev/null && mv "$F.r" "$F" || true`,
+ ` fi`,
+ ` sleep ${intervalSeconds}`,
+ `done`
+ ].join('\n')
+}
+
+const AGGREGATE_AWK = [
+ `BEGIN {`,
+ ` n = split(cards, a, " ")`,
+ ` for (i = 1; i <= n; i++) { cnt[a[i]] = 0; pu[a[i]] = 0; ph[a[i]] = 0; su[a[i]] = 0; fs[a[i]] = 0; ls[a[i]] = 0 }`,
+ `}`,
+ `{`,
+ ` ts = $0; sub(/.*"ts":/, "", ts); sub(/[^0-9].*/, "", ts)`,
+ ` c = $0; sub(/.*"card":/, "", c); sub(/[^0-9].*/, "", c)`,
+ ` u = $0; sub(/.*"util":/, "", u); sub(/[^0-9.].*/, "", u)`,
+ ` h = $0; sub(/.*"hbm":/, "", h); sub(/[^0-9.].*/, "", h)`,
+ ` if (c == "") next`,
+ ` cnt[c]++`,
+ ` if (ts != "") { t = ts + 0; if (fs[c] == 0 || t < fs[c]) fs[c] = t; if (t > ls[c]) ls[c] = t }`,
+ ` if (u + 0 > pu[c]) pu[c] = u + 0`,
+ ` if (h + 0 > ph[c]) ph[c] = h + 0`,
+ ` su[c] += u + 0`,
+ `}`,
+ `END {`,
+ ` printf "{\\"v\\":\\"${NPU_METRICS_MARKER}\\",\\"ts\\":%d,\\"cards\\":[", now`,
+ ` sep = ""`,
+ ` for (c in cnt) {`,
+ ` au = cnt[c] > 0 ? su[c] / cnt[c] : 0`,
+ ` printf "%s{\\"c\\":\\"%s\\",\\"n\\":%d,\\"pu\\":%.1f,\\"au\\":%.1f,\\"ph\\":%.1f,\\"fs\\":%d,\\"ls\\":%d}", sep, c, cnt[c], pu[c], au, ph[c], fs[c], ls[c]`,
+ ` sep = ","`,
+ ` }`,
+ ` printf "]}\\n"`,
+ `}`
+].join('\n')
+
+export function buildNpuAggregateScript(): string {
+ return [
+ `# ${NPU_METRICS_MARKER} aggregate`,
+ `F=${NPU_SAMPLES_FILE}`,
+ `CARDS=$(${listNpuCardsCommand()})`,
+ `NOW=$(date +%s)`,
+ `touch "$F" 2>/dev/null || true`,
+ `awk -v now="$NOW" -v cards="$CARDS" '${AGGREGATE_AWK}' "$F" 2>/dev/null`
+ ].join('\n')
+}
+
+function commandHasMarker(command: string[] | undefined): boolean {
+ return (command ?? []).join(' ').includes(NPU_METRICS_MARKER)
+}
+
+function appendExecCommand(
+ existing: string[] | undefined,
+ appendLines: string[]
+): string[] {
+ if (
+ existing &&
+ existing.length >= 3 &&
+ /(^|\/)sh$/.test(existing[0]) &&
+ existing[1] === '-c'
+ ) {
+ return [
+ existing[0],
+ existing[1],
+ [...existing.slice(2), ...appendLines].join('\n')
+ ]
+ }
+ const prefix = (existing ?? []).map(
+ part => `'${part.replace(/'/g, "'\\''")}'`
+ )
+ return ['sh', '-c', [prefix.join(' '), ...appendLines].join('\n')]
+}
+
+export function injectNpuMetrics(
+ container: k8s.V1Container,
+ intervalSeconds = DEFAULT_NPU_SAMPLE_INTERVAL_SECONDS
+): boolean {
+ const lifecycle = container.lifecycle ?? {}
+ if (
+ commandHasMarker(lifecycle.postStart?.exec?.command) ||
+ commandHasMarker(lifecycle.preStop?.exec?.command)
+ ) {
+ return false
+ }
+
+ if (
+ (lifecycle.postStart && !lifecycle.postStart.exec) ||
+ (lifecycle.preStop && !lifecycle.preStop.exec)
+ ) {
+ core.warning(
+ 'npu-metrics: container already has a non-exec lifecycle handler, skip injection'
+ )
+ return false
+ }
+
+ const samplerLine = `(${buildNpuSamplerScript(intervalSeconds)}) >/dev/null 2>&1 &`
+ const aggregateLine = `{ ( ${buildNpuAggregateScript()} ) | head -c ${NPU_AGGREGATE_MAX_BYTES} > ${NPU_TERMINATION_LOG}; } 2>/dev/null || true`
+
+ lifecycle.postStart = {
+ exec: {
+ command: appendExecCommand(lifecycle.postStart?.exec?.command, [
+ `# ${NPU_METRICS_MARKER} postStart: start detached npu sampler`,
+ samplerLine
+ ])
+ }
+ }
+ lifecycle.preStop = {
+ exec: {
+ command: appendExecCommand(lifecycle.preStop?.exec?.command, [
+ `# ${NPU_METRICS_MARKER} preStop: write npu aggregate to termination log`,
+ aggregateLine
+ ])
+ }
+ }
+ container.lifecycle = lifecycle
+ return true
+}
+
+export function maybeInjectNpuMetrics(container: k8s.V1Container): boolean {
+ if (!npuMetricsEnabled()) {
+ return false
+ }
+ if (!containerHasNpuRequest(container)) {
+ return false
+ }
+ return injectNpuMetrics(container)
+}
+
+export function parseNpuAggregate(
+ raw: string | undefined
+): NpuAggregatePayload | undefined {
+ if (!raw) {
+ return undefined
+ }
+ const start = raw.indexOf('{"v":')
+ if (start === -1) {
+ return undefined
+ }
+ const candidates = [raw.slice(start)]
+ const end = raw.lastIndexOf('}')
+ if (end > start) {
+ candidates.push(raw.slice(start, end + 1))
+ }
+ for (const candidate of candidates) {
+ let parsed: any
+ try {
+ parsed = JSON.parse(candidate)
+ } catch {
+ continue
+ }
+ if (parsed?.v !== NPU_METRICS_MARKER || !Array.isArray(parsed.cards)) {
+ continue
+ }
+ const cards: NpuCardAggregate[] = []
+ for (const c of parsed.cards) {
+ if (c === null || typeof c !== 'object' || c.c === undefined) {
+ continue
+ }
+ cards.push({
+ card: String(c.c),
+ samples: Number(c.n) || 0,
+ peakUtil: Number(c.pu) || 0,
+ avgUtil: Number(c.au) || 0,
+ peakHbm: Number(c.ph) || 0,
+ firstTs: Number(c.fs) || 0,
+ lastTs: Number(c.ls) || 0
+ })
+ }
+ return { version: parsed.v, ts: Number(parsed.ts) || 0, cards }
+ }
+ return undefined
+}
+
+export function isIdleRecord(
+ cards: NpuCardAggregate[],
+ threshold = DEFAULT_NPU_IDLE_THRESHOLD_PERCENT
+): boolean {
+ if (!cards.length) {
+ return false
+ }
+ return cards.every(c => c.peakUtil < threshold && c.avgUtil < threshold)
+}
+
+export function escapePromLabelValue(value: string): string {
+ return value.replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/\n/g, '\\n')
+}
+
+export function formatPromLabels(labels: Record<string, string>): string {
+ return Object.entries(labels)
+ .filter(([, v]) => v !== undefined)
+ .map(([k, v]) => `${k}="${escapePromLabelValue(v)}"`)
+ .join(',')
+}
+
+const NPU_JOB_METRIC_LABEL_KEYS = [
+ 'cluster',
+ 'namespace',
+ 'repo',
+ 'run_id',
+ 'pod',
+ 'pod_type',
+ 'card',
+ 'npu_type',
+ 'cards_requested',
+ 'result',
+ 'idle'
+]
+
+export function buildNpuJobMetricsBody(
+ labels: NpuJobLabels,
+ cards: NpuCardAggregate[],
+ idleThreshold = DEFAULT_NPU_IDLE_THRESHOLD_PERCENT
+): string {
+ const base: Record<string, string> = {
+ cluster: labels.cluster,
+ namespace: labels.namespace,
+ repo: labels.repo,
+ run_id: labels.runId,
+ pod: labels.pod,
+ pod_type: labels.podType,
+ npu_type: labels.npuType,
+ cards_requested: labels.cardsRequested,
+ result: labels.result
+ }
+ const idle = isIdleRecord(cards, idleThreshold) ? 'true' : 'false'
+ const lines: string[] = []
+ lines.push('# TYPE custom_npu_job_peak_util_percent gauge')
+ lines.push('# TYPE custom_npu_job_avg_util_percent gauge')
+ lines.push('# TYPE custom_npu_job_duration_seconds gauge')
+ lines.push('# TYPE custom_npu_job_sample_count gauge')
+ for (const card of cards) {
+ const cardLabels = { ...base, card: card.card, idle }
+ const l = formatPromLabels(
+ Object.fromEntries(
+ NPU_JOB_METRIC_LABEL_KEYS.map(k => [k, cardLabels[k]])
+ ) as Record<string, string>
+ )
+ lines.push(`custom_npu_job_peak_util_percent{${l}} ${card.peakUtil}`)
+ lines.push(`custom_npu_job_avg_util_percent{${l}} ${card.avgUtil}`)
+ const duration = Math.max(card.lastTs - card.firstTs, 0)
+ lines.push(`custom_npu_job_duration_seconds{${l}} ${duration}`)
+ lines.push(`custom_npu_job_sample_count{${l}} ${card.samples}`)
+ }
+ lines.push('# TYPE custom_npu_job_push_total counter')
+ lines.push(
+ `custom_npu_job_push_total{cluster="${escapePromLabelValue(labels.cluster)}",namespace="${escapePromLabelValue(labels.namespace)}",status="success"} 1`
+ )
+ return lines.join('\n') + '\n'
+}
+
+export function buildNpuPushFailedBody(labels: NpuJobLabels): string {
+ return (
+ `# TYPE custom_npu_job_push_total counter\n` +
+ `custom_npu_job_push_total{cluster="${escapePromLabelValue(labels.cluster)}",namespace="${escapePromLabelValue(labels.namespace)}",status="failed"} 1\n`
+ )
+}
+
+export function buildPushgatewayPath(
+ cluster: string,
+ ns: string,
+ pod: string
+): string {
+ return `/metrics/job/${NPU_METRICS_PUSH_JOB}/cluster/${encodeURIComponent(
+ cluster
+ )}/ns/${encodeURIComponent(ns)}/pod/${encodeURIComponent(pod)}`
+}
+
+export async function withTimeout<T>(
+ promise: Promise<T>,
+ timeoutMs: number
+): Promise<T> {
+ return await new Promise<T>((resolve, reject) => {
+ const timer = setTimeout(() => reject(new Error('timeout')), timeoutMs)
+ promise.then(
+ value => {
+ clearTimeout(timer)
+ resolve(value)
+ },
+ err => {
+ clearTimeout(timer)
+ reject(err)
+ }
+ )
+ })
+}
+
+export interface PushRequest {
+ url: string
+ method: string
+ body: string
+ username?: string
+ password?: string
+ timeoutMs?: number
+}
+
+export type RequestFn = (req: PushRequest) => Promise<{ statusCode: number }>
+
+export async function nodeRequest(req: PushRequest): Promise<{
+ statusCode: number
+}> {
+ const target = new URL(req.url)
+ const client = target.protocol === 'http:' ? http : https
+ const headers: Record<string, string> = {
+ 'Content-Type': 'text/plain'
+ }
+ if (req.username !== undefined || req.password !== undefined) {
+ const basic = Buffer.from(
+ `${req.username ?? ''}:${req.password ?? ''}`
+ ).toString('base64')
+ headers['Authorization'] = `Basic ${basic}`
+ }
+ return await new Promise((resolve, reject) => {
+ const request = client.request(
+ target,
+ {
+ method: req.method,
+ headers,
+ timeout: req.timeoutMs ?? 5000,
+ rejectUnauthorized: false
+ },
+ response => {
+ response.resume()
+ response.on('end', () =>
+ resolve({ statusCode: response.statusCode ?? 0 })
+ )
+ }
+ )
+ request.on('timeout', () => request.destroy(new Error('request timeout')))
+ request.on('error', reject)
+ request.end(req.body)
+ })
+}
+
+export async function pushToPushgateway(
+ config: NpuPushConfig,
+ path: string,
+ body: string,
+ options?: { retries?: number; retryDelayMs?: number; requestFn?: RequestFn }
+): Promise<void> {
+ const retries = options?.retries ?? 2
+ const retryDelayMs = options?.retryDelayMs ?? 1000
+ const requestFn = options?.requestFn ?? nodeRequest
+ const url = `${config.url.replace(/\/+$/, '')}${path}`
+ let lastError: unknown
+ for (let attempt = 0; attempt <= retries; attempt++) {
+ if (attempt > 0) {
+ await new Promise(resolve => setTimeout(resolve, retryDelayMs))
+ }
+ try {
+ const response = await requestFn({
+ url,
+ method: 'POST',
+ body,
+ username: config.username,
+ password: config.password
+ })
+ if (response.statusCode >= 200 && response.statusCode < 300) {
+ return
+ }
+ lastError = new Error(`pushgateway returned ${response.statusCode}`)
+ } catch (err) {
+ lastError = err
+ }
+ }
+ throw lastError instanceof Error ? lastError : new Error(String(lastError))
+}
+
+export function decodePushSecret(
+ secret: k8s.V1Secret | undefined
+): NpuPushConfig | undefined {
+ const data = secret?.data ?? {}
+ const decode = (key: string): string | undefined => {
+ const encoded = data[key]
+ if (!encoded) {
+ return undefined
+ }
+ try {
+ const decoded = Buffer.from(encoded, 'base64').toString('utf8')
+ return decoded.length ? decoded : undefined
+ } catch {
+ return undefined
+ }
+ }
+ const url = decode('PUSHGATEWAY_URL')
+ if (!url) {
+ return undefined
+ }
+ return {
+ url,
+ username: decode('PUSHGATEWAY_USER'),
+ password: decode('PUSHGATEWAY_PASSWORD')
+ }
+}
+
+export async function readNpuPushConfig(): Promise<NpuPushConfig | undefined> {
+ try {
+ const secret = await getSecretByName(NPU_METRICS_SECRET_NAME)
+ return decodePushSecret(secret)
+ } catch (err) {
+ core.debug(
+ `npu-metrics: secret ${NPU_METRICS_SECRET_NAME} unavailable, skip push: ${formatError(err)}`
+ )
+ return undefined
+ }
+}
+
+function parseScaleSetSuffix(value: string | undefined): string | undefined {
+ Iif (!value) {
+ return undefined
+ }
+ const match = value.match(SCALE_SET_NAME_REGEX)
+ return match ? match[1] : undefined
+}
+
+export function extractClusterFromRunnerPod(pod: k8s.V1Pod): string {
+ const labels = pod.metadata?.labels ?? {}
+ const annotations = pod.metadata?.annotations ?? {}
+ const entries: [string, string][] = [
+ ...Object.entries(labels),
+ ...Object.entries(annotations)
+ ]
+ for (const [key, value] of entries) {
+ if (/scale[-_]?set|runner[-_]?deployment/i.test(key)) {
+ const suffix = parseScaleSetSuffix(value)
+ Eif (suffix) {
+ return suffix
+ }
+ }
+ }
+ for (const [, value] of entries) {
+ const suffix = parseScaleSetSuffix(value)
+ if (suffix) {
+ return suffix
+ }
+ }
+ return parseScaleSetSuffix(pod.metadata?.name) ?? 'unknown'
+}
+
+export function resolveNpuLabelsFromPod(
+ pod: k8s.V1Pod,
+ podName: string
+): {
+ podType: string
+ npuType: string
+ cardsRequested: string
+ result: string
+} {
+ const podType = podName.includes('-step-') ? 'step' : 'workflow'
+ const npuTypes = new Set<string>()
+ let cards = 0
+ for (const container of pod.spec?.containers ?? []) {
+ for (const resources of [
+ container.resources?.requests,
+ container.resources?.limits
+ ]) {
+ for (const [key, quantity] of Object.entries(resources ?? {})) {
+ const match = key.match(NPU_RESOURCE_REGEX)
+ Eif (match) {
+ npuTypes.add(match[1])
+ cards = Math.max(cards, parseInt(String(quantity), 10) || 0)
+ }
+ }
+ }
+ }
+ const mainStatus = (pod.status?.containerStatuses ?? []).find(
+ s => s.name === JOB_CONTAINER_NAME
+ )
+ const terminated = mainStatus?.state?.terminated
+ const result = terminated
+ ? String(terminated.exitCode)
+ : (pod.status?.phase ?? 'unknown')
+ return {
+ podType,
+ npuType: Array.from(npuTypes).sort().join(',') || 'unknown',
+ cardsRequested: String(cards || 0),
+ result
+ }
+}
+
+export function repoFromEnv(
+ env: Record<string, string | undefined> = process.env
+): string {
+ if (env.GITHUB_REPOSITORY) {
+ return env.GITHUB_REPOSITORY
+ }
+ const workspace = env.GITHUB_WORKSPACE ?? ''
+ const parts = workspace.split('/').filter(p => p.length > 0)
+ if (parts.length >= 2) {
+ return `${parts[parts.length - 2]}/${parts[parts.length - 1]}`
+ }
+ return 'unknown'
+}
+
+export async function collectNpuAggregateFromPod(
+ pod: k8s.V1Pod
+): Promise<NpuAggregatePayload | undefined> {
+ const podName = pod.metadata?.name
+ if (!podName) {
+ return undefined
+ }
+ const mainStatus = (pod.status?.containerStatuses ?? []).find(
+ s => s.name === JOB_CONTAINER_NAME
+ )
+ const terminatedMessage = mainStatus?.state?.terminated?.message
+ if (terminatedMessage) {
+ return parseNpuAggregate(terminatedMessage)
+ }
+ if (!mainStatus?.state?.running) {
+ return undefined
+ }
+ const { output } = await execPodStepWithOutput(
+ ['sh', '-c', buildNpuAggregateScript()],
+ podName,
+ JOB_CONTAINER_NAME,
+ 5
+ )
+ return parseNpuAggregate(output)
+}
+
+export async function collectAndPushNpuMetrics(): Promise<void> {
+ if (!npuMetricsEnabled()) {
+ return
+ }
+ let pods: k8s.V1Pod[] = []
+ try {
+ pods = await listPodsByRunnerInstance()
+ } catch (err) {
+ core.debug(`npu-metrics: pod list failed: ${formatError(err)}`)
+ return
+ }
+ const npuPods = pods.filter(
+ pod => pod.metadata?.name && hasNpuRequest(pod.spec)
+ )
+ if (!npuPods.length) {
+ return
+ }
+
+ let cluster = 'unknown'
+ try {
+ const runnerPod = await getPodByName(getRunnerPodName())
+ cluster = extractClusterFromRunnerPod(runnerPod)
+ } catch (err) {
+ core.debug(`npu-metrics: cluster label unresolved: ${formatError(err)}`)
+ }
+
+ const config = await readNpuPushConfig()
+ if (!config) {
+ core.debug(
+ `npu-metrics: push config missing (secret ${NPU_METRICS_SECRET_NAME}), skip push`
+ )
+ return
+ }
+
+ for (const pod of npuPods) {
+ const podName = pod.metadata?.name as string
+ try {
+ await collectAndPushForPod(pod, podName, cluster, config)
+ } catch (err) {
+ core.debug(
+ `npu-metrics: collection failed for ${podName}: ${formatError(err)}`
+ )
+ }
+ }
+}
+
+async function collectAndPushForPod(
+ pod: k8s.V1Pod,
+ podName: string,
+ cluster: string,
+ config: NpuPushConfig
+): Promise<void> {
+ const payload = await collectNpuAggregateFromPod(pod)
+ if (!payload || !payload.cards.length) {
+ core.debug(`npu-metrics: no aggregate payload for ${podName}, skip`)
+ return
+ }
+
+ const npuInfo = resolveNpuLabelsFromPod(pod, podName)
+ const labels: NpuJobLabels = {
+ cluster,
+ namespace: namespace(),
+ pod: podName,
+ podType: npuInfo.podType,
+ repo: repoFromEnv(),
+ runId: process.env.GITHUB_RUN_ID ?? '',
+ npuType: npuInfo.npuType,
+ cardsRequested: npuInfo.cardsRequested,
+ result: npuInfo.result
+ }
+
+ const body = buildNpuJobMetricsBody(
+ labels,
+ payload.cards,
+ getIdleThresholdPercent()
+ )
+ const path = buildPushgatewayPath(labels.cluster, labels.namespace, podName)
+ try {
+ await pushToPushgateway(config, path, body)
+ core.debug(`npu-metrics: pushed job record for ${podName}`)
+ } catch (err) {
+ core.warning(`npu-metrics: push failed for ${podName}: ${formatError(err)}`)
+ try {
+ await pushToPushgateway(config, path, buildNpuPushFailedBody(labels))
+ } catch {
+ // pushgateway unreachable, nothing else to record
+ }
+ }
+}
+ |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| File | ++ | Statements | ++ | Branches | ++ | Functions | ++ | Lines | ++ |
|---|---|---|---|---|---|---|---|---|---|
| src/hooks | +
+
+ |
+ 70.5% | +251/356 | +53.64% | +162/302 | +80% | +24/30 | +71.68% | +238/332 | +
| src/k8s | +
+
+ |
+ 53.94% | +472/875 | +54.1% | +323/597 | +47.69% | +62/130 | +54.59% | +428/784 | +
| src/k8s/utils | +
+
+ |
+ 33.65% | +177/526 | +16.43% | +93/566 | +22.85% | +16/70 | +36.09% | +161/446 | +
| tests | +
+
+ |
+ 33.82% | +23/68 | +0% | +0/7 | +14.28% | +2/14 | +32.83% | +22/67 | +
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 | 7x +7x +7x +7x + + + + + + +7x +21x + +42x + + + + +42x + + | import * as core from '@actions/core'
+import { formatError } from '../k8s/utils'
+import { prunePods, pruneSecrets } from '../k8s'
+import {
+ collectAndPushNpuMetrics,
+ NPU_COLLECT_TIMEOUT_MS,
+ npuMetricsEnabled,
+ withTimeout
+} from '../k8s/utils/npu-metrics'
+
+export async function cleanupJob(): Promise<void> {
+ if (InpuMetricsEnabled()) {
+ try {
+ await withTimeout(collectAndPushNpuMetrics(), NPU_COLLECT_TIMEOUT_MS)
+ } catch (err) {
+ core.debug(`npu-metrics: collection failed: ${formatError(err)}`)
+ }
+ }
+ await Promise.all([prunePods(), pruneSecrets()])
+}
+ |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 +33 +34 +35 +36 +37 +38 +39 +40 +41 +42 +43 +44 +45 +46 +47 +48 +49 +50 +51 +52 +53 +54 +55 +56 +57 +58 +59 +60 +61 +62 +63 +64 +65 +66 | 15x + +15x +105x +105x +74x + + + +31x + + +15x +11x + + + + + +15x +17x + + + + + +15x +4x +4x +3x + +1x + + +15x +3x + + + + + +15x +15x +15x +15x +15x + +15x + + +67x + + +15x +4x + + +15x +3x + + +15x +1x + +15x + | import { v4 as uuidv4 } from 'uuid'
+
+export function getRunnerPodName(): string {
+ const name = process.env.ACTIONS_RUNNER_POD_NAME
+ if (!name) {
+ throw new Error(
+ "'ACTIONS_RUNNER_POD_NAME' env is required, please contact your self hosted runner administrator"
+ )
+ }
+ return name
+}
+
+export function getJobPodName(): string {
+ return `${getRunnerPodName().substring(
+ 0,
+ MAX_POD_NAME_LENGTH - '-workflow'.length
+ )}-workflow`
+}
+
+export function getStepPodName(): string {
+ return `${getRunnerPodName().substring(
+ 0,
+ MAX_POD_NAME_LENGTH - ('-step-'.length + STEP_POD_NAME_SUFFIX_LENGTH)
+ )}-step-${uuidv4().substring(0, STEP_POD_NAME_SUFFIX_LENGTH)}`
+}
+
+export function getVolumeClaimName(): string {
+ const name = process.env.ACTIONS_RUNNER_CLAIM_NAME
+ if (!name) {
+ return `${getRunnerPodName()}-work`
+ }
+ return name
+}
+
+export function getSecretName(): string {
+ return `${getRunnerPodName().substring(
+ 0,
+ MAX_POD_NAME_LENGTH - ('-secret-'.length + STEP_POD_NAME_SUFFIX_LENGTH)
+ )}-secret-${uuidv4().substring(0, STEP_POD_NAME_SUFFIX_LENGTH)}`
+}
+
+export const MAX_POD_NAME_LENGTH = 63
+export const STEP_POD_NAME_SUFFIX_LENGTH = 8
+export const CONTAINER_EXTENSION_PREFIX = '$'
+export const JOB_CONTAINER_NAME = 'job'
+export const JOB_CONTAINER_EXTENSION_NAME = '$job'
+
+export class RunnerInstanceLabel {
+ private podName: string
+ constructor() {
+ this.podName = getRunnerPodName()
+ }
+
+ get key(): string {
+ return 'runner-pod'
+ }
+
+ get value(): string {
+ return this.podName
+ }
+
+ toString(): string {
+ return `runner-pod=${this.podName}`
+ }
+}
+ |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| File | ++ | Statements | ++ | Branches | ++ | Functions | ++ | Lines | ++ |
|---|---|---|---|---|---|---|---|---|---|
| cleanup-job.ts | +
+
+ |
+ 88.88% | +8/9 | +66.66% | +2/3 | +100% | +1/1 | +88.88% | +8/9 | +
| constants.ts | +
+
+ |
+ 100% | +32/32 | +100% | +4/4 | +100% | +10/10 | +100% | +31/31 | +
| index.ts | +
+
+ |
+ 100% | +4/4 | +100% | +0/0 | +100% | +0/0 | +100% | +4/4 | +
| prepare-job.ts | +
+
+ |
+ 86.57% | +129/149 | +71.27% | +134/188 | +88.88% | +8/9 | +88.8% | +119/134 | +
| run-container-step.ts | +
+
+ |
+ 29.46% | +33/112 | +9.89% | +9/91 | +28.57% | +2/7 | +31.13% | +33/106 | +
| run-script-step.ts | +
+
+ |
+ 90% | +45/50 | +81.25% | +13/16 | +100% | +3/3 | +89.58% | +43/48 | +
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 +2 +3 +4 +5 | 7x +7x +7x +7x + | export * from './cleanup-job' +export * from './prepare-job' +export * from './run-script-step' +export * from './run-container-step' + |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 +33 +34 +35 +36 +37 +38 +39 +40 +41 +42 +43 +44 +45 +46 +47 +48 +49 +50 +51 +52 +53 +54 +55 +56 +57 +58 +59 +60 +61 +62 +63 +64 +65 +66 +67 +68 +69 +70 +71 +72 +73 +74 +75 +76 +77 +78 +79 +80 +81 +82 +83 +84 +85 +86 +87 +88 +89 +90 +91 +92 +93 +94 +95 +96 +97 +98 +99 +100 +101 +102 +103 +104 +105 +106 +107 +108 +109 +110 +111 +112 +113 +114 +115 +116 +117 +118 +119 +120 +121 +122 +123 +124 +125 +126 +127 +128 +129 +130 +131 +132 +133 +134 +135 +136 +137 +138 +139 +140 +141 +142 +143 +144 +145 +146 +147 +148 +149 +150 +151 +152 +153 +154 +155 +156 +157 +158 +159 +160 +161 +162 +163 +164 +165 +166 +167 +168 +169 +170 +171 +172 +173 +174 +175 +176 +177 +178 +179 +180 +181 +182 +183 +184 +185 +186 +187 +188 +189 +190 +191 +192 +193 +194 +195 +196 +197 +198 +199 +200 +201 +202 +203 +204 +205 +206 +207 +208 +209 +210 +211 +212 +213 +214 +215 +216 +217 +218 +219 +220 +221 +222 +223 +224 +225 +226 +227 +228 +229 +230 +231 +232 +233 +234 +235 +236 +237 +238 +239 +240 +241 +242 +243 +244 +245 +246 +247 +248 +249 +250 +251 +252 +253 +254 +255 +256 +257 +258 +259 +260 +261 +262 +263 +264 +265 +266 +267 +268 +269 +270 +271 +272 +273 +274 +275 +276 +277 +278 +279 +280 +281 +282 +283 +284 +285 +286 +287 +288 +289 +290 +291 +292 +293 +294 +295 +296 +297 +298 +299 +300 +301 +302 +303 +304 +305 +306 +307 +308 +309 +310 +311 +312 +313 +314 +315 +316 +317 +318 +319 +320 +321 +322 +323 +324 +325 +326 +327 +328 +329 +330 +331 +332 +333 +334 +335 +336 +337 +338 +339 +340 +341 +342 +343 +344 +345 +346 +347 +348 +349 +350 +351 +352 +353 +354 +355 +356 +357 +358 +359 +360 +361 +362 +363 +364 | 8x + +8x + + + + + + +8x + + + + + + + + + +8x + + + + + + + + + + + +8x + + + + +8x +8x + +8x + + + +10x +1x + + +18x + +9x + +9x +9x +8x + + + + + +8x + + +9x +9x +9x +1x +2x +2x +2x + + +1x +1x +2x +2x +2x + + +2x +2x + + + + +2x +2x +2x + + + +9x +1x + + +8x + +8x + + + + + + + +4x +2x + + + + + + +2x +2x + + + + + + +2x +2x + + +2x +2x + + + +2x + + +1x +1x + + +1x +1x +1x +1x + + + + + + + + +2x + + + + + + + + + +2x + + +6x + + +6x + + + +6x +6x +1x + +5x + + +5x + + + + +10x + + + + + + +2x + + + +1x +1x + + +8x + +8x + + + + + + + + + + + + + + + + + + + +4x + +4x + +4x + + + + +1x +1x +1x + +3x +3x + + + + + + + + + +3x + + +3x + + + + + + + +3x +1x + +3x +1x +1x +1x + + + + + +1x + + + + + +3x + +1x + + + +1x +3x + +2x +2x + + + + + + + +2x + + + + + + +3x + + +8x + + +29x + + +21x +1x +1x + + +21x + + + + +21x +1x + + +21x +18x + + +21x +10x + + +21x +21x + + +3x +2x + + + +21x + + + + +21x +20x + + + + + +21x + +21x +20x + + +1x +1x + + +1x +1x + + +1x + + | import * as core from '@actions/core'
+import * as k8s from '@kubernetes/client-node'
+import {
+ JobContainerInfo,
+ ContextPorts,
+ PrepareJobArgs,
+ writeToResponseFile,
+ ServiceContainerInfo
+} from 'hooklib'
+import {
+ containerPorts,
+ createJobPod,
+ isPodContainerAlpine,
+ prunePods,
+ waitForPodPhases,
+ getPrepareJobTimeoutSeconds,
+ execCpToPod,
+ execPodStep
+} from '../k8s'
+import {
+ CONTAINER_VOLUMES,
+ DEFAULT_CONTAINER_ENTRY_POINT,
+ DEFAULT_CONTAINER_ENTRY_POINT_ARGS,
+ formatError,
+ generateContainerName,
+ mergeContainerWithOptions,
+ readExtensionFromFile,
+ PodPhase,
+ fixArgs,
+ prepareJobScript
+} from '../k8s/utils'
+import {
+ CONTAINER_EXTENSION_PREFIX,
+ getJobPodName,
+ JOB_CONTAINER_NAME
+} from './constants'
+import { maybeInjectNpuMetrics } from '../k8s/utils/npu-metrics'
+import { dirname } from 'path'
+
+export async function prepareJob(
+ args: PrepareJobArgs,
+ responseFile
+): Promise<void> {
+ if (!args.container) {
+ throw new Error('Job Container is required.')
+ }
+
+ await prunePods()
+
+ const extension = readExtensionFromFile()
+
+ let container: k8s.V1Container | undefined = undefined
+ if (args.container?.image) {
+ container = createContainerSpec(
+ args.container,
+ JOB_CONTAINER_NAME,
+ true,
+ extension
+ )
+ maybeInjectNpuMetrics(container)
+ }
+
+ let services: k8s.V1Container[] = []
+ let serviceNames: string[] = []
+ if (args.services?.length) {
+ const occurrences = new Map<string, number>()
+ for (const s of args.services) {
+ const base = generateContainerName(s.image)
+ occurrences.set(base, (occurrences.get(base) || 0) + 1)
+ }
+
+ const indices = new Map<string, number>()
+ services = args.services.map(service => {
+ const base = generateContainerName(service.image)
+ const total = occurrences.get(base) || 0
+ const idx = indices.get(base) || 0
+
+ let name: string
+ if (total > 1) {
+ name = `${base}-${idx}`
+ } else E{
+ name = base
+ }
+
+ indices.set(base, idx + 1)
+ serviceNames.push(name)
+ return createContainerSpec(service, name, false, extension)
+ })
+ }
+
+ if (!container && !services?.length) {
+ throw new Error('No containers exist, skipping hook invocation')
+ }
+
+ let createdPod: k8s.V1Pod | undefined = undefined
+ try {
+ createdPod = await createJobPod(
+ getJobPodName(),
+ container,
+ services,
+ args.container.registry,
+ extension
+ )
+ } catch (err) {
+ await prunePods()
+ core.debug(`createPod failed: ${JSON.stringify(err)}`)
+ // The k8s client throws HttpException whose message is a multi-line string
+ // containing the raw HTTP dump. Extract the human-readable "message" field
+ // from the embedded JSON body so the log shows something like:
+ // failed to create job pod:
+ // spec.volumes[5].name: Duplicate value: "bad-hostpath"
+ // instead of the full HTTP dump.
+ const raw = err instanceof Error ? err.message : String(err)
+ let detail = raw
+ // The k8s HttpException message is a multi-line dump:
+ // HTTP-Code: 422
+ // Message: Unknown API Status Code!
+ // Body: "{\"kind\":\"Status\",\"message\":\"...\\n\"}"
+ // Headers: {...}
+ // Extract the Body JSON string, unescape it, and pull out "message".
+ try {
+ const bodyStart = raw.indexOf('Body: "')
+ // The boundary may be '"\nHeaders:' (real newline) or the literal
+ // string ends before Headers — use the last '"' before 'Headers:' as fallback
+ const headersIdx = raw.indexOf('Headers:')
+ const bodyEnd =
+ headersIdx !== -1
+ ? raw.lastIndexOf('"', headersIdx) // last " before Headers:
+ : raw.indexOf('"\nHeaders:')
+ if (bodyStart !== -1 && bodyEnd !== -1 && bodyEnd > bodyStart) {
+ // Body content is a JSON string literal (without surrounding quotes).
+ // Wrap it in quotes and JSON.parse to properly unescape \" \\ \n \t etc.
+ const escaped = raw.substring(bodyStart + 7, bodyEnd)
+ const bodyStr = JSON.parse('"' + escaped + '"')
+ // bodyStr may be plain text (e.g. 502 Bad Gateway) instead of JSON.
+ // Parse separately so a non-JSON body still yields a friendly message.
+ try {
+ const parsed = JSON.parse(bodyStr)
+ Eif (typeof parsed?.message === 'string') {
+ detail = parsed.message
+ }
+ } catch {
+ detail = bodyStr
+ }
+ }
+ } catch {
+ // Parsing failed — fall through and show the raw string
+ }
+ const errorMessage = [
+ 'failed to create job pod:',
+ ` ✗ ${detail}`,
+ '-'.repeat(60),
+ ' → Pod spec was rejected by the k8s API. Check:',
+ ' - resources.requests does not exceed resources.limits',
+ ' - volumeMounts reference a volume defined in spec.volumes',
+ ' - envFrom / valueFrom reference existing Secrets / ConfigMaps',
+ ' - Field types match the k8s schema (kubectl explain pod.spec.containers)'
+ ].join('\n')
+ throw new Error(errorMessage)
+ }
+
+ Iif (!createdPod?.metadata?.name) {
+ throw new Error('created pod should have metadata.name')
+ }
+ core.debug(
+ `Job pod created, waiting for it to come online ${createdPod?.metadata?.name}`
+ )
+
+ const runnerWorkspaceEnv = process.env.RUNNER_WORKSPACE
+ if (!runnerWorkspaceEnv) {
+ throw new Error('RUNNER_WORKSPACE environment variable is not set')
+ }
+ const runnerWorkspace = dirname(runnerWorkspaceEnv)
+
+ let prepareScript: { containerPath: string; runnerPath: string } | undefined
+ Iif (args.container?.userMountVolumes?.length) {
+ prepareScript = prepareJobScript(args.container.userMountVolumes || [])
+ }
+
+ try {
+ await waitForPodPhases(
+ createdPod.metadata.name,
+ new Set([PodPhase.RUNNING]),
+ new Set([PodPhase.PENDING]),
+ getPrepareJobTimeoutSeconds()
+ )
+ } catch (err) {
+ await prunePods()
+ // Unwrap nested "Error: " prefix so the message renders as:
+ // pod failed to come online:
+ // <detail from waitForPodPhases, already formatted with sections>
+ const detail = err instanceof Error ? err.message : String(err)
+ throw new Error(`pod failed to come online:\n${detail}`)
+ }
+
+ await execCpToPod(createdPod.metadata.name, runnerWorkspace, '/__w')
+
+ if (EprepareScript) {
+ await execPodStep(
+ ['sh', '-e', prepareScript.containerPath],
+ createdPod.metadata.name,
+ JOB_CONTAINER_NAME
+ )
+
+ const promises: Promise<void>[] = []
+ for (const vol of args?.container?.userMountVolumes || []) {
+ promises.push(
+ execCpToPod(
+ createdPod.metadata.name,
+ vol.sourceVolumePath,
+ vol.targetVolumePath
+ )
+ )
+ }
+ await Promise.all(promises)
+ }
+
+ core.debug('Job pod is ready for traffic')
+
+ let isAlpine = false
+ try {
+ isAlpine = await isPodContainerAlpine(
+ createdPod.metadata.name,
+ JOB_CONTAINER_NAME
+ )
+ } catch (err) {
+ const message = formatError(err)
+ core.debug(`Failed to determine if the pod is alpine: ${message}`)
+ throw new Error(`failed to determine if the pod is alpine: ${message}`)
+ }
+ core.debug(`Setting isAlpine to ${isAlpine}`)
+ generateResponseFile(responseFile, args, createdPod, isAlpine, serviceNames)
+}
+
+function generateResponseFile(
+ responseFile: string,
+ args: PrepareJobArgs,
+ appPod: k8s.V1Pod,
+ isAlpine: boolean,
+ serviceNames?: string[]
+): void {
+ Iif (!appPod.metadata?.name) {
+ throw new Error('app pod must have metadata.name specified')
+ }
+ const response = {
+ state: {
+ jobPod: appPod.metadata.name
+ },
+ context: {},
+ isAlpine
+ }
+
+ const mainContainer = appPod.spec?.containers?.find(
+ c => c.name === JOB_CONTAINER_NAME
+ )
+ if (mainContainer) {
+ const mainContainerContextPorts: ContextPorts = {}
+ Eif (mainContainer?.ports) {
+ for (const port of mainContainer.ports) {
+ mainContainerContextPorts[port.containerPort] =
+ mainContainerContextPorts.hostPort
+ }
+ }
+
+ response.context['container'] = {
+ image: mainContainer.image,
+ ports: mainContainerContextPorts
+ }
+ }
+
+ if (args.services?.length) {
+ const serviceContainerNames =
+ serviceNames && serviceNames.length
+ ? serviceNames
+ : args.services?.map(s => generateContainerName(s.image)) || []
+
+ response.context['services'] = appPod?.spec?.containers
+ ?.filter(c => serviceContainerNames.includes(c.name))
+ .map(c => {
+ const ctxPorts: ContextPorts = {}
+ Iif (c.ports?.length) {
+ for (const port of c.ports) {
+ if (port.containerPort && port.hostPort) {
+ ctxPorts[port.containerPort.toString()] = port.hostPort.toString()
+ }
+ }
+ }
+
+ return {
+ image: c.image,
+ ports: ctxPorts
+ }
+ })
+ }
+
+ writeToResponseFile(responseFile, JSON.stringify(response))
+}
+
+export function createContainerSpec(
+ container: JobContainerInfo | ServiceContainerInfo,
+ name: string,
+ jobContainer = false,
+ extension?: k8s.V1PodTemplateSpec
+): k8s.V1Container {
+ if (!container.entryPoint && jobContainer) {
+ container.entryPoint = DEFAULT_CONTAINER_ENTRY_POINT
+ container.entryPointArgs = DEFAULT_CONTAINER_ENTRY_POINT_ARGS
+ }
+
+ const podContainer = {
+ name,
+ image: container.image,
+ ports: containerPorts(container)
+ } as k8s.V1Container
+ if (container['workingDirectory']) {
+ podContainer.workingDir = container['workingDirectory']
+ }
+
+ if (container.entryPoint) {
+ podContainer.command = [container.entryPoint]
+ }
+
+ if (container.entryPointArgs && container.entryPointArgs.length > 0) {
+ podContainer.args = fixArgs(container.entryPointArgs)
+ }
+
+ podContainer.env = []
+ for (const [key, value] of Object.entries(
+ container['environmentVariables'] || {}
+ )) {
+ if (value && key !== 'HOME') {
+ podContainer.env.push({ name: key, value })
+ }
+ }
+
+ podContainer.env.push({
+ name: 'GITHUB_ACTIONS',
+ value: 'true'
+ })
+
+ if (!('CI' in (container['environmentVariables'] || {}))) {
+ podContainer.env.push({
+ name: 'CI',
+ value: 'true'
+ })
+ }
+
+ podContainer.volumeMounts = CONTAINER_VOLUMES
+
+ if (!extension) {
+ return podContainer
+ }
+
+ const from = extension.spec?.containers?.find(
+ c => c.name === CONTAINER_EXTENSION_PREFIX + name
+ )
+
+ Eif (from) {
+ mergeContainerWithOptions(podContainer, from)
+ }
+
+ return podContainer
+}
+ |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 +33 +34 +35 +36 +37 +38 +39 +40 +41 +42 +43 +44 +45 +46 +47 +48 +49 +50 +51 +52 +53 +54 +55 +56 +57 +58 +59 +60 +61 +62 +63 +64 +65 +66 +67 +68 +69 +70 +71 +72 +73 +74 +75 +76 +77 +78 +79 +80 +81 +82 +83 +84 +85 +86 +87 +88 +89 +90 +91 +92 +93 +94 +95 +96 +97 +98 +99 +100 +101 +102 +103 +104 +105 +106 +107 +108 +109 +110 +111 +112 +113 +114 +115 +116 +117 +118 +119 +120 +121 +122 +123 +124 +125 +126 +127 +128 +129 +130 +131 +132 +133 +134 +135 +136 +137 +138 +139 +140 +141 +142 +143 +144 +145 +146 +147 +148 +149 +150 +151 +152 +153 +154 +155 +156 +157 +158 +159 +160 +161 +162 +163 +164 +165 +166 +167 +168 +169 +170 +171 +172 +173 +174 +175 +176 +177 +178 +179 +180 +181 +182 +183 +184 +185 +186 +187 +188 +189 +190 +191 +192 +193 +194 +195 +196 +197 +198 +199 +200 +201 +202 +203 +204 +205 +206 +207 +208 +209 +210 +211 +212 +213 +214 +215 +216 +217 +218 +219 +220 +221 +222 +223 +224 +225 +226 +227 +228 +229 +230 +231 +232 +233 +234 +235 +236 +237 +238 +239 +240 +241 +242 +243 +244 +245 +246 +247 +248 +249 +250 +251 +252 +253 +254 +255 +256 +257 +258 +259 +260 +261 +262 +263 +264 +265 +266 +267 +268 +269 +270 +271 +272 | 8x +8x +8x + +8x +8x + + + + + + + + + + + + +8x + + + + + + + + +8x + + + + + +8x + +8x + + +16x +1x + + +15x +1x + + + + +14x +14x +14x +13x + + +14x + +14x +14x + + + +14x + +14x +14x +14x + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +14x +14x +14x +14x +14x +14x + +14x + +14x +14x + + + + + + + + + + + + | import * as core from '@actions/core'
+import * as fs from 'fs'
+import * as k8s from '@kubernetes/client-node'
+import { RunContainerStepArgs } from 'hooklib'
+import { dirname } from 'path'
+import {
+ createContainerStepPod,
+ deletePod,
+ describePodFailure,
+ execCpFromPod,
+ execCpToPod,
+ execPodStepWithOutput,
+ getContainerTerminatedErrors,
+ getPodByName,
+ getPrepareJobTimeoutSeconds,
+ getTerminatedReasonHint,
+ waitForPodPhases
+} from '../k8s'
+import {
+ CONTAINER_VOLUMES,
+ formatError,
+ mergeContainerWithOptions,
+ PodPhase,
+ readExtensionFromFile,
+ DEFAULT_CONTAINER_ENTRY_POINT_ARGS,
+ writeContainerStepScript
+} from '../k8s/utils'
+import {
+ getJobPodName,
+ getStepPodName,
+ JOB_CONTAINER_EXTENSION_NAME,
+ JOB_CONTAINER_NAME
+} from './constants'
+import { maybeInjectNpuMetrics } from '../k8s/utils/npu-metrics'
+
+export async function runContainerStep(
+ stepContainer: RunContainerStepArgs
+): Promise<number> {
+ if (stepContainer.dockerfile) {
+ throw new Error('Building container actions is not currently supported')
+ }
+
+ if (!stepContainer.entryPoint) {
+ throw new Error(
+ 'failed to start the container since the entrypoint is overwritten'
+ )
+ }
+
+ const envs = stepContainer.environmentVariables || {}
+ envs['GITHUB_ACTIONS'] = 'true'
+ if (!('CI' in envs)) {
+ envs.CI = 'true'
+ }
+
+ const extension = readExtensionFromFile()
+
+ const container = createContainerSpec(stepContainer, extension)
+ maybeInjectNpuMetrics(container)
+
+ let pod: k8s.V1Pod
+ try {
+ pod = await createContainerStepPod(getStepPodName(), container, extension)
+ } catch (err) {
+ const message = formatError(err)
+ core.debug(`createContainerStepPod failed: ${message}`)
+ throw new Error(`failed to run container step: ${message}`)
+ }
+
+ if (!pod.metadata?.name) {
+ throw new Error(
+ `Expected job ${JSON.stringify(
+ pod
+ )} to have correctly set the metadata.name`
+ )
+ }
+ const podName = pod.metadata.name
+
+ try {
+ await waitForPodPhases(
+ podName,
+ new Set([PodPhase.RUNNING]),
+ new Set([PodPhase.PENDING, PodPhase.UNKNOWN]),
+ getPrepareJobTimeoutSeconds()
+ )
+
+ const runnerWorkspaceEnv = process.env.RUNNER_WORKSPACE
+ const githubWorkspaceEnv = process.env.GITHUB_WORKSPACE
+ if (!runnerWorkspaceEnv || !githubWorkspaceEnv) {
+ throw new Error(
+ 'RUNNER_WORKSPACE or GITHUB_WORKSPACE environment variable is not set'
+ )
+ }
+ const runnerWorkspace = dirname(runnerWorkspaceEnv)
+ const githubWorkspace = githubWorkspaceEnv
+ const parts = githubWorkspace.split('/').slice(-2)
+ if (parts.length !== 2) {
+ throw new Error(`Invalid github workspace directory: ${githubWorkspace}`)
+ }
+ const relativeWorkspace = parts.join('/')
+
+ core.debug(
+ `Copying files from pod ${getJobPodName()} to ${runnerWorkspace}/${relativeWorkspace}`
+ )
+ await execCpFromPod(getJobPodName(), `/__w`, `${runnerWorkspace}`)
+
+ const { containerPath, runnerPath } = writeContainerStepScript(
+ `${runnerWorkspace}/__w/_temp`,
+ githubWorkspace,
+ stepContainer.entryPoint,
+ stepContainer.entryPointArgs,
+ envs
+ )
+
+ await execCpToPod(podName, `${runnerWorkspace}/__w`, '/__w')
+
+ fs.rmSync(`${runnerWorkspace}/__w`, { recursive: true, force: true })
+
+ try {
+ core.debug(`Executing container step script in pod ${podName}`)
+ const { code, output } = await execPodStepWithOutput(
+ ['sh', '-e', containerPath],
+ pod.metadata.name,
+ JOB_CONTAINER_NAME
+ )
+ if (code === 0) {
+ return 0
+ }
+ // Non-zero exit: surface a structured error so the user can tell whether
+ // it was their script or the container that failed. Read container
+ // status BEFORE deletePod runs (in the outer finally) to inspect the
+ // terminated reason, if it is already available.
+ const classification = await classifyScriptError(
+ pod.metadata.name,
+ code,
+ output
+ )
+ throw new Error(classification)
+ } catch (err) {
+ core.debug(`execPodStep failed: ${formatError(err)}`)
+ // Re-throw our classified errors verbatim; wrap anything else.
+ if (
+ err instanceof Error &&
+ (err.message.startsWith('Step failed:') ||
+ err.message.startsWith('failed to run script step'))
+ ) {
+ throw err
+ }
+ const message = formatError(err)
+ throw new Error(`failed to run container step: ${message}`)
+ } finally {
+ fs.rmSync(runnerPath, { force: true })
+ }
+ } catch (error) {
+ try {
+ const errorPod = await getPodByName(podName)
+ const terminatedErrors = getContainerTerminatedErrors(errorPod)
+ if (terminatedErrors.length > 0) {
+ const details = await describePodFailure(podName)
+ core.error(
+ `Pod ${podName} has unrecoverable container errors:\n${terminatedErrors.join('\n')}\n${details}`
+ )
+ }
+ } catch {
+ // Best-effort: pod may already be deleted or unreachable
+ }
+ core.error(`Failed to run container step: ${error}`)
+ throw error
+ } finally {
+ await deletePod(podName).catch(err => {
+ core.error(`Failed to delete step pod ${podName}: ${err}`)
+ })
+ }
+}
+
+// Inspect the pod's container status to determine whether a non-zero exit
+// code came from the user's script (container terminated cleanly with
+// reason=Completed) or from a container-level failure (OOMKilled, etc.).
+// The container state may not yet be 'terminated' when called (k8s updates
+// it asynchronously), so we default to treating unknown state as a script
+// issue and let the user check their script first.
+async function classifyScriptError(
+ podName: string,
+ exitCode: number,
+ tailOutput: string
+): Promise<string> {
+ const sep = '-'.repeat(60)
+ const errors: string[] = [` ✗ exit code: ${exitCode}`]
+ const sections: string[] = []
+
+ try {
+ const pod = await getPodByName(podName)
+ const cs = pod.status?.containerStatuses?.find(
+ s => s.name === JOB_CONTAINER_NAME
+ )
+ const term = cs?.state?.terminated
+ if (term) {
+ const reason = term.reason ?? 'Completed'
+ const isContainerFault =
+ reason === 'OOMKilled' ||
+ reason === 'Error' ||
+ reason === 'FailedPostStartHookError' ||
+ (term.exitCode === 137 && reason !== 'Completed')
+ if (isContainerFault) {
+ const detail = term.message ? `\n ${term.message}` : ''
+ const hint = `\n${getTerminatedReasonHint(reason, term.exitCode)}`
+ errors.push(
+ ` ✗ container "${JOB_CONTAINER_NAME}": ${reason} (exit code ${term.exitCode})${detail}${hint}`
+ )
+ } else {
+ errors.push(
+ ` → your script exited with a non-zero code; please check your script for errors`
+ )
+ sections.push(
+ `Container status: ${reason} (exit code ${term.exitCode})`
+ )
+ }
+ } else {
+ // Container state unavailable — treat as script issue by default
+ errors.push(` → please check your script for errors`)
+ }
+ if (cs?.state?.waiting) {
+ errors.push(
+ ` ✗ container "${JOB_CONTAINER_NAME}" waiting: ${cs.state.waiting.reason ?? 'unknown'}`
+ )
+ }
+ } catch {
+ // pod already gone or API error — default hint
+ errors.push(` → please check your script for errors`)
+ }
+
+ if (tailOutput) {
+ const outputLines = tailOutput
+ .split('\n')
+ .map(l => ` ${l}`)
+ .join('\n')
+ sections.push(`Last output:\n${outputLines}`)
+ }
+
+ let result = `failed to run script step:\n${errors.join('\n')}`
+ if (sections.length) {
+ result += `\n${sep}\n${sections.join('\n')}`
+ }
+ return result
+}
+
+function createContainerSpec(
+ container: RunContainerStepArgs,
+ extension?: k8s.V1PodTemplateSpec
+): k8s.V1Container {
+ const podContainer = new k8s.V1Container()
+ podContainer.name = JOB_CONTAINER_NAME
+ podContainer.image = container.image
+ podContainer.workingDir = '/__w'
+ podContainer.command = ['tail']
+ podContainer.args = DEFAULT_CONTAINER_ENTRY_POINT_ARGS
+
+ podContainer.volumeMounts = CONTAINER_VOLUMES
+
+ Eif (!extension) {
+ return podContainer
+ }
+
+ const from = extension.spec?.containers?.find(
+ c => c.name === JOB_CONTAINER_EXTENSION_NAME
+ )
+ if (from) {
+ mergeContainerWithOptions(podContainer, from)
+ }
+
+ return podContainer
+}
+ |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 +33 +34 +35 +36 +37 +38 +39 +40 +41 +42 +43 +44 +45 +46 +47 +48 +49 +50 +51 +52 +53 +54 +55 +56 +57 +58 +59 +60 +61 +62 +63 +64 +65 +66 +67 +68 +69 +70 +71 +72 +73 +74 +75 +76 +77 +78 +79 +80 +81 +82 +83 +84 +85 +86 +87 +88 +89 +90 +91 +92 +93 +94 +95 +96 +97 +98 +99 +100 +101 +102 +103 +104 +105 +106 +107 +108 +109 +110 +111 +112 +113 +114 +115 +116 +117 +118 +119 +120 +121 +122 +123 +124 +125 +126 +127 +128 +129 +130 +131 +132 +133 +134 +135 +136 +137 +138 +139 +140 +141 +142 +143 +144 +145 +146 +147 +148 | +7x +7x + +7x + + + + + +7x +7x +7x + + + +3x +3x + + + +3x +3x +3x + +7x + +3x + +3x +3x +3x + + +7x + + + + +7x +2x + + + + + +5x +5x + + + + + + + +5x +5x +5x +5x + +10x + + + + + + + + +10x + + + + + +5x + + + + + + + + + + + + + + + + + +10x + + + + + + + + + + + +5x +5x + +5x + + + + +4x +3x + + +4x +4x + + + +3x + +1x +1x + +5x +5x + + + + + + +1x + + +2x + + + + + + + + + | /* eslint-disable @typescript-eslint/no-unused-vars */
+import * as fs from 'fs'
+import * as core from '@actions/core'
+import { RunScriptStepArgs } from 'hooklib'
+import {
+ execCpFromPod,
+ execCpToPod,
+ execPodStep,
+ execPodStepWithOutput
+} from '../k8s'
+import { formatError, writeRunScript } from '../k8s/utils'
+import { JOB_CONTAINER_NAME } from './constants'
+import { dirname } from 'path'
+import * as shlex from 'shlex'
+
+function formatScriptError(exitCode: number, tailOutput: string): string {
+ const sep = '-'.repeat(60)
+ const errors = [
+ ` ✗ exit code: ${exitCode}`,
+ ` → your script exited with a non-zero code; please check your script for errors`
+ ]
+ const sections: string[] = []
+ Eif (tailOutput) {
+ const outputLines = tailOutput
+ .split('\n')
+ .map(l => ` ${l}`)
+ .join('\n')
+ sections.push(`Last output:\n${outputLines}`)
+ }
+ let result = `failed to run script step:\n${errors.join('\n')}`
+ Eif (sections.length) result += `\n${sep}\n${sections.join('\n')}`
+ return result
+}
+
+export async function runScriptStep(
+ args: RunScriptStepArgs,
+ state
+): Promise<void> {
+ // Validate that pod was created successfully (prepareJob succeeded)
+ if (!state?.jobPod) {
+ throw new Error(
+ 'jobPod must be set - ensure prepareJob completed successfully before running script steps'
+ )
+ }
+
+ // Write the entrypoint first. This will be later coppied to the workflow pod
+ const { entryPoint, entryPointArgs, environmentVariables } = args
+ const { containerPath, runnerPath } = writeRunScript(
+ args.workingDirectory,
+ entryPoint,
+ entryPointArgs,
+ args.prependPath,
+ environmentVariables
+ )
+
+ const workdir = dirname(process.env.RUNNER_WORKSPACE as string)
+ const runnerTemp = `${workdir}/_temp`
+ const containerTemp = '/__w/_temp'
+ const containerTempSrc = '/__w/_temp_pre'
+ // Ensure base and staging dirs exist before copying
+ await execPodStep(
+ [
+ 'sh',
+ '-c',
+ 'mkdir -p /__w && mkdir -p /__w/_temp && mkdir -p /__w/_temp_pre'
+ ],
+ state.jobPod,
+ JOB_CONTAINER_NAME
+ )
+ await execCpToPod(state.jobPod, runnerTemp, containerTempSrc)
+
+ // Copy GitHub directories from temp to /github
+ // Merge strategy:
+ // - Overwrite files in _runner_file_commands
+ // - Append files not already present elsewhere
+ const mergeCommands = [
+ 'set -e',
+ 'mkdir -p /__w/_temp /__w/_temp_pre',
+ 'SRC=/__w/_temp_pre',
+ 'DST=/__w/_temp',
+ // Overwrite _runner_file_commands
+ 'cp -a "$SRC/_runner_file_commands/." "$DST/_runner_file_commands"',
+ `find "$SRC" -type f ! -path "*/_runner_file_commands/*" -exec sh -c '
+ rel="\${1#$2/}"
+ target="$3/$rel"
+ mkdir -p "$(dirname "$target")"
+ cp -a "$1" "$target"
+ ' _ {} "$SRC" "$DST" \\;`,
+ // Remove _temp_pre after merging
+ 'rm -rf /__w/_temp_pre'
+ ]
+
+ try {
+ await execPodStep(
+ ['sh', '-c', mergeCommands.join(' && ')],
+ state.jobPod,
+ JOB_CONTAINER_NAME
+ )
+ } catch (err) {
+ const message = formatError(err)
+ core.debug(`Failed to merge temp directories: ${message}`)
+ throw new Error(`failed to merge temp dirs: ${message}`)
+ }
+
+ // Execute the entrypoint script
+ args.entryPoint = 'sh'
+ args.entryPointArgs = ['-e', containerPath]
+ try {
+ const { code, output } = await execPodStepWithOutput(
+ [args.entryPoint, ...args.entryPointArgs],
+ state.jobPod,
+ JOB_CONTAINER_NAME
+ )
+ if (code !== 0) {
+ throw new Error(formatScriptError(code, output))
+ }
+ } catch (err) {
+ core.debug(`execPodStep failed: ${formatError(err)}`)
+ if (
+ err instanceof Error &&
+ err.message.startsWith('failed to run script step')
+ ) {
+ throw err
+ }
+ const message = formatError(err)
+ throw new Error(`failed to run script step: ${message}`)
+ } finally {
+ try {
+ fs.rmSync(runnerPath, { force: true })
+ } catch (removeErr) {
+ core.debug(`Failed to remove file ${runnerPath}: ${removeErr}`)
+ }
+ }
+
+ try {
+ core.debug(
+ `Copying from job pod '${state.jobPod}' ${containerTemp} to ${runnerTemp}`
+ )
+ await execCpFromPod(
+ state.jobPod,
+ `${containerTemp}/_runner_file_commands`,
+ `${workdir}/_temp`
+ )
+ } catch (error) {
+ core.warning('Failed to copy _temp from pod')
+ }
+}
+ |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 +33 +34 +35 +36 +37 +38 +39 +40 +41 +42 +43 +44 +45 +46 +47 +48 +49 +50 +51 +52 +53 +54 +55 +56 +57 +58 +59 +60 +61 +62 +63 +64 +65 +66 +67 +68 +69 +70 +71 +72 +73 +74 +75 +76 +77 +78 +79 +80 +81 +82 +83 +84 +85 +86 +87 +88 +89 +90 +91 +92 +93 +94 +95 +96 +97 +98 +99 +100 +101 +102 +103 +104 +105 +106 +107 +108 +109 +110 +111 +112 +113 +114 +115 +116 +117 +118 +119 +120 +121 +122 +123 +124 +125 +126 +127 +128 +129 +130 +131 +132 +133 +134 +135 +136 +137 | 14x + + + + + + + + + +14x + + + +8x +8x + + +14x +8x +8x +8x +14x + + +8x +8x + + +14x +22x +22x + + + +22x + + +14x +8x + + +8x +8x +8x + +8x +4x +4x + + + +14x +8x + + +8x + +8x +5x +1x +1x + +5x +1x + + +1x +1x +1x + + + +1x + + + + + + + + + +8x + +8x +1x + + +1x +1x + + + +8x + + + + +8x +1x +1x + + + +8x +29x + +16x + +13x +12x +12x +12x +4x +4x + +12x + + + + + + + + +1x + + + + +1x + + + +14x + | import * as core from '@actions/core'
+
+export interface HeartbeatWebSocket {
+ readyState: number
+ ping(): void
+ close(): void
+ on(event: string, listener: (...args: any[]) => void): this
+ once(event: string, listener: (...args: any[]) => void): this
+}
+
+export function parsePositiveMsEnv(
+ value: string | undefined,
+ fallback: number
+): number {
+ const parsed = Number.parseInt(value ?? '', 10)
+ return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback
+}
+
+export class WebSocketHeartbeat {
+ private pingInterval: ReturnType<typeof setInterval> | null = null
+ private pongTimeout: ReturnType<typeof setTimeout> | null = null
+ private lastHeartbeatLog = 0
+ private static readonly LOG_INTERVAL_MS = 2 * 60 * 1000
+
+ constructor(
+ private readonly pingPeriodMs: number,
+ private readonly pongDeadlineMs: number
+ ) {}
+
+ private shouldLog(): boolean {
+ const now = Date.now()
+ Iif (now - this.lastHeartbeatLog >= WebSocketHeartbeat.LOG_INTERVAL_MS) {
+ this.lastHeartbeatLog = now
+ return true
+ }
+ return false
+ }
+
+ stop(): void {
+ Iif (this.shouldLog()) {
+ core.debug('[Heartbeat] stopping heartbeat')
+ }
+ Eif (this.pingInterval) {
+ clearInterval(this.pingInterval)
+ this.pingInterval = null
+ }
+ if (this.pongTimeout) {
+ clearTimeout(this.pongTimeout)
+ this.pongTimeout = null
+ }
+ }
+
+ start(ws: HeartbeatWebSocket, reject: (err: Error) => void): void {
+ core.debug(
+ `[Heartbeat] Starting with period=${this.pingPeriodMs}ms, deadline=${this.pongDeadlineMs}ms`
+ )
+ this.lastHeartbeatLog = Date.now()
+
+ const resetPongTimeout = (): void => {
+ if (this.pongTimeout) {
+ clearTimeout(this.pongTimeout)
+ this.pongTimeout = null
+ }
+ this.pongTimeout = setTimeout(() => {
+ core.warning(
+ `[Heartbeat] No pong received in ${this.pongDeadlineMs}ms, closing stale connection`
+ )
+ this.stop()
+ try {
+ ws.close()
+ } catch {
+ // ignore errors closing an already-closing socket
+ }
+ reject(
+ new Error(
+ `WebSocket heartbeat timeout: no pong within ${this.pongDeadlineMs}ms`
+ )
+ )
+ }, this.pongDeadlineMs)
+ }
+
+ // Arm the deadline only after the first ping is sent, not immediately on
+ // start, so a slow CONNECTING socket cannot time out before any exchange.
+ let deadlineArmed = false
+
+ ws.on('pong', () => {
+ Iif (this.shouldLog()) {
+ core.debug('[Heartbeat] Pong received')
+ }
+ Eif (deadlineArmed) {
+ resetPongTimeout()
+ }
+ })
+
+ ws.on('error', (err: Error) => {
+ core.error(`[Heartbeat] WebSocket error: ${err.message}`)
+ this.stop()
+ })
+
+ ws.on('close', () => {
+ core.debug('[Heartbeat] WebSocket closed, stopping heartbeat')
+ this.stop()
+ })
+
+ // WebSocket readyState: 0 = CONNECTING, 1 = OPEN, 2 = CLOSING, 3 = CLOSED
+ this.pingInterval = setInterval(() => {
+ if (ws.readyState === 0) {
+ // Still connecting — skip this tick but keep the interval alive
+ return
+ }
+ if (ws.readyState === 1) {
+ try {
+ ws.ping()
+ if (!deadlineArmed) {
+ deadlineArmed = true
+ resetPongTimeout()
+ }
+ Iif (this.shouldLog()) {
+ core.debug('[Heartbeat] Ping sent')
+ }
+ } catch (err) {
+ core.error(`[Heartbeat] Ping failed: ${err}`)
+ this.stop()
+ }
+ } else {
+ // CLOSING (2) or CLOSED (3)
+ Iif (this.shouldLog()) {
+ core.debug(
+ `[Heartbeat] WebSocket closing/closed (readyState=${ws.readyState}), stopping heartbeat`
+ )
+ }
+ this.stop()
+ }
+ }, this.pingPeriodMs)
+ }
+}
+ |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| File | ++ | Statements | ++ | Branches | ++ | Functions | ++ | Lines | ++ |
|---|---|---|---|---|---|---|---|---|---|
| heartbeat.ts | +
+
+ |
+ 85.71% | +60/70 | +78.12% | +25/32 | +91.66% | +11/12 | +85.29% | +58/68 | +
| index.ts | +
+
+ |
+ 51.18% | +412/805 | +52.74% | +298/565 | +43.22% | +51/118 | +51.67% | +370/716 | +
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 +33 +34 +35 +36 +37 +38 +39 +40 +41 +42 +43 +44 +45 +46 +47 +48 +49 +50 +51 +52 +53 +54 +55 +56 +57 +58 +59 +60 +61 +62 +63 +64 +65 +66 +67 +68 +69 +70 +71 +72 +73 +74 +75 +76 +77 +78 +79 +80 +81 +82 +83 +84 +85 +86 +87 +88 +89 +90 +91 +92 +93 +94 +95 +96 +97 +98 +99 +100 +101 +102 +103 +104 +105 +106 +107 +108 +109 +110 +111 +112 +113 +114 +115 +116 +117 +118 +119 +120 +121 +122 +123 +124 +125 +126 +127 +128 +129 +130 +131 +132 +133 +134 +135 +136 +137 +138 +139 +140 +141 +142 +143 +144 +145 +146 +147 +148 +149 +150 +151 +152 +153 +154 +155 +156 +157 +158 +159 +160 +161 +162 +163 +164 +165 +166 +167 +168 +169 +170 +171 +172 +173 +174 +175 +176 +177 +178 +179 +180 +181 +182 +183 +184 +185 +186 +187 +188 +189 +190 +191 +192 +193 +194 +195 +196 +197 +198 +199 +200 +201 +202 +203 +204 +205 +206 +207 +208 +209 +210 +211 +212 +213 +214 +215 +216 +217 +218 +219 +220 +221 +222 +223 +224 +225 +226 +227 +228 +229 +230 +231 +232 +233 +234 +235 +236 +237 +238 +239 +240 +241 +242 +243 +244 +245 +246 +247 +248 +249 +250 +251 +252 +253 +254 +255 +256 +257 +258 +259 +260 +261 +262 +263 +264 +265 +266 +267 +268 +269 +270 +271 +272 +273 +274 +275 +276 +277 +278 +279 +280 +281 +282 +283 +284 +285 +286 +287 +288 +289 +290 +291 +292 +293 +294 +295 +296 +297 +298 +299 +300 +301 +302 +303 +304 +305 +306 +307 +308 +309 +310 +311 +312 +313 +314 +315 +316 +317 +318 +319 +320 +321 +322 +323 +324 +325 +326 +327 +328 +329 +330 +331 +332 +333 +334 +335 +336 +337 +338 +339 +340 +341 +342 +343 +344 +345 +346 +347 +348 +349 +350 +351 +352 +353 +354 +355 +356 +357 +358 +359 +360 +361 +362 +363 +364 +365 +366 +367 +368 +369 +370 +371 +372 +373 +374 +375 +376 +377 +378 +379 +380 +381 +382 +383 +384 +385 +386 +387 +388 +389 +390 +391 +392 +393 +394 +395 +396 +397 +398 +399 +400 +401 +402 +403 +404 +405 +406 +407 +408 +409 +410 +411 +412 +413 +414 +415 +416 +417 +418 +419 +420 +421 +422 +423 +424 +425 +426 +427 +428 +429 +430 +431 +432 +433 +434 +435 +436 +437 +438 +439 +440 +441 +442 +443 +444 +445 +446 +447 +448 +449 +450 +451 +452 +453 +454 +455 +456 +457 +458 +459 +460 +461 +462 +463 +464 +465 +466 +467 +468 +469 +470 +471 +472 +473 +474 +475 +476 +477 +478 +479 +480 +481 +482 +483 +484 +485 +486 +487 +488 +489 +490 +491 +492 +493 +494 +495 +496 +497 +498 +499 +500 +501 +502 +503 +504 +505 +506 +507 +508 +509 +510 +511 +512 +513 +514 +515 +516 +517 +518 +519 +520 +521 +522 +523 +524 +525 +526 +527 +528 +529 +530 +531 +532 +533 +534 +535 +536 +537 +538 +539 +540 +541 +542 +543 +544 +545 +546 +547 +548 +549 +550 +551 +552 +553 +554 +555 +556 +557 +558 +559 +560 +561 +562 +563 +564 +565 +566 +567 +568 +569 +570 +571 +572 +573 +574 +575 +576 +577 +578 +579 +580 +581 +582 +583 +584 +585 +586 +587 +588 +589 +590 +591 +592 +593 +594 +595 +596 +597 +598 +599 +600 +601 +602 +603 +604 +605 +606 +607 +608 +609 +610 +611 +612 +613 +614 +615 +616 +617 +618 +619 +620 +621 +622 +623 +624 +625 +626 +627 +628 +629 +630 +631 +632 +633 +634 +635 +636 +637 +638 +639 +640 +641 +642 +643 +644 +645 +646 +647 +648 +649 +650 +651 +652 +653 +654 +655 +656 +657 +658 +659 +660 +661 +662 +663 +664 +665 +666 +667 +668 +669 +670 +671 +672 +673 +674 +675 +676 +677 +678 +679 +680 +681 +682 +683 +684 +685 +686 +687 +688 +689 +690 +691 +692 +693 +694 +695 +696 +697 +698 +699 +700 +701 +702 +703 +704 +705 +706 +707 +708 +709 +710 +711 +712 +713 +714 +715 +716 +717 +718 +719 +720 +721 +722 +723 +724 +725 +726 +727 +728 +729 +730 +731 +732 +733 +734 +735 +736 +737 +738 +739 +740 +741 +742 +743 +744 +745 +746 +747 +748 +749 +750 +751 +752 +753 +754 +755 +756 +757 +758 +759 +760 +761 +762 +763 +764 +765 +766 +767 +768 +769 +770 +771 +772 +773 +774 +775 +776 +777 +778 +779 +780 +781 +782 +783 +784 +785 +786 +787 +788 +789 +790 +791 +792 +793 +794 +795 +796 +797 +798 +799 +800 +801 +802 +803 +804 +805 +806 +807 +808 +809 +810 +811 +812 +813 +814 +815 +816 +817 +818 +819 +820 +821 +822 +823 +824 +825 +826 +827 +828 +829 +830 +831 +832 +833 +834 +835 +836 +837 +838 +839 +840 +841 +842 +843 +844 +845 +846 +847 +848 +849 +850 +851 +852 +853 +854 +855 +856 +857 +858 +859 +860 +861 +862 +863 +864 +865 +866 +867 +868 +869 +870 +871 +872 +873 +874 +875 +876 +877 +878 +879 +880 +881 +882 +883 +884 +885 +886 +887 +888 +889 +890 +891 +892 +893 +894 +895 +896 +897 +898 +899 +900 +901 +902 +903 +904 +905 +906 +907 +908 +909 +910 +911 +912 +913 +914 +915 +916 +917 +918 +919 +920 +921 +922 +923 +924 +925 +926 +927 +928 +929 +930 +931 +932 +933 +934 +935 +936 +937 +938 +939 +940 +941 +942 +943 +944 +945 +946 +947 +948 +949 +950 +951 +952 +953 +954 +955 +956 +957 +958 +959 +960 +961 +962 +963 +964 +965 +966 +967 +968 +969 +970 +971 +972 +973 +974 +975 +976 +977 +978 +979 +980 +981 +982 +983 +984 +985 +986 +987 +988 +989 +990 +991 +992 +993 +994 +995 +996 +997 +998 +999 +1000 +1001 +1002 +1003 +1004 +1005 +1006 +1007 +1008 +1009 +1010 +1011 +1012 +1013 +1014 +1015 +1016 +1017 +1018 +1019 +1020 +1021 +1022 +1023 +1024 +1025 +1026 +1027 +1028 +1029 +1030 +1031 +1032 +1033 +1034 +1035 +1036 +1037 +1038 +1039 +1040 +1041 +1042 +1043 +1044 +1045 +1046 +1047 +1048 +1049 +1050 +1051 +1052 +1053 +1054 +1055 +1056 +1057 +1058 +1059 +1060 +1061 +1062 +1063 +1064 +1065 +1066 +1067 +1068 +1069 +1070 +1071 +1072 +1073 +1074 +1075 +1076 +1077 +1078 +1079 +1080 +1081 +1082 +1083 +1084 +1085 +1086 +1087 +1088 +1089 +1090 +1091 +1092 +1093 +1094 +1095 +1096 +1097 +1098 +1099 +1100 +1101 +1102 +1103 +1104 +1105 +1106 +1107 +1108 +1109 +1110 +1111 +1112 +1113 +1114 +1115 +1116 +1117 +1118 +1119 +1120 +1121 +1122 +1123 +1124 +1125 +1126 +1127 +1128 +1129 +1130 +1131 +1132 +1133 +1134 +1135 +1136 +1137 +1138 +1139 +1140 +1141 +1142 +1143 +1144 +1145 +1146 +1147 +1148 +1149 +1150 +1151 +1152 +1153 +1154 +1155 +1156 +1157 +1158 +1159 +1160 +1161 +1162 +1163 +1164 +1165 +1166 +1167 +1168 +1169 +1170 +1171 +1172 +1173 +1174 +1175 +1176 +1177 +1178 +1179 +1180 +1181 +1182 +1183 +1184 +1185 +1186 +1187 +1188 +1189 +1190 +1191 +1192 +1193 +1194 +1195 +1196 +1197 +1198 +1199 +1200 +1201 +1202 +1203 +1204 +1205 +1206 +1207 +1208 +1209 +1210 +1211 +1212 +1213 +1214 +1215 +1216 +1217 +1218 +1219 +1220 +1221 +1222 +1223 +1224 +1225 +1226 +1227 +1228 +1229 +1230 +1231 +1232 +1233 +1234 +1235 +1236 +1237 +1238 +1239 +1240 +1241 +1242 +1243 +1244 +1245 +1246 +1247 +1248 +1249 +1250 +1251 +1252 +1253 +1254 +1255 +1256 +1257 +1258 +1259 +1260 +1261 +1262 +1263 +1264 +1265 +1266 +1267 +1268 +1269 +1270 +1271 +1272 +1273 +1274 +1275 +1276 +1277 +1278 +1279 +1280 +1281 +1282 +1283 +1284 +1285 +1286 +1287 +1288 +1289 +1290 +1291 +1292 +1293 +1294 +1295 +1296 +1297 +1298 +1299 +1300 +1301 +1302 +1303 +1304 +1305 +1306 +1307 +1308 +1309 +1310 +1311 +1312 +1313 +1314 +1315 +1316 +1317 +1318 +1319 +1320 +1321 +1322 +1323 +1324 +1325 +1326 +1327 +1328 +1329 +1330 +1331 +1332 +1333 +1334 +1335 +1336 +1337 +1338 +1339 +1340 +1341 +1342 +1343 +1344 +1345 +1346 +1347 +1348 +1349 +1350 +1351 +1352 +1353 +1354 +1355 +1356 +1357 +1358 +1359 +1360 +1361 +1362 +1363 +1364 +1365 +1366 +1367 +1368 +1369 +1370 +1371 +1372 +1373 +1374 +1375 +1376 +1377 +1378 +1379 +1380 +1381 +1382 +1383 +1384 +1385 +1386 +1387 +1388 +1389 +1390 +1391 +1392 +1393 +1394 +1395 +1396 +1397 +1398 +1399 +1400 +1401 +1402 +1403 +1404 +1405 +1406 +1407 +1408 +1409 +1410 +1411 +1412 +1413 +1414 +1415 +1416 +1417 +1418 +1419 +1420 +1421 +1422 +1423 +1424 +1425 +1426 +1427 +1428 +1429 +1430 +1431 +1432 +1433 +1434 +1435 +1436 +1437 +1438 +1439 +1440 +1441 +1442 +1443 +1444 +1445 +1446 +1447 +1448 +1449 +1450 +1451 +1452 +1453 +1454 +1455 +1456 +1457 +1458 +1459 +1460 +1461 +1462 +1463 +1464 +1465 +1466 +1467 +1468 +1469 +1470 +1471 +1472 +1473 +1474 +1475 +1476 +1477 +1478 +1479 +1480 +1481 +1482 +1483 +1484 +1485 +1486 +1487 +1488 +1489 +1490 +1491 +1492 +1493 +1494 +1495 +1496 +1497 +1498 +1499 +1500 +1501 +1502 +1503 +1504 +1505 +1506 +1507 +1508 +1509 +1510 +1511 +1512 +1513 +1514 +1515 +1516 +1517 +1518 +1519 +1520 +1521 +1522 +1523 +1524 +1525 +1526 +1527 +1528 +1529 +1530 +1531 +1532 +1533 +1534 +1535 +1536 +1537 +1538 +1539 +1540 +1541 +1542 +1543 +1544 +1545 +1546 +1547 +1548 +1549 +1550 +1551 +1552 +1553 +1554 +1555 +1556 +1557 +1558 +1559 +1560 +1561 +1562 +1563 +1564 +1565 +1566 +1567 +1568 +1569 +1570 +1571 +1572 +1573 +1574 +1575 +1576 +1577 +1578 +1579 +1580 +1581 +1582 +1583 +1584 +1585 +1586 +1587 +1588 +1589 +1590 +1591 +1592 +1593 +1594 +1595 +1596 +1597 +1598 +1599 +1600 +1601 +1602 +1603 +1604 +1605 +1606 +1607 +1608 +1609 +1610 +1611 +1612 +1613 +1614 +1615 +1616 +1617 +1618 +1619 +1620 +1621 +1622 +1623 +1624 +1625 +1626 +1627 +1628 +1629 +1630 +1631 +1632 +1633 +1634 +1635 +1636 +1637 +1638 +1639 +1640 +1641 +1642 +1643 +1644 +1645 +1646 +1647 +1648 +1649 +1650 +1651 +1652 +1653 +1654 +1655 +1656 +1657 +1658 +1659 +1660 +1661 +1662 +1663 +1664 +1665 +1666 +1667 +1668 +1669 +1670 +1671 +1672 +1673 +1674 +1675 +1676 +1677 +1678 +1679 +1680 +1681 +1682 +1683 +1684 +1685 +1686 +1687 +1688 +1689 +1690 +1691 +1692 +1693 +1694 +1695 +1696 +1697 +1698 +1699 +1700 +1701 +1702 +1703 +1704 +1705 +1706 +1707 +1708 +1709 +1710 +1711 +1712 +1713 +1714 +1715 +1716 +1717 +1718 +1719 +1720 +1721 +1722 +1723 +1724 +1725 +1726 +1727 +1728 +1729 +1730 +1731 +1732 +1733 +1734 +1735 +1736 +1737 +1738 +1739 +1740 +1741 +1742 +1743 +1744 +1745 +1746 +1747 +1748 +1749 +1750 +1751 +1752 +1753 +1754 +1755 +1756 +1757 +1758 +1759 +1760 +1761 +1762 +1763 +1764 +1765 +1766 +1767 +1768 +1769 +1770 +1771 +1772 +1773 +1774 +1775 +1776 +1777 +1778 +1779 +1780 +1781 +1782 +1783 +1784 +1785 +1786 +1787 +1788 +1789 +1790 +1791 +1792 +1793 +1794 +1795 +1796 +1797 +1798 +1799 +1800 +1801 +1802 +1803 +1804 +1805 +1806 +1807 +1808 +1809 +1810 +1811 +1812 +1813 +1814 +1815 +1816 +1817 +1818 +1819 +1820 +1821 +1822 +1823 +1824 +1825 +1826 +1827 +1828 +1829 +1830 +1831 +1832 +1833 +1834 +1835 +1836 +1837 +1838 +1839 | 13x +13x +13x +13x +13x +13x +13x +13x +13x + +13x + + + + +13x + + + + + + + + + + + +13x +13x + + +13x + +13x + +13x +13x +13x + +13x + +13x + + + + + + + + + + + + + + + + + + + + + + + + + + +13x + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +13x + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +13x + + + + + + + +13x + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +13x + + + +13x + + +7x +7x + + +7x +7x +36x +36x +36x + + + +7x +7x +35x +35x +35x +70x + + +7x +7x +3x +3x +3x +6x + + +7x +7x + + + +7x + + + + + +7x + +35x +35x +35x +35x + + + + + +7x + +3x +3x +3x +3x + + + + + + + + + +7x +4x +4x + + +7x +7x + + + + + + + + + + +5x +5x +5x +3x + + + + +2x +2x +1x + +1x + + + + + + + + + + +2x +2x +2x +2x +1x + +1x + + + + + +13x + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +13x + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +13x + + + + +3x + +3x +90x + +90x + + +90x + + + + + + + +90x +90x +180x +90x + + + + + + + + + + + + + + + + + + + + +90x + + + +90x +90x +90x +3x + + + +174x + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +13x + + + + +2x +2x + + + + + + + +2x + + + +2x +60x + + +60x +60x +60x +60x + + + + + + + +60x +60x + +120x + + + + +60x +60x +60x + + + + + + + + + + + + + + + + + + + +60x + + + +60x +60x +60x +2x + + + +116x + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +13x +3x +3x + +3x + + + +3x + + + + + +13x + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +13x + + + + + + + + + + + + + + + + + + + + + + + + + + + +13x + + + + + + +13x +21x + + + + + + + + + + + + + + + +13x + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +13x + + + + + + + + + + + + + + + + + + + + + + +13x + + + + + + + + + + + + + +13x + + + + + + + +13x + + + + + +13x +28x +28x +26x + +2x +3x +3x +3x +3x + + +2x + + + + + +13x +23x +23x +21x + +2x +3x +3x +3x +3x + + +2x + + +13x + +24x +24x +21x + +3x +7x +7x +7x +4x + + +3x + + + + + + + + +13x + + +19x +58x + + + + + + + + + + + +13x +17x +17x + + + + + + + + + + + + + + + + + + + + + +9x + +4x + + + + + + + +3x + +1x + +1x + + + +13x +18x +18x +18x + + + +36x +14x +14x +9x +9x +9x +9x + + +18x + + +13x + + + +11x +3x + +8x +2x + +6x +5x + +1x + + + + + +1x + +3x + + +1x + + +13x +21x +21x +21x + + + +42x +17x +17x +11x +11x +11x +11x + + +21x + + + +10x + +4x + + + + + + +1x + + + + + + +4x + + + + + + +1x + + + + + + + + + + + +13x +21x +21x + + + + + +21x + + + +19x + +2x + + + + +2x + + +19x +19x +19x +19x + + + + +3x + + + + +16x + + + +5x + + + + +5x + +11x +1x + +10x +10x +10x +10x +10x +10x + +19x + + + + + + + + +13x + + +9x + +1x + + + + +8x +8x + + +8x +8x + + +8x +8x +1x + + + +8x +1x +1x +1x +1x + + +8x + + +8x +8x + + + +8x +8x +3x +3x + + +2x +1x +1x + + + + +3x + +3x +1x +1x +1x + + + + +8x +2x + + + +8x +8x +3x + + +8x + + +8x + + + + + + + + +8x + + + +6x + +2x + + + + +2x + + +6x +6x +3x + + +3x + + + +3x + +3x +3x +3x +3x + +3x + + + + +3x + + + + + + + + +13x +6x + + + + + + + + +13x + + + + +6x + +6x + +6x + + + +6x + + + + + +6x + + + + + + + +13x + + + +18x + +10x +10x +15x +15x + +15x + + + + +6x + + + + + +12x + +2x + + + + + +4x + +9x +9x +3x + + + + + + + +10x +2x +2x +2x + + + + +2x + + + + + + + +4x +5x +3x +3x + + + + + +2x + + + + + + + + + + + + +13x + +5x + +5x +5x + + + + + + + + + + + + + + +24x + + + + + +13x + + + + + + + +13x +16x +3x + +13x + + + +3x + + + +1x +1x + + + + +13x + + + + + + + + + + + + + + + + + + + + + + + + +13x +21x + + + + + + + + + + + + + + +13x + + + + + + + + + +13x + + + + + + + + + + + + + + + + + + + + +13x + + + + + + + + + + + + + + + + + + + + + +13x +276x +213x + + +63x +63x +63x + + + + + + + + + + + + + + + + + + + + +13x +13x +13x +13x +13x +3x + + + +13x +14x +7x + +7x +7x +2x + +5x +5x + +5x + + + +13x + +13x + + +21x +21x +21x + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +13x + + + + + + +13x + + + + + + +13x +21x + + + + + + | import * as core from '@actions/core'
+import * as fs from 'fs'
+import * as path from 'path'
+import { spawn } from 'child_process'
+import * as k8s from '@kubernetes/client-node'
+import tar from 'tar-fs'
+import * as stream from 'stream'
+import { WritableStreamBuffer } from 'stream-buffers'
+import { createHash } from 'crypto'
+import type { ContainerInfo, Registry } from 'hooklib'
+import {
+ getSecretName,
+ JOB_CONTAINER_NAME,
+ RunnerInstanceLabel
+} from '../hooks/constants'
+import {
+ PodPhase,
+ formatError,
+ mergePodSpecWithOptions,
+ mergeObjectMeta,
+ fixArgs,
+ listDirAllCommand,
+ sleep,
+ EXTERNALS_VOLUME_NAME,
+ GITHUB_VOLUME_NAME,
+ WORK_VOLUME
+} from './utils'
+import * as shlex from 'shlex'
+import { parsePositiveMsEnv, WebSocketHeartbeat } from './heartbeat'
+import type { HeartbeatWebSocket } from './heartbeat'
+
+const kc = new k8s.KubeConfig()
+
+kc.loadFromDefault()
+
+const k8sApi = kc.makeApiClient(k8s.CoreV1Api)
+const k8sBatchV1Api = kc.makeApiClient(k8s.BatchV1Api)
+const k8sAuthorizationV1Api = kc.makeApiClient(k8s.AuthorizationV1Api)
+
+const DEFAULT_WAIT_FOR_POD_TIME_SECONDS = 10 * 60 // 10 min
+
+export const requiredPermissions = [
+ {
+ group: '',
+ verbs: ['get', 'list', 'create', 'delete'],
+ resource: 'pods',
+ subresource: ''
+ },
+ {
+ group: '',
+ verbs: ['get', 'create'],
+ resource: 'pods',
+ subresource: 'exec'
+ },
+ {
+ group: '',
+ verbs: ['get', 'list', 'watch'],
+ resource: 'pods',
+ subresource: 'log'
+ },
+ {
+ group: '',
+ verbs: ['create', 'delete', 'get', 'list'],
+ resource: 'secrets',
+ subresource: ''
+ }
+]
+
+export async function createJobPod(
+ name: string,
+ jobContainer?: k8s.V1Container,
+ services?: k8s.V1Container[],
+ registry?: Registry,
+ extension?: k8s.V1PodTemplateSpec
+): Promise<k8s.V1Pod> {
+ const containers: k8s.V1Container[] = []
+ if (jobContainer) {
+ containers.push(jobContainer)
+ }
+ if (services?.length) {
+ containers.push(...services)
+ }
+
+ const appPod = new k8s.V1Pod()
+
+ appPod.apiVersion = 'v1'
+ appPod.kind = 'Pod'
+
+ appPod.metadata = new k8s.V1ObjectMeta()
+ appPod.metadata.name = name
+
+ const instanceLabel = new RunnerInstanceLabel()
+ appPod.metadata.labels = {
+ [instanceLabel.key]: instanceLabel.value
+ }
+ appPod.metadata.annotations = {}
+
+ appPod.spec = new k8s.V1PodSpec()
+ appPod.spec.containers = containers
+ appPod.spec.securityContext = {
+ fsGroup: 1001
+ }
+
+ // Extract working directory from GITHUB_WORKSPACE
+ // GITHUB_WORKSPACE is like /__w/repo-name/repo-name
+ const githubWorkspace = process.env.GITHUB_WORKSPACE
+ const workingDirPath = githubWorkspace?.split('/').slice(-2).join('/') ?? ''
+
+ const initCommands = [
+ 'mkdir -p /mnt/externals',
+ 'mkdir -p /mnt/work',
+ 'mkdir -p /mnt/github',
+ 'mv /home/runner/externals/* /mnt/externals/'
+ ]
+
+ if (workingDirPath) {
+ initCommands.push(`mkdir -p /mnt/work/${workingDirPath}`)
+ }
+
+ appPod.spec.initContainers = [
+ {
+ name: 'fs-init',
+ image:
+ process.env.ACTIONS_RUNNER_IMAGE ||
+ 'ghcr.io/actions/actions-runner:latest',
+ command: ['sh', '-c', initCommands.join(' && ')],
+ securityContext: {
+ runAsGroup: 1001,
+ runAsUser: 1001
+ },
+ volumeMounts: [
+ {
+ name: EXTERNALS_VOLUME_NAME,
+ mountPath: '/mnt/externals'
+ },
+ {
+ name: WORK_VOLUME,
+ mountPath: '/mnt/work'
+ },
+ {
+ name: GITHUB_VOLUME_NAME,
+ mountPath: '/mnt/github'
+ }
+ ]
+ }
+ ]
+
+ appPod.spec.restartPolicy = 'Never'
+
+ appPod.spec.volumes = [
+ {
+ name: EXTERNALS_VOLUME_NAME,
+ emptyDir: {}
+ },
+ {
+ name: GITHUB_VOLUME_NAME,
+ emptyDir: {}
+ },
+ {
+ name: WORK_VOLUME,
+ emptyDir: {}
+ }
+ ]
+
+ if (registry) {
+ const secret = await createDockerSecret(registry)
+ if (!secret?.metadata?.name) {
+ throw new Error(`created secret does not have secret.metadata.name`)
+ }
+ const secretReference = new k8s.V1LocalObjectReference()
+ secretReference.name = secret.metadata.name
+ appPod.spec.imagePullSecrets = [secretReference]
+ }
+
+ if (extension?.metadata) {
+ mergeObjectMeta(appPod, extension.metadata)
+ }
+
+ if (extension?.spec) {
+ mergePodSpecWithOptions(appPod.spec, extension.spec)
+ }
+
+ return await k8sApi.createNamespacedPod({
+ namespace: namespace(),
+ body: appPod
+ })
+}
+
+export async function createContainerStepPod(
+ name: string,
+ container: k8s.V1Container,
+ extension?: k8s.V1PodTemplateSpec
+): Promise<k8s.V1Pod> {
+ const appPod = new k8s.V1Pod()
+
+ appPod.apiVersion = 'v1'
+ appPod.kind = 'Pod'
+
+ appPod.metadata = new k8s.V1ObjectMeta()
+ appPod.metadata.name = name
+
+ const instanceLabel = new RunnerInstanceLabel()
+ appPod.metadata.labels = {
+ [instanceLabel.key]: instanceLabel.value
+ }
+ appPod.metadata.annotations = {}
+
+ appPod.spec = new k8s.V1PodSpec()
+ appPod.spec.containers = [container]
+
+ appPod.spec.restartPolicy = 'Never'
+
+ appPod.spec.volumes = [
+ {
+ name: EXTERNALS_VOLUME_NAME,
+ emptyDir: {}
+ },
+ {
+ name: GITHUB_VOLUME_NAME,
+ emptyDir: {}
+ },
+ {
+ name: WORK_VOLUME,
+ emptyDir: {}
+ }
+ ]
+
+ if (extension?.metadata) {
+ mergeObjectMeta(appPod, extension.metadata)
+ }
+
+ if (extension?.spec) {
+ mergePodSpecWithOptions(appPod.spec, extension.spec)
+ }
+
+ return await k8sApi.createNamespacedPod({
+ namespace: namespace(),
+ body: appPod
+ })
+}
+
+export async function deletePod(name: string): Promise<void> {
+ await k8sApi.deleteNamespacedPod({
+ name,
+ namespace: namespace(),
+ gracePeriodSeconds: 0
+ })
+}
+
+export async function execPodStep(
+ command: string[],
+ podName: string,
+ containerName: string,
+ stdin?: stream.Readable
+): Promise<number> {
+ const exec = new k8s.Exec(kc)
+ core.debug(
+ `[execPodStep] Starting: cmd="${command[0]}" (${command.length} args), pod=${podName}, container=${containerName}`
+ )
+
+ command = fixArgs(command)
+
+ const DEFAULT_PING_PERIOD_MS = 5000
+ const pingPeriodMs = parsePositiveMsEnv(
+ process.env.ACTIONS_RUNNER_HEARTBEAT_PERIOD_MS,
+ DEFAULT_PING_PERIOD_MS
+ )
+ const pongDeadlineMs = parsePositiveMsEnv(
+ process.env.ACTIONS_RUNNER_HEARTBEAT_DEADLINE_MS,
+ pingPeriodMs * 12 + 1000
+ )
+ core.debug(
+ `[execPodStep] Heartbeat config: pingPeriodMs=${pingPeriodMs}, pongDeadlineMs=${pongDeadlineMs}`
+ )
+
+ const heartbeat = new WebSocketHeartbeat(pingPeriodMs, pongDeadlineMs)
+
+ return new Promise<number>((resolve, reject) => {
+ core.debug('[execPodStep] About to call exec.exec')
+ let ws: HeartbeatWebSocket | null = null
+
+ exec
+ .exec(
+ namespace(),
+ podName,
+ containerName,
+ command,
+ process.stdout,
+ process.stderr,
+ stdin ?? null,
+ false /* tty */,
+ async resp => {
+ core.debug(
+ `[execPodStep] execPodStep response: ${JSON.stringify(resp)}`
+ )
+
+ heartbeat.stop()
+
+ // Close WebSocket and wait for it before resolving/rejecting
+ const closeWebSocket = async (): Promise<void> => {
+ const socket = ws
+ if (
+ socket &&
+ (socket.readyState === 1 || socket.readyState === 0)
+ ) {
+ return new Promise<void>(closeResolve => {
+ const closeTimeout = setTimeout(() => {
+ core.warning(
+ '[execPodStep] WebSocket close timeout, forcing cleanup'
+ )
+ closeResolve()
+ }, 5000)
+
+ socket.once('close', () => {
+ clearTimeout(closeTimeout)
+ core.debug('[execPodStep] WebSocket closed cleanly')
+ closeResolve()
+ })
+ socket.close()
+ })
+ }
+ }
+
+ if (resp.status === 'Success') {
+ core.debug(`[execPodStep] Success, code: ${resp.code}`)
+ await closeWebSocket()
+ resolve(resp.code || 0)
+ } else {
+ core.debug(
+ `[execPodStep] Failure: ${JSON.stringify({ message: resp?.message, details: resp?.details })}`
+ )
+ await closeWebSocket()
+ reject(new Error(resp?.message || 'execPodStep failed'))
+ }
+ }
+ )
+ .then(websocket => {
+ core.debug('[execPodStep] exec.exec resolved, ws object received')
+ ws = websocket
+ if (ws) {
+ heartbeat.start(ws, reject)
+ } else {
+ core.warning('[Heartbeat] WebSocket is null, heartbeat not started')
+ }
+ })
+ .catch(async e => {
+ heartbeat.stop()
+ core.error(`[execPodStep] exec.exec threw error: ${e}`)
+
+ // Close WebSocket before rejecting with timeout protection
+ const socket = ws
+ if (socket && (socket.readyState === 1 || socket.readyState === 0)) {
+ await new Promise<void>(closeResolve => {
+ const closeTimeout = setTimeout(() => {
+ core.warning(
+ '[execPodStep] WebSocket close timeout in error handler'
+ )
+ closeResolve()
+ }, 5000)
+
+ socket.once('close', () => {
+ clearTimeout(closeTimeout)
+ closeResolve()
+ })
+ socket.close()
+ })
+ }
+
+ reject(e)
+ })
+ })
+}
+
+// Variant of execPodStep that also captures the last `tailLines` lines of
+// combined stdout/stderr so callers can surface the script's own output on
+// failure. Returns the exit code plus the tail (empty string on success or if
+// nothing was emitted). The original streams are still tee'd to
+// process.stdout/process.stderr so GitHub Actions logs are unchanged.
+export async function execPodStepWithOutput(
+ command: string[],
+ podName: string,
+ containerName: string,
+ tailLines = 20,
+ stdin?: stream.Readable
+): Promise<{ code: number; output: string }> {
+ const exec = new k8s.Exec(kc)
+ command = fixArgs(command)
+
+ // Ring buffer of the last N non-empty lines (cap to keep memory bounded).
+ const buffer: string[] = []
+ const push = (line: string): void => {
+ Iif (line.length === 0) return
+ buffer.push(line)
+ if (buffer.length > tailLines) buffer.shift()
+ }
+
+ // Separate pending buffers per stream to prevent stdout/stderr interleaving.
+ let pendingOut = ''
+ const ingestOut = (chunk: Buffer | string): void => {
+ pendingOut += chunk.toString('utf8')
+ const lines = pendingOut.split(/\r?\n/)
+ pendingOut = lines.pop() ?? ''
+ for (const line of lines) push(line)
+ }
+
+ let pendingErr = ''
+ const ingestErr = (chunk: Buffer | string): void => {
+ pendingErr += chunk.toString('utf8')
+ const lines = pendingErr.split(/\r?\n/)
+ pendingErr = lines.pop() ?? ''
+ for (const line of lines) push(line)
+ }
+
+ const flushPending = (): void => {
+ Iif (pendingOut) {
+ push(pendingOut)
+ pendingOut = ''
+ }
+ Iif (pendingErr) {
+ push(pendingErr)
+ pendingErr = ''
+ }
+ }
+
+ const capture = new stream.Writable({
+ write(chunk: Buffer | string, _enc, cb) {
+ try {
+ ingestOut(chunk)
+ process.stdout.write(chunk)
+ cb()
+ } catch (e) {
+ cb(e as Error)
+ }
+ }
+ })
+ const captureErr = new stream.Writable({
+ write(chunk: Buffer | string, _enc, cb) {
+ try {
+ ingestErr(chunk)
+ process.stderr.write(chunk)
+ cb()
+ } catch (e) {
+ cb(e as Error)
+ }
+ }
+ })
+
+ // Parse an exit code from a k8s exec error/status message.
+ // Handles both "command terminated with exit code N" and
+ // "command terminated with non-zero exit code: command terminated with exit code N".
+ const parseExitCode = (msg: string | undefined): number | null => {
+ const m = msg?.match(/exit code[:\s]+(\d+)/i)
+ return m ? parseInt(m[1], 10) : null
+ }
+
+ return await new Promise(function (resolve, reject) {
+ exec
+ .exec(
+ namespace(),
+ podName,
+ containerName,
+ command,
+ capture,
+ captureErr,
+ stdin ?? null,
+ false /* tty */,
+ resp => {
+ core.debug(`execPodStepWithOutput response: ${JSON.stringify(resp)}`)
+ flushPending()
+ if (resp.status === 'Success') {
+ resolve({ code: resp.code || 0, output: buffer.join('\n') })
+ } else {
+ // k8s exec returns status='Failure' for non-zero script exit.
+ // resp.code may be undefined depending on k8s version; fall back
+ // to parsing the exit code from the message string.
+ const code = parseExitCode(resp?.message)
+ if (code !== null) {
+ resolve({ code, output: buffer.join('\n') })
+ } else {
+ reject(new Error(resp?.message || 'execPodStepWithOutput failed'))
+ }
+ }
+ }
+ )
+ .catch(e => {
+ // In @kubernetes/client-node v1.x the promise returned by exec()
+ // itself rejects for non-zero exits (the status callback may or may
+ // not fire first). Detect this case via the error message so the
+ // caller can classify the error instead of treating it as a hook
+ // failure. Genuine failures (connection drop, etc.) still reject.
+ flushPending()
+ const errMsg = e instanceof Error ? e.message : String(e)
+ const code = parseExitCode(errMsg)
+ if (code !== null) {
+ resolve({ code, output: buffer.join('\n') })
+ } else {
+ reject(e)
+ }
+ })
+ })
+}
+
+export async function execCalculateOutputHashSorted(
+ podName: string,
+ containerName: string,
+ command: string[]
+): Promise<{ hash: string; lines: string[] }> {
+ const exec = new k8s.Exec(kc)
+
+ let output = ''
+ const outputWriter = new stream.Writable({
+ write(chunk, _enc, cb) {
+ try {
+ output += chunk.toString('utf8')
+ cb()
+ } catch (e) {
+ cb(e as Error)
+ }
+ }
+ })
+
+ await new Promise<void>((resolve, reject) => {
+ exec
+ .exec(
+ namespace(),
+ podName,
+ containerName,
+ command,
+ outputWriter, // capture stdout
+ process.stderr,
+ null,
+ false /* tty */,
+ resp => {
+ core.debug(`internalExecOutput response: ${JSON.stringify(resp)}`)
+ if (resp.status === 'Success') {
+ resolve()
+ } else {
+ core.debug(
+ JSON.stringify({
+ message: resp?.message,
+ details: resp?.details
+ })
+ )
+ reject(new Error(resp?.message || 'internalExecOutput failed'))
+ }
+ }
+ )
+ .catch(e => reject(e))
+ })
+
+ outputWriter.end()
+
+ // Sort lines for consistent ordering across platforms
+ const lines = output
+ .split('\n')
+ .filter(line => line.length > 0)
+ .sort()
+ const sortedOutput = lines.join('\n') + '\n'
+
+ const hash = createHash('sha256')
+ hash.update(sortedOutput)
+ return { hash: hash.digest('hex'), lines }
+}
+
+export async function localCalculateOutputHashSorted(
+ commands: string[]
+): Promise<{ hash: string; lines: string[] }> {
+ return await new Promise<{ hash: string; lines: string[] }>(
+ (resolve, reject) => {
+ const child = spawn(commands[0], commands.slice(1), {
+ stdio: ['ignore', 'pipe', 'ignore']
+ })
+
+ let output = ''
+ child.stdout.on('data', chunk => {
+ output += chunk.toString('utf8')
+ })
+ child.on('error', reject)
+ child.on('close', (code: number) => {
+ if (code === 0) {
+ // Sort lines for consistent ordering across distributions/platforms
+ const lines = output
+ .split('\n')
+ .filter(line => line.length > 0)
+ .sort()
+ const sortedOutput = lines.join('\n') + '\n'
+
+ const hash = createHash('sha256')
+ hash.update(sortedOutput)
+ resolve({ hash: hash.digest('hex'), lines })
+ } else {
+ reject(new Error(`child process exited with code ${code}`))
+ }
+ })
+ }
+ )
+}
+
+export async function execCpToPod(
+ podName: string,
+ runnerPath: string,
+ containerPath: string
+): Promise<void> {
+ core.debug(`Copying ${runnerPath} to pod ${podName} at ${containerPath}`)
+
+ let attempt = 0
+ while (Itrue) {
+ try {
+ const exec = new k8s.Exec(kc)
+ // Use tar to extract with --no-same-owner to avoid ownership issues.
+ // Then use find to fix permissions. The -m flag helps but we also need to fix permissions after.
+ const command = [
+ 'sh',
+ '-c',
+ `tar xf - --no-same-owner -C ${shlex.quote(containerPath)} 2>/dev/null; ` +
+ `find ${shlex.quote(containerPath)} -type f -exec chmod u+rw {} \\; 2>/dev/null; ` +
+ `find ${shlex.quote(containerPath)} -type d -exec chmod u+rwx {} \\; 2>/dev/null; ` +
+ `sync 2>/dev/null || true`
+ ]
+ const readStream = tar.pack(runnerPath)
+ const errStream = new WritableStreamBuffer()
+ await new Promise((resolve, reject) => {
+ exec
+ .exec(
+ namespace(),
+ podName,
+ JOB_CONTAINER_NAME,
+ command,
+ null,
+ errStream,
+ readStream,
+ false,
+ async status => {
+ if (errStream.size()) {
+ return reject(
+ new Error(
+ `Error from execCpToPod - status: ${status.status}, details: \n ${errStream.getContentsAsString()}`
+ )
+ )
+ }
+ resolve(status)
+ }
+ )
+ .catch(e => reject(e))
+ })
+ break
+ } catch (error) {
+ core.debug(`cpToPod: Attempt ${attempt + 1} failed: ${error}`)
+ attempt++
+ if (attempt >= 30) {
+ throw new Error(
+ `cpToPod failed after ${attempt} attempts: ${formatError(error)}`
+ )
+ }
+ await sleep(1000)
+ }
+ }
+
+ let attempts = 15
+ const delay = 1000
+ for (let i = 0; i < attempts; i++) {
+ try {
+ const { hash: want, lines: wantLines } =
+ await localCalculateOutputHashSorted([
+ 'sh',
+ '-c',
+ listDirAllCommand(runnerPath)
+ ])
+
+ const { hash: got, lines: gotLines } =
+ await execCalculateOutputHashSorted(podName, JOB_CONTAINER_NAME, [
+ 'sh',
+ '-c',
+ listDirAllCommand(containerPath)
+ ])
+
+ if (got !== want) {
+ core.debug(
+ `[cpToPod hash mismatch attempt ${i + 1}/${attempts}] want='${want}' got='${got}'`
+ )
+ core.debug(
+ `[cpToPod] runner file count=${wantLines.length} pod file count=${gotLines.length}`
+ )
+ const wantMap = new Map(
+ wantLines.map(l => [l.replace(/^\d+ /, ''), l.split(' ')[0]])
+ )
+ const gotMap = new Map(
+ gotLines.map(l => [l.replace(/^\d+ /, ''), l.split(' ')[0]])
+ )
+ const onlyInWant = wantLines.filter(
+ l => !gotMap.has(l.replace(/^\d+ /, ''))
+ )
+ const onlyInGot = gotLines.filter(
+ l => !wantMap.has(l.replace(/^\d+ /, ''))
+ )
+ const sizeDiff = wantLines
+ .filter(l => {
+ const name = l.replace(/^\d+ /, '')
+ return gotMap.has(name) && gotMap.get(name) !== wantMap.get(name)
+ })
+ .map(l => {
+ const name = l.replace(/^\d+ /, '')
+ return `${name}: runner=${wantMap.get(name)} pod=${gotMap.get(name)}`
+ })
+ core.debug(`[cpToPod] only in runner: ${JSON.stringify(onlyInWant)}`)
+ core.debug(`[cpToPod] only in pod: ${JSON.stringify(onlyInGot)}`)
+ core.debug(`[cpToPod] size mismatch: ${JSON.stringify(sizeDiff)}`)
+ await sleep(delay)
+ continue
+ }
+
+ break
+ } catch (error) {
+ core.debug(`Attempt ${i + 1} failed: ${error}`)
+ await sleep(delay)
+ }
+ }
+}
+
+export async function execCpFromPod(
+ podName: string,
+ containerPath: string,
+ parentRunnerPath: string
+): Promise<void> {
+ const targetRunnerPath = `${parentRunnerPath}/${path.basename(containerPath)}`
+ core.debug(
+ `Copying from pod ${podName} ${containerPath} to ${targetRunnerPath}`
+ )
+
+ // Clear target before extracting so deleted-on-pod files don't linger locally.
+ // tar.extract() appends into the destination; without this, stale files (e.g.
+ // git-credentials removed by checkout cleanup inside the pod) cause a permanent
+ // hash mismatch that no amount of retrying can resolve.
+ Iif (fs.existsSync(targetRunnerPath)) {
+ fs.rmSync(targetRunnerPath, { recursive: true, force: true })
+ }
+
+ let attempt = 0
+ while (Itrue) {
+ try {
+ // make temporary directory
+ const exec = new k8s.Exec(kc)
+ const containerPaths = containerPath.split('/')
+ const dirname = containerPaths.pop() as string
+ const command = [
+ 'tar',
+ 'cf',
+ '-',
+ '-C',
+ containerPaths.join('/') || '/',
+ dirname
+ ]
+ const writerStream = tar.extract(parentRunnerPath)
+ const errStream = new WritableStreamBuffer()
+
+ await new Promise((resolve, reject) => {
+ // Resolve only after writerStream finishes flushing to disk.
+ // The k8s status callback fires when the pod-side tar process exits,
+ // but tar-fs may still be writing buffered data to the local filesystem.
+ // Waiting for 'finish' ensures all files are on disk before hash check.
+ writerStream.on('finish', resolve)
+ writerStream.on('error', reject)
+ exec
+ .exec(
+ namespace(),
+ podName,
+ JOB_CONTAINER_NAME,
+ command,
+ writerStream,
+ errStream,
+ null,
+ false,
+ async (_s: k8s.V1Status) => {
+ if (errStream.size()) {
+ return reject(
+ new Error(
+ `Error from cpFromPod - details: \n ${errStream.getContentsAsString()}`
+ )
+ )
+ }
+ }
+ )
+ .catch(e => reject(e))
+ })
+ break
+ } catch (error) {
+ core.debug(`Attempt ${attempt + 1} failed: ${error}`)
+ attempt++
+ if (attempt >= 30) {
+ throw new Error(
+ `execCpFromPod failed after ${attempt} attempts: ${formatError(error)}`
+ )
+ }
+ await sleep(1000)
+ }
+ }
+
+ let attempts = 15
+ const delay = 1000
+ for (let i = 0; i < attempts; i++) {
+ try {
+ const { hash: want, lines: wantLines } =
+ await execCalculateOutputHashSorted(podName, JOB_CONTAINER_NAME, [
+ 'sh',
+ '-c',
+ listDirAllCommand(containerPath)
+ ])
+
+ const { hash: got, lines: gotLines } =
+ await localCalculateOutputHashSorted([
+ 'sh',
+ '-c',
+ listDirAllCommand(targetRunnerPath)
+ ])
+
+ if (got !== want) {
+ core.debug(
+ `[cpFromPod hash mismatch attempt ${i + 1}/${attempts}] want='${want}' got='${got}'`
+ )
+ core.debug(
+ `[cpFromPod] pod file count=${wantLines.length} runner file count=${gotLines.length}`
+ )
+ const wantMap = new Map(
+ wantLines.map(l => [l.replace(/^\d+ /, ''), l.split(' ')[0]])
+ )
+ const gotMap = new Map(
+ gotLines.map(l => [l.replace(/^\d+ /, ''), l.split(' ')[0]])
+ )
+ const onlyInWant = wantLines.filter(
+ l => !gotMap.has(l.replace(/^\d+ /, ''))
+ )
+ const onlyInGot = gotLines.filter(
+ l => !wantMap.has(l.replace(/^\d+ /, ''))
+ )
+ const sizeDiff = wantLines
+ .filter(l => {
+ const name = l.replace(/^\d+ /, '')
+ return gotMap.has(name) && gotMap.get(name) !== wantMap.get(name)
+ })
+ .map(l => {
+ const name = l.replace(/^\d+ /, '')
+ return `${name}: pod=${wantMap.get(name)} runner=${gotMap.get(name)}`
+ })
+ core.debug(`[cpFromPod] only in pod: ${JSON.stringify(onlyInWant)}`)
+ core.debug(`[cpFromPod] only in runner: ${JSON.stringify(onlyInGot)}`)
+ core.debug(`[cpFromPod] size mismatch: ${JSON.stringify(sizeDiff)}`)
+ await sleep(delay)
+ continue
+ }
+
+ break
+ } catch (error) {
+ core.debug(`Attempt ${i + 1} failed: ${error}`)
+ await sleep(delay)
+ }
+ }
+}
+
+export async function waitForJobToComplete(jobName: string): Promise<void> {
+ const backOffManager = new BackOffManager()
+ while (Itrue) {
+ try {
+ if (await isJobSucceeded(jobName)) {
+ return
+ }
+ } catch (error) {
+ throw new Error(`job ${jobName} has failed: ${formatError(error)}`)
+ }
+ await backOffManager.backOff()
+ }
+}
+
+export async function createDockerSecret(
+ registry: Registry
+): Promise<k8s.V1Secret> {
+ const authContent = {
+ auths: {
+ [registry.serverUrl || 'https://index.docker.io/v1/']: {
+ username: registry.username,
+ password: registry.password,
+ auth: Buffer.from(`${registry.username}:${registry.password}`).toString(
+ 'base64'
+ )
+ }
+ }
+ }
+
+ const runnerInstanceLabel = new RunnerInstanceLabel()
+
+ const secretName = getSecretName()
+ const secret = new k8s.V1Secret()
+ secret.immutable = true
+ secret.apiVersion = 'v1'
+ secret.metadata = new k8s.V1ObjectMeta()
+ secret.metadata.name = secretName
+ secret.metadata.namespace = namespace()
+ secret.metadata.labels = {
+ [runnerInstanceLabel.key]: runnerInstanceLabel.value
+ }
+ secret.type = 'kubernetes.io/dockerconfigjson'
+ secret.kind = 'Secret'
+ secret.data = {
+ '.dockerconfigjson': Buffer.from(JSON.stringify(authContent)).toString(
+ 'base64'
+ )
+ }
+
+ return await k8sApi.createNamespacedSecret({
+ namespace: namespace(),
+ body: secret
+ })
+}
+
+export async function createSecretForEnvs(envs: {
+ [key: string]: string
+}): Promise<string> {
+ const runnerInstanceLabel = new RunnerInstanceLabel()
+
+ const secret = new k8s.V1Secret()
+ const secretName = getSecretName()
+ secret.immutable = true
+ secret.apiVersion = 'v1'
+ secret.metadata = new k8s.V1ObjectMeta()
+ secret.metadata.name = secretName
+
+ secret.metadata.labels = {
+ [runnerInstanceLabel.key]: runnerInstanceLabel.value
+ }
+ secret.kind = 'Secret'
+ secret.data = {}
+ for (const [key, value] of Object.entries(envs)) {
+ secret.data[key] = Buffer.from(value).toString('base64')
+ }
+
+ await k8sApi.createNamespacedSecret({
+ namespace: namespace(),
+ body: secret
+ })
+ return secretName
+}
+
+export async function deleteSecret(name: string): Promise<void> {
+ await k8sApi.deleteNamespacedSecret({
+ name,
+ namespace: namespace()
+ })
+}
+
+export async function pruneSecrets(): Promise<void> {
+ const secretList = await k8sApi.listNamespacedSecret({
+ namespace: namespace(),
+ labelSelector: new RunnerInstanceLabel().toString()
+ })
+ if (!secretList.items.length) {
+ return
+ }
+
+ await Promise.all(
+ secretList.items.map(
+ async secret =>
+ secret.metadata?.name && (await deleteSecret(secret.metadata.name))
+ )
+ )
+}
+
+export const UNRECOVERABLE_WAITING_REASONS = new Set([
+ // k8s has already retried image pull multiple times with exponential backoff.
+ // ErrImagePull is excluded: it fires on the first failure (could be a transient
+ // TLS timeout or network blip) and k8s will naturally promote it to
+ // ImagePullBackOff after a moment. Fast-failing on ErrImagePull would kill
+ // jobs that would have succeeded on the next pull attempt.
+ 'ImagePullBackOff',
+ // Image name is syntactically invalid — cannot self-heal without a config fix.
+ 'InvalidImageName',
+ // Container spec is invalid (bad env vars, resource limits, securityContext) —
+ // cannot self-heal without a config fix.
+ 'CreateContainerConfigError'
+ // CreateContainerError is excluded: it is sometimes emitted transiently by the
+ // container runtime (e.g. during a node-level runtime restart). It can
+ // self-resolve on the next kubelet retry cycle.
+])
+
+// Pod *event* reasons (from the event stream, not container status) that
+// indicate a permanent failure the pod will never recover from on its own.
+// These surface as Warning events on the pod (visible via `kubectl describe`)
+// rather than in container.status.state.waiting.reason, so they need a separate
+// list + a separate (async) detection path.
+//
+// - FailedMount: a volume cannot be mounted (e.g. hostPath `type: Directory`
+// pointing at a path that does not exist on the node, missing PVC, missing
+// Secret/ConfigMap volume). The container stays in `ContainerCreating`
+// waiting state, which is NOT in UNRECOVERABLE_WAITING_REASONS, so without
+// this check the hook polls until the 3600s timeout.
+// - FailedBinding: a PVC could not be bound (no matching PV, storage class
+// misconfiguration). Usually paired with FailedMount once the pod retries.
+//
+// FailedScheduling is included but gated by PERMANENT_SCHEDULING_PATTERNS:
+// only messages that positively identify a permanent configuration error
+// trigger fast-fail. Resource shortages ("Insufficient cpu/memory/gpu")
+// are silently skipped so the pod keeps queuing until the timeout.
+export const UNRECOVERABLE_EVENT_REASONS = new Set([
+ 'FailedMount',
+ 'FailedBinding',
+ 'FailedScheduling'
+])
+
+// Patterns that POSITIVELY IDENTIFY a permanent, unrecoverable scheduling
+// failure in a FailedScheduling event message. Fast-fail fires ONLY when
+// one of these matches. Anything else (resource shortages, unknown format,
+// absent message) is treated as transient — the pod keeps queuing.
+//
+// Permanent examples (WILL fast-fail):
+// "0/3 nodes: 3 node(s) didn't match Pod's node affinity/selector."
+// "0/3 nodes: 3 node(s) had untolerated taint {gpu: true}."
+// "0/3 nodes: persistentvolumeclaim "my-pvc" not found."
+//
+// Transient examples (will NOT fast-fail, keep queuing):
+// "0/3 nodes: 3 Insufficient nvidia.com/gpu."
+// "0/3 nodes: 3 Insufficient memory."
+// "0/1 nodes are available" ← ambiguous, unknown
+//
+// Extend at runtime via env var ACTIONS_RUNNER_K8S_PERMANENT_SCHEDULING_PATTERNS
+// (comma-separated regex strings; cannot remove built-in entries).
+export const PERMANENT_SCHEDULING_PATTERNS: readonly RegExp[] = [
+ // Node selector / affinity label mismatch — no node carries the required
+ // labels. Will not self-resolve without a pod-spec or node-label change.
+ /node\(s\) didn't match Pod's node affinity\/selector/i,
+ /node\(s\) didn't match node affinity/i,
+ // Untolerated taint — every node carries a taint the pod does not tolerate.
+ // Will not self-resolve without adding a toleration or removing the taint.
+ /node\(s\) had untolerated taint/i,
+ // PVC referenced in the pod spec does not exist in the namespace. The
+ // scheduler refuses to place the pod until the PVC is created. The workflow
+ // job spec is wrong — it won't self-resolve.
+ /persistentvolumeclaim "[^"]+" not found/i
+]
+
+export const UNRECOVERABLE_TERMINATED_REASONS = new Set([
+ 'OOMKilled',
+ 'Error',
+ 'FailedPostStartHookError'
+])
+
+// Maximum number of Warning events to include in a pod's failure description so
+// that the GitHub Actions log is not flooded.
+const MAX_DIAGNOSTIC_EVENTS = 10
+
+// The fast-fail whitelist can be extended (not narrowed) from the environment so
+// operators can add deterministic terminal reasons without a code change. This
+// reuses the same env-var configuration pattern as the prepare-job timeout. When
+// the variable is unset, behaviour is identical to the built-in defaults.
+export function getUnrecoverableWaitingReasons(): Set<string> {
+ const extra = process.env['ACTIONS_RUNNER_K8S_UNRECOVERABLE_WAITING_REASONS']
+ if (!extra) {
+ return UNRECOVERABLE_WAITING_REASONS
+ }
+ const reasons = new Set(UNRECOVERABLE_WAITING_REASONS)
+ for (const reason of extra.split(',')) {
+ const trimmed = reason.trim()
+ Eif (trimmed) {
+ reasons.add(trimmed)
+ }
+ }
+ return reasons
+}
+
+// Mirrors getUnrecoverableWaitingReasons() but for pod *event* reasons. The
+// env var ACTIONS_RUNNER_K8S_UNRECOVERABLE_EVENT_REASONS adds to (never removes
+// from) the built-in UNRECOVERABLE_EVENT_REASONS set.
+export function getUnrecoverableEventReasons(): Set<string> {
+ const extra = process.env['ACTIONS_RUNNER_K8S_UNRECOVERABLE_EVENT_REASONS']
+ if (!extra) {
+ return UNRECOVERABLE_EVENT_REASONS
+ }
+ const reasons = new Set(UNRECOVERABLE_EVENT_REASONS)
+ for (const reason of extra.split(',')) {
+ const trimmed = reason.trim()
+ Eif (trimmed) {
+ reasons.add(trimmed)
+ }
+ }
+ return reasons
+}
+
+export function getUnrecoverableTerminatedReasons(): Set<string> {
+ const extra =
+ process.env['ACTIONS_RUNNER_K8S_UNRECOVERABLE_TERMINATED_REASONS']
+ if (!extra) {
+ return UNRECOVERABLE_TERMINATED_REASONS
+ }
+ const reasons = new Set(UNRECOVERABLE_TERMINATED_REASONS)
+ for (const reason of extra.split(',')) {
+ const trimmed = reason.trim()
+ if (trimmed) {
+ reasons.add(trimmed)
+ }
+ }
+ return reasons
+}
+
+// Returns true when a FailedScheduling event message positively identifies a
+// permanent configuration error that will not self-resolve. Returns false
+// (treat as transient, keep queuing) for:
+// - absent/empty messages → unknown, assume transient
+// - resource shortages → Insufficient cpu/memory/gpu/etc.
+// - any unrecognised format → unknown, assume transient
+export function isPermanentSchedulingFailure(
+ message: string | undefined
+): boolean {
+ if (!message) return false
+ return getPermanentSchedulingPatterns().some(p => p.test(message))
+}
+
+// Returns PERMANENT_SCHEDULING_PATTERNS extended by any extra patterns from
+// ACTIONS_RUNNER_K8S_PERMANENT_SCHEDULING_PATTERNS (comma-separated regexes).
+// Invalid regex strings are skipped with a warning.
+// The compiled result is cached: env var changes during a single process are
+// rare (effectively never in a runner job), but this avoids repeated split +
+// RegExp construction on every FailedScheduling poll iteration.
+let _cachedSchedulingPatterns: readonly RegExp[] | undefined
+let _lastSchedulingPatternsEnv: string | undefined
+
+export function getPermanentSchedulingPatterns(): readonly RegExp[] {
+ const extra = process.env['ACTIONS_RUNNER_K8S_PERMANENT_SCHEDULING_PATTERNS']
+ Eif (!extra) return PERMANENT_SCHEDULING_PATTERNS
+ if (_cachedSchedulingPatterns && _lastSchedulingPatternsEnv === extra) {
+ return _cachedSchedulingPatterns
+ }
+ const patterns: RegExp[] = [...PERMANENT_SCHEDULING_PATTERNS]
+ for (const raw of extra.split(',')) {
+ const trimmed = raw.trim()
+ if (!trimmed) continue
+ try {
+ patterns.push(new RegExp(trimmed, 'i'))
+ } catch {
+ core.warning(
+ `ACTIONS_RUNNER_K8S_PERMANENT_SCHEDULING_PATTERNS: invalid regex "${trimmed}", skipped`
+ )
+ }
+ }
+ _lastSchedulingPatternsEnv = extra
+ _cachedSchedulingPatterns = patterns
+ return patterns
+}
+
+function getWaitingReasonHint(reason: string): string {
+ switch (reason) {
+ case 'ImagePullBackOff':
+ return [
+ ` → Image pull has failed repeatedly (k8s retried with exponential backoff). Check:`,
+ ` - Image name and tag are correct and exist in the registry`,
+ ` - If private registry: imagePullSecret is configured and credentials are valid`,
+ ` - Network connectivity from the node to the registry (DNS, firewall, proxy, TLS)`,
+ ` Run: kubectl describe pod <pod> | grep -A10 "Events"`
+ ].join('\n')
+ case 'InvalidImageName':
+ return ` → Image name is malformed. Check the workflow/job container image configuration.`
+ case 'CreateContainerConfigError':
+ return ` → Container config is invalid. Check env vars, resource limits, and securityContext in the job spec.`
+ default:
+ return ` → Check pod events with: kubectl describe pod <pod>`
+ }
+}
+
+export function getContainerErrors(pod: k8s.V1Pod): string[] {
+ const errors: string[] = []
+ const unrecoverableReasons = getUnrecoverableWaitingReasons()
+ const allStatuses = [
+ ...(pod.status?.initContainerStatuses ?? []),
+ ...(pod.status?.containerStatuses ?? [])
+ ]
+ for (const cs of allStatuses) {
+ const waiting = cs.state?.waiting
+ if (waiting?.reason && unrecoverableReasons.has(waiting.reason)) {
+ const reason = ` ✗ container "${cs.name}": ${waiting.reason}`
+ const detail = waiting.message ? `\n ${waiting.message}` : ''
+ const hint = `\n${getWaitingReasonHint(waiting.reason)}`
+ errors.push(`${reason}${detail}${hint}`)
+ }
+ }
+ return errors
+}
+
+export function getTerminatedReasonHint(
+ reason: string,
+ exitCode: number | undefined
+): string {
+ if (reason === 'OOMKilled') {
+ return ` → Container exceeded its memory limit and was killed by the OOM killer.\n Increase the memory limit in the job spec or reduce memory usage in the script.`
+ }
+ if (reason === 'FailedPostStartHookError') {
+ return ` → The postStart lifecycle hook failed. Check the hook command and its exit code.`
+ }
+ if (reason === 'Error') {
+ switch (exitCode) {
+ case 137:
+ return ` → Exit code 137: process was killed (SIGKILL). Likely OOM or forceful termination.\n Check memory usage and resource limits.`
+ case 139:
+ return ` → Exit code 139: segmentation fault (SIGSEGV). The process crashed due to a memory access error.`
+ case 126:
+ return ` → Exit code 126: permission denied. The script or binary is not executable.\n Check file permissions inside the container image.`
+ case 127:
+ return ` → Exit code 127: command not found. The script or binary does not exist in the container.\n Check the image contents and the command/entrypoint configuration.`
+ default:
+ return ` → Script or process exited with a non-zero code (${exitCode ?? 'unknown'}).\n Check the step output above for error messages.\n Common causes: script logic errors, missing dependencies, unhandled exceptions.`
+ }
+ }
+ return ` → Check pod logs with: kubectl logs <pod> -c ${reason}`
+}
+
+export function getContainerTerminatedErrors(pod: k8s.V1Pod): string[] {
+ const errors: string[] = []
+ const unrecoverableReasons = getUnrecoverableTerminatedReasons()
+ const allStatuses = [
+ ...(pod.status?.initContainerStatuses ?? []),
+ ...(pod.status?.containerStatuses ?? [])
+ ]
+ for (const cs of allStatuses) {
+ const terminated = cs.state?.terminated
+ if (terminated?.reason && unrecoverableReasons.has(terminated.reason)) {
+ const reason = ` ✗ container "${cs.name}": ${terminated.reason} (exit code ${terminated.exitCode})`
+ const detail = terminated.message ? `\n ${terminated.message}` : ''
+ const hint = `\n${getTerminatedReasonHint(terminated.reason, terminated.exitCode)}`
+ errors.push(`${reason}${detail}${hint}`)
+ }
+ }
+ return errors
+}
+
+function getEventReasonHint(reason: string): string {
+ switch (reason) {
+ case 'FailedMount':
+ return [
+ ` → A volume could not be mounted. Check:`,
+ ` - PVC is bound (kubectl get pvc)`,
+ ` - Secret/ConfigMap referenced in the volume exists`,
+ ` - hostPath directories exist on the scheduled node`
+ ].join('\n')
+ case 'FailedBinding':
+ return [
+ ` → A PVC could not be bound to a PV. Check:`,
+ ` - StorageClass exists and has a provisioner`,
+ ` - Sufficient capacity is available`,
+ ` - Access mode (ReadWriteOnce/ReadWriteMany) matches available PVs`
+ ].join('\n')
+ case 'FailedScheduling':
+ return [
+ ` → Pod cannot be scheduled due to a permanent configuration error. Check:`,
+ ` - nodeSelector / nodeAffinity labels match at least one node`,
+ ` - All tolerations are present for node taints`,
+ ` - PVCs referenced in the pod spec exist in the namespace`
+ ].join('\n')
+ default:
+ return ` → Check pod events with: kubectl describe pod <pod>`
+ }
+}
+
+// Inspects the pod's Warning events for reasons in UNRECOVERABLE_EVENT_REASONS.
+// For FailedScheduling, only fast-fails when the message positively matches
+// a known-permanent configuration error (see PERMANENT_SCHEDULING_PATTERNS).
+// Resource shortages ("Insufficient cpu/memory/gpu") are silently skipped so
+// the pod keeps queuing until the timeout. Best-effort: if events cannot be
+// listed (e.g. the optional 'events' RBAC permission is missing) it returns []
+// instead of blocking container-level fast-fail detection. Deduplicates by
+// reason so a repeatedly-retried event is reported once.
+export async function getPodEventErrors(podName: string): Promise<string[]> {
+ const unrecoverableReasons = getUnrecoverableEventReasons()
+ Iif (unrecoverableReasons.size === 0) {
+ return []
+ }
+
+ let items: k8s.CoreV1Event[]
+ try {
+ const result = await k8sApi.listNamespacedEvent({
+ namespace: namespace(),
+ fieldSelector: `involvedObject.name=${podName}`
+ })
+ items = result.items ?? []
+ } catch (err) {
+ core.debug(
+ `Could not list events for pod ${podName} during fast-fail check: ${
+ err instanceof Error ? err.message : String(err)
+ }`
+ )
+ return []
+ }
+
+ const errors: string[] = []
+ const seenReasons = new Set<string>()
+ for (const e of items) {
+ if (
+ e.type !== 'Warning' ||
+ !e.reason ||
+ !unrecoverableReasons.has(e.reason)
+ ) {
+ continue
+ }
+ // FailedScheduling: only fast-fail when the message positively matches a
+ // known-permanent config error. Resource shortages and unknown messages
+ // are treated as transient — let the pod keep queuing.
+ if (
+ e.reason === 'FailedScheduling' &&
+ !isPermanentSchedulingFailure(e.message)
+ ) {
+ core.debug(
+ `[fast-fail] Skipping FailedScheduling (not a recognised permanent error): ${
+ e.message ?? '(no message)'
+ }`
+ )
+ continue
+ }
+ if (seenReasons.has(e.reason)) {
+ continue
+ }
+ seenReasons.add(e.reason)
+ const count = e.count && e.count > 1 ? ` (x${e.count})` : ''
+ const reason = ` ✗ event: ${e.reason}${count}`
+ const detail = e.message ? `\n ${e.message}` : ''
+ const hint = getEventReasonHint(e.reason)
+ errors.push(`${reason}${detail}\n${hint}`)
+ }
+ return errors
+}
+
+// describePodFailure aggregates everything that might explain why a pod is not
+// healthy into a single human-readable string. It makes NO success/failure
+// judgement of its own (a terminated exitCode=0 container is just reported as
+// such) -- callers decide what is "bad". It never throws: if it cannot read the
+// pod or list events it returns/embeds a best-effort note instead, so it is safe
+// to call from any error path.
+export async function describePodFailure(podName: string): Promise<string> {
+ let pod: k8s.V1Pod
+ try {
+ pod = await readPod(podName)
+ } catch (err) {
+ return `Could not read pod ${podName} for diagnostics: ${
+ err instanceof Error ? err.message : String(err)
+ }`
+ }
+
+ const sections: string[] = []
+ const status = pod.status
+
+ // --- Pod-level status ---
+ const podLines: string[] = []
+ const phaseLabel = status?.phase
+ ? `${status.phase}${status.reason ? ` (${status.reason})` : ''}`
+ : 'Unknown'
+ podLines.push(`Pod status: ${phaseLabel}`)
+ if (status?.message) {
+ podLines.push(` ${status.message}`)
+ }
+
+ // Surface conditions that are False (e.g. PodScheduled=False from FailedScheduling)
+ for (const cond of status?.conditions ?? []) {
+ Eif (cond.status === 'False') {
+ const reason = cond.reason ? ` (${cond.reason})` : ''
+ const msg = cond.message ? `: ${cond.message}` : ''
+ podLines.push(` ✗ ${cond.type}=False${reason}${msg}`)
+ }
+ }
+ sections.push(podLines.join('\n'))
+
+ // --- Container-level status (non-zero exits and non-unrecoverable waits) ---
+ const unrecoverableReasons = getUnrecoverableWaitingReasons()
+ const allStatuses = [
+ ...(status?.initContainerStatuses ?? []),
+ ...(status?.containerStatuses ?? [])
+ ]
+ const containerLines: string[] = []
+ for (const cs of allStatuses) {
+ const waiting = cs.state?.waiting
+ if (waiting?.reason) {
+ // Skip reasons already surfaced by getContainerErrors() in the caller's
+ // first line to avoid printing the same error twice.
+ if (!unrecoverableReasons.has(waiting.reason)) {
+ const msg = waiting.message ? `\n ${waiting.message}` : ''
+ containerLines.push(
+ ` ✗ container "${cs.name}" waiting: ${waiting.reason}${msg}`
+ )
+ }
+ }
+ const terminated = cs.state?.terminated
+ // Only surface non-zero exits; exit 0 (e.g. fs-init Completed) is noise.
+ if (terminated && terminated.exitCode !== 0) {
+ const reason = terminated.reason ?? 'Unknown'
+ const msg = terminated.message ? `\n ${terminated.message}` : ''
+ containerLines.push(
+ ` ✗ container "${cs.name}" terminated: ${reason} (exit code ${terminated.exitCode})${msg}`
+ )
+ }
+ }
+ if (containerLines.length) {
+ sections.push(`Container details:\n${containerLines.join('\n')}`)
+ }
+
+ // --- Warning events ---
+ const eventLines = await describePodWarningEvents(podName)
+ if (eventLines.length) {
+ sections.push(`Recent warning events:\n${eventLines.join('\n')}`)
+ }
+
+ Iif (!sections.length) {
+ return `No additional diagnostic information available for pod ${podName}`
+ }
+ return sections.join('\n\n')
+}
+
+// Reads the most recent Warning events for a pod. Best-effort: listing events
+// requires the optional "events" get/list permission, so any error (including
+// RBAC Forbidden) is swallowed and an empty list is returned.
+async function describePodWarningEvents(podName: string): Promise<string[]> {
+ let items: k8s.CoreV1Event[]
+ try {
+ const result = await k8sApi.listNamespacedEvent({
+ namespace: namespace(),
+ fieldSelector: `involvedObject.name=${podName}`
+ })
+ items = result.items ?? []
+ } catch (err) {
+ core.debug(
+ `Could not list events for pod ${podName} (the 'events' permission may be missing): ${
+ err instanceof Error ? err.message : String(err)
+ }`
+ )
+ return []
+ }
+
+ const warnings = items.filter(e => e.type === 'Warning')
+ if (!warnings.length) {
+ return []
+ }
+
+ const eventTime = (e: k8s.CoreV1Event): number => {
+ const t = e.lastTimestamp ?? e.eventTime ?? e.firstTimestamp
+ return t ? new Date(t).getTime() : 0
+ }
+ warnings.sort((a, b) => eventTime(a) - eventTime(b))
+
+ const recent = warnings.slice(-MAX_DIAGNOSTIC_EVENTS)
+ const lines = recent.map(e => {
+ const count = e.count && e.count > 1 ? ` (x${e.count})` : ''
+ return ` [${e.reason ?? 'Warning'}]${count} ${e.message ?? ''}`.trimEnd()
+ })
+ Iif (warnings.length > recent.length) {
+ lines.unshift(
+ ` (showing ${recent.length} of ${warnings.length} warning events)`
+ )
+ }
+ return lines
+}
+
+// Inspects the pod's status.conditions for scheduling failures. FailedScheduling
+// is intentionally excluded from fast-fail detection (see UNRECOVERABLE_EVENT_REASONS
+// comment) — scheduling failures are transient resource waits that should queue
+// until the timeout, not terminate early. This function always returns [] as a
+// result, but is kept so checkUnrecoverableErrors compiles and the dedup logic
+// remains intact for future use.
+export function getPodConditionErrors(_p: k8s.V1Pod): string[] {
+ return []
+}
+
+// Aggregates the three independent error-detection sources (container waiting
+// reasons, pod Warning events, and pod conditions) into a single list, applying
+// the cross-source deduplication rule (FailedScheduling event + Unschedulable
+// condition = same failure, report once). Returns [] when no unrecoverable
+// error is detected. The caller decides what to do with the list -- typically
+// attach describePodFailure diagnostics and throw.
+export async function checkUnrecoverableErrors(
+ pod: k8s.V1Pod,
+ podName: string
+): Promise<string[]> {
+ // Deterministic terminal errors on a container (e.g. ImagePullBackOff).
+ const containerErrors = getContainerErrors(pod)
+ // Terminated container errors (e.g. OOMKilled, non-zero exit) with hints.
+ const terminatedErrors = getContainerTerminatedErrors(pod)
+ // Best-effort: returns [] when the optional events RBAC permission is absent.
+ const eventErrors = await getPodEventErrors(podName)
+ // Conditions are set near-instantly by the scheduler while events may take
+ // a few extra seconds to propagate, so always run the check. Deduplicate
+ // against events so the same FailedScheduling isn't printed twice.
+ const conditionErrors = getPodConditionErrors(pod).filter(
+ c =>
+ !eventErrors.some(
+ e => e.includes('FailedScheduling') && c.includes('Unschedulable')
+ )
+ )
+ return [
+ ...containerErrors,
+ ...terminatedErrors,
+ ...eventErrors,
+ ...conditionErrors
+ ]
+}
+
+export async function waitForPodPhases(
+ podName: string,
+ awaitingPhases: Set<PodPhase>,
+ backOffPhases: Set<PodPhase>,
+ maxTimeSeconds = DEFAULT_WAIT_FOR_POD_TIME_SECONDS
+): Promise<void> {
+ const backOffManager = new BackOffManager(maxTimeSeconds)
+ let phase: PodPhase = PodPhase.UNKNOWN
+ while (Itrue) {
+ let pod: k8s.V1Pod
+ try {
+ pod = await readPod(podName)
+ } catch (err) {
+ // Transient API error (network blip, API server busy): log and back off
+ // rather than crashing the loop. The timeout will eventually fire if the
+ // pod stays permanently unreadable (e.g. RBAC issue).
+ core.warning(
+ `[waitForPodPhases] Could not read pod ${podName}, will retry after backoff: ${
+ err instanceof Error ? err.message : String(err)
+ }`
+ )
+ try {
+ await backOffManager.backOff()
+ } catch {
+ throw new Error(
+ `Pod ${podName} timed out after ${maxTimeSeconds}s (pod read failed: ${
+ err instanceof Error ? err.message : String(err)
+ })\n${'-'.repeat(60)}\n(pod was unreadable; no further diagnostics available)`
+ )
+ }
+ continue
+ }
+ phase = parsePodPhase(pod)
+ if (awaitingPhases.has(phase)) {
+ return
+ }
+
+ // The pod reached a phase we are not willing to keep waiting on
+ // (a terminal/unhealthy phase). First check for unrecoverable
+ // container-level errors (e.g. OOMKilled, non-zero exit) so they are
+ // reported with actionable hints. Fall back to the generic "is unhealthy"
+ // message when no specific error is found.
+ if (!backOffPhases.has(phase)) {
+ const errors = await checkUnrecoverableErrors(pod, podName)
+ const details = await describePodFailure(podName)
+ Iif (errors.length > 0) {
+ throw new Error(
+ `Pod ${podName} has unrecoverable errors:\n${errors.join('\n')}\n${'-'.repeat(60)}\n${details}`
+ )
+ }
+ throw new Error(
+ `Pod ${podName} is unhealthy (phase: ${phase})\n${'-'.repeat(60)}\n${details}`
+ )
+ }
+
+ // Still in a back-off phase, but a deterministic unrecoverable error
+ // was detected (container / event / condition). Fail fast with
+ // diagnostics instead of waiting out the full timeout.
+ const errors = await checkUnrecoverableErrors(pod, podName)
+ if (errors.length > 0) {
+ const details = await describePodFailure(podName)
+ throw new Error(
+ `Pod ${podName} has unrecoverable errors:\n${errors.join('\n')}\n${'-'.repeat(60)}\n${details}`
+ )
+ }
+
+ try {
+ await backOffManager.backOff()
+ } catch (_err) {
+ // BackOffManager throws "backoff timeout" when maxTimeSeconds is exceeded.
+ // Don't surface that bare message: collect diagnostics first so the user
+ // can see WHY the pod never became ready.
+ const details = await describePodFailure(podName)
+ throw new Error(
+ `Pod ${podName} timed out after ${maxTimeSeconds}s (phase: ${phase})\n${'-'.repeat(60)}\n${details}`
+ )
+ }
+ }
+}
+
+export function getPrepareJobTimeoutSeconds(): number {
+ const envTimeoutSeconds =
+ process.env['ACTIONS_RUNNER_PREPARE_JOB_TIMEOUT_SECONDS']
+
+ Eif (!envTimeoutSeconds) {
+ return DEFAULT_WAIT_FOR_POD_TIME_SECONDS
+ }
+
+ const timeoutSeconds = parseInt(envTimeoutSeconds, 10)
+ if (!timeoutSeconds || timeoutSeconds <= 0) {
+ core.warning(
+ `Prepare job timeout is invalid ("${timeoutSeconds}"): use an int > 0`
+ )
+ return DEFAULT_WAIT_FOR_POD_TIME_SECONDS
+ }
+
+ return timeoutSeconds
+}
+
+async function readPod(podName: string): Promise<k8s.V1Pod> {
+ return await k8sApi.readNamespacedPod({
+ name: podName,
+ namespace: namespace()
+ })
+}
+
+const podPhaseLookup = new Set<string>([
+ PodPhase.PENDING,
+ PodPhase.RUNNING,
+ PodPhase.SUCCEEDED,
+ PodPhase.FAILED,
+ PodPhase.UNKNOWN
+])
+
+export function parsePodPhase(pod: k8s.V1Pod): PodPhase {
+ if (!pod.status?.phase || !podPhaseLookup.has(pod.status.phase)) {
+ return PodPhase.UNKNOWN
+ }
+ return pod.status.phase as PodPhase
+}
+
+async function isJobSucceeded(name: string): Promise<boolean> {
+ const job = await k8sBatchV1Api.readNamespacedJob({
+ name,
+ namespace: namespace()
+ })
+ Eif (job.status?.failed) {
+ throw new Error(`job ${name} has failed`)
+ }
+ return !!job.status?.succeeded
+}
+
+export async function getPodLogs(
+ podName: string,
+ containerName: string
+): Promise<void> {
+ const log = new k8s.Log(kc)
+ const logStream = new stream.PassThrough()
+ logStream.on('data', chunk => {
+ // use write rather than console.log to prevent double line feed
+ process.stdout.write(chunk)
+ })
+
+ await log.log(namespace(), podName, containerName, logStream, {
+ follow: true,
+ pretty: false,
+ timestamps: false
+ })
+ await new Promise((resolve, reject) => {
+ logStream.on('end', () => resolve(null))
+ logStream.on('error', err => {
+ process.stderr.write(err.message)
+ reject(err)
+ })
+ })
+}
+
+export async function prunePods(): Promise<void> {
+ const podList = await k8sApi.listNamespacedPod({
+ namespace: namespace(),
+ labelSelector: new RunnerInstanceLabel().toString()
+ })
+ if (!podList.items.length) {
+ return
+ }
+
+ await Promise.all(
+ podList.items.map(
+ async pod => pod.metadata?.name && (await deletePod(pod.metadata.name))
+ )
+ )
+}
+
+export async function getPodStatus(
+ name: string
+): Promise<k8s.V1PodStatus | undefined> {
+ const pod = await k8sApi.readNamespacedPod({
+ name,
+ namespace: namespace()
+ })
+ return pod.status
+}
+
+export async function isAuthPermissionsOK(): Promise<boolean> {
+ const sar = new k8s.V1SelfSubjectAccessReview()
+ const asyncs: Promise<k8s.V1SelfSubjectAccessReview>[] = []
+ for (const resource of requiredPermissions) {
+ for (const verb of resource.verbs) {
+ sar.spec = new k8s.V1SelfSubjectAccessReviewSpec()
+ sar.spec.resourceAttributes = new k8s.V1ResourceAttributes()
+ sar.spec.resourceAttributes.verb = verb
+ sar.spec.resourceAttributes.namespace = namespace()
+ sar.spec.resourceAttributes.group = resource.group
+ sar.spec.resourceAttributes.resource = resource.resource
+ sar.spec.resourceAttributes.subresource = resource.subresource
+ asyncs.push(
+ k8sAuthorizationV1Api.createSelfSubjectAccessReview({ body: sar })
+ )
+ }
+ }
+ const responses = await Promise.all(asyncs)
+ return responses.every(resp => resp.status?.allowed)
+}
+
+export async function isPodContainerAlpine(
+ podName: string,
+ containerName: string
+): Promise<boolean> {
+ let isAlpine = true
+ try {
+ await execPodStep(
+ [
+ 'sh',
+ '-c',
+ `[ $(cat /etc/*release* | grep -i -e "^ID=*alpine*" -c) != 0 ] || exit 1`
+ ],
+ podName,
+ containerName
+ )
+ } catch {
+ isAlpine = false
+ }
+
+ return isAlpine
+}
+
+export function namespace(): string {
+ if (process.env['ACTIONS_RUNNER_KUBERNETES_NAMESPACE']) {
+ return process.env['ACTIONS_RUNNER_KUBERNETES_NAMESPACE']
+ }
+
+ const context = kc.getContexts().find(ctx => ctx.namespace)
+ Eif (context?.namespace) {
+ return context.namespace
+ }
+
+ // When running in-cluster the kubeconfig context has no namespace field;
+ // read it from the mounted ServiceAccount file instead.
+ const saNamespaceFile =
+ '/var/run/secrets/kubernetes.io/serviceaccount/namespace'
+ try {
+ const ns = fs.readFileSync(saNamespaceFile, 'utf8').trim()
+ if (ns) {
+ return ns
+ }
+ } catch {
+ // not running in-cluster, fall through to error
+ }
+
+ throw new Error(
+ 'Failed to determine namespace. Set the ACTIONS_RUNNER_KUBERNETES_NAMESPACE environment variable or ensure the kubeconfig context includes a namespace.'
+ )
+}
+
+class BackOffManager {
+ private backOffSeconds = 1
+ totalTime = 0
+ constructor(private throwAfterSeconds?: number) {
+ if (!throwAfterSeconds || throwAfterSeconds < 0) {
+ this.throwAfterSeconds = undefined
+ }
+ }
+
+ async backOff(): Promise<void> {
+ await new Promise(resolve =>
+ setTimeout(resolve, this.backOffSeconds * 1000)
+ )
+ this.totalTime += this.backOffSeconds
+ if (this.throwAfterSeconds && this.throwAfterSeconds < this.totalTime) {
+ throw new Error('backoff timeout')
+ }
+ Eif (this.backOffSeconds < 20) {
+ this.backOffSeconds *= 2
+ }
+ Iif (this.backOffSeconds > 20) {
+ this.backOffSeconds = 20
+ }
+ }
+}
+
+export function containerPorts(
+ container: ContainerInfo
+): k8s.V1ContainerPort[] {
+ const ports: k8s.V1ContainerPort[] = []
+ Eif (!container.portMappings?.length) {
+ return ports
+ }
+ for (const portDefinition of container.portMappings) {
+ const portProtoSplit = portDefinition.split('/')
+ if (portProtoSplit.length > 2) {
+ throw new Error(`Unexpected port format: ${portDefinition}`)
+ }
+
+ const port = new k8s.V1ContainerPort()
+ port.protocol =
+ portProtoSplit.length === 2 ? portProtoSplit[1].toUpperCase() : 'TCP'
+
+ const portSplit = portProtoSplit[0].split(':')
+ if (portSplit.length > 2) {
+ throw new Error('ports should have at most one ":" separator')
+ }
+
+ const parsePort = (p: string): number => {
+ const num = Number(p)
+ if (!Number.isInteger(num) || num < 1 || num > 65535) {
+ throw new Error(`invalid container port: ${p}`)
+ }
+ return num
+ }
+
+ if (portSplit.length === 1) {
+ port.containerPort = parsePort(portSplit[0])
+ } else {
+ port.hostPort = parsePort(portSplit[0])
+ port.containerPort = parsePort(portSplit[1])
+ }
+
+ ports.push(port)
+ }
+ return ports
+}
+
+export async function getPodByName(name): Promise<k8s.V1Pod> {
+ return await k8sApi.readNamespacedPod({
+ name,
+ namespace: namespace()
+ })
+}
+
+export async function getSecretByName(name: string): Promise<k8s.V1Secret> {
+ return await k8sApi.readNamespacedSecret({
+ name,
+ namespace: namespace()
+ })
+}
+
+export async function listPodsByRunnerInstance(): Promise<k8s.V1Pod[]> {
+ const podList = await k8sApi.listNamespacedPod({
+ namespace: namespace(),
+ labelSelector: new RunnerInstanceLabel().toString()
+ })
+ return podList.items ?? []
+}
+ |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| File | ++ | Statements | ++ | Branches | ++ | Functions | ++ | Lines | ++ |
|---|---|---|---|---|---|---|---|---|---|
| index.ts | +
+
+ |
+ 57.38% | +101/176 | +43.85% | +75/171 | +47.36% | +9/19 | +60.66% | +91/150 | +
| npu-metrics.ts | +
+
+ |
+ 21.71% | +76/350 | +4.55% | +18/395 | +13.72% | +7/51 | +23.64% | +70/296 | +
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 +33 +34 +35 +36 +37 +38 +39 +40 +41 +42 +43 +44 +45 +46 +47 +48 +49 +50 +51 +52 +53 +54 +55 +56 +57 +58 +59 +60 +61 +62 +63 +64 +65 +66 +67 +68 +69 +70 +71 +72 +73 +74 +75 +76 +77 +78 +79 +80 +81 +82 +83 +84 +85 +86 +87 +88 +89 +90 +91 +92 +93 +94 +95 +96 +97 +98 +99 +100 +101 +102 +103 +104 +105 +106 +107 +108 +109 +110 +111 +112 +113 +114 +115 +116 +117 +118 +119 +120 +121 +122 +123 +124 +125 +126 +127 +128 +129 +130 +131 +132 +133 +134 +135 +136 +137 +138 +139 +140 +141 +142 +143 +144 +145 +146 +147 +148 +149 +150 +151 +152 +153 +154 +155 +156 +157 +158 +159 +160 +161 +162 +163 +164 +165 +166 +167 +168 +169 +170 +171 +172 +173 +174 +175 +176 +177 +178 +179 +180 +181 +182 +183 +184 +185 +186 +187 +188 +189 +190 +191 +192 +193 +194 +195 +196 +197 +198 +199 +200 +201 +202 +203 +204 +205 +206 +207 +208 +209 +210 +211 +212 +213 +214 +215 +216 +217 +218 +219 +220 +221 +222 +223 +224 +225 +226 +227 +228 +229 +230 +231 +232 +233 +234 +235 +236 +237 +238 +239 +240 +241 +242 +243 +244 +245 +246 +247 +248 +249 +250 +251 +252 +253 +254 +255 +256 +257 +258 +259 +260 +261 +262 +263 +264 +265 +266 +267 +268 +269 +270 +271 +272 +273 +274 +275 +276 +277 +278 +279 +280 +281 +282 +283 +284 +285 +286 +287 +288 +289 +290 +291 +292 +293 +294 +295 +296 +297 +298 +299 +300 +301 +302 +303 +304 +305 +306 +307 +308 +309 +310 +311 +312 +313 +314 +315 +316 +317 +318 +319 +320 +321 +322 +323 +324 +325 +326 +327 +328 +329 +330 +331 +332 +333 +334 +335 +336 +337 +338 +339 +340 +341 +342 +343 +344 +345 +346 +347 +348 +349 +350 +351 +352 +353 +354 +355 +356 +357 +358 +359 | +14x +14x +14x +14x +14x +14x + + +14x +14x + +14x +14x + +14x +14x +14x + +14x + + + + + + + + + + + + + + +14x + + + + + + + + + + + + + + + + + + + + + + + +14x + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +14x + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +14x +12x +12x + +12x +2x + + +10x + + + + + + + + + + +14x + + + +6x +6x +2x +1x + +2x +4x +1x +1x +3x +1x +1x +2x + + +2x +1x +1x + +1x + + + + +14x + + + +4x +4x +1x + +1x + + +3x +1x +1x + +2x + + + + +14x + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +14x +23x +23x +23x + + + + + + + + +14x + + + +14x +14x +14x +14x +14x +14x +14x + + + +3x +3x + + +3x +3x + + +14x + + +17x +3x + +14x + + +14x + + + +14x + + + + + + + + + +14x +63x +2x + + + + + +61x + +61x +3x +3x +3x +3x + + + + + +58x +47x + + +11x + + + + +6x +6x +3x + +3x +3x + +2x + + + + + +5x + + | import * as k8s from '@kubernetes/client-node'
+import * as fs from 'fs'
+import * as yaml from 'js-yaml'
+import * as core from '@actions/core'
+import { v1 as uuidv4 } from 'uuid'
+import { CONTAINER_EXTENSION_PREFIX } from '../../hooks/constants'
+import * as shlex from 'shlex'
+import { Mount } from 'hooklib'
+
+export const DEFAULT_CONTAINER_ENTRY_POINT_ARGS = [`-f`, `/dev/null`]
+export const DEFAULT_CONTAINER_ENTRY_POINT = 'tail'
+
+export const ENV_HOOK_TEMPLATE_PATH = 'ACTIONS_RUNNER_CONTAINER_HOOK_TEMPLATE'
+export const ENV_USE_KUBE_SCHEDULER = 'ACTIONS_RUNNER_USE_KUBE_SCHEDULER'
+
+export const EXTERNALS_VOLUME_NAME = 'externals'
+export const GITHUB_VOLUME_NAME = 'github'
+export const WORK_VOLUME = 'work'
+
+export const CONTAINER_VOLUMES: k8s.V1VolumeMount[] = [
+ {
+ name: EXTERNALS_VOLUME_NAME,
+ mountPath: '/__e'
+ },
+ {
+ name: WORK_VOLUME,
+ mountPath: '/__w'
+ },
+ {
+ name: GITHUB_VOLUME_NAME,
+ mountPath: '/github'
+ }
+]
+
+export function prepareJobScript(userVolumeMounts: Mount[]): {
+ containerPath: string
+ runnerPath: string
+} {
+ let mountDirs = userVolumeMounts
+ .map(m => shlex.quote(m.targetVolumePath))
+ .join(' ')
+
+ const content = `#!/bin/sh -l
+set -e
+cp -R /__w/_temp/_github_home /github/home
+cp -R /__w/_temp/_github_workflow /github/workflow
+mkdir -p ${mountDirs}
+`
+
+ const filename = `${uuidv4()}.sh`
+ const entryPointPath = `${process.env.RUNNER_TEMP}/${filename}`
+ fs.writeFileSync(entryPointPath, content)
+ return {
+ containerPath: `/__w/_temp/${filename}`,
+ runnerPath: entryPointPath
+ }
+}
+
+export function writeRunScript(
+ workingDirectory: string,
+ entryPoint: string,
+ entryPointArgs?: string[],
+ prependPath?: string[],
+ environmentVariables?: { [key: string]: string }
+): { containerPath: string; runnerPath: string } {
+ let exportPath = ''
+ if (prependPath?.length) {
+ // TODO: remove compatibility with typeof prependPath === 'string' as we bump to next major version, the hooks will lose PrependPath compat with runners 2.293.0 and older
+ const prepend =
+ typeof prependPath === 'string' ? prependPath : prependPath.join(':')
+ exportPath = `export PATH=${prepend}:$PATH`
+ }
+
+ let environmentPrefix = scriptEnv(environmentVariables)
+
+ const content = `#!/bin/sh -l
+set -e
+rm "$0" # remove script after running
+${exportPath}
+cd ${workingDirectory} && \
+exec ${environmentPrefix} ${entryPoint} ${
+ entryPointArgs?.length ? entryPointArgs.join(' ') : ''
+ }
+`
+ const filename = `${uuidv4()}.sh`
+ const entryPointPath = `${process.env.RUNNER_TEMP}/${filename}`
+ fs.writeFileSync(entryPointPath, content)
+ return {
+ containerPath: `/__w/_temp/${filename}`,
+ runnerPath: entryPointPath
+ }
+}
+
+export function writeContainerStepScript(
+ dst: string,
+ workingDirectory: string,
+ entryPoint: string,
+ entryPointArgs?: string[],
+ environmentVariables?: { [key: string]: string }
+): { containerPath: string; runnerPath: string } {
+ let environmentPrefix = scriptEnv(environmentVariables)
+
+ const parts = workingDirectory.split('/').slice(-2)
+ if (parts.length !== 2) {
+ throw new Error(`Invalid working directory: ${workingDirectory}`)
+ }
+
+ const content = `#!/bin/sh -l
+rm "$0" # remove script after running
+mv /__w/_temp/_github_home /github/home && \
+mv /__w/_temp/_github_workflow /github/workflow && \
+mv /__w/_temp/_runner_file_commands /github/file_commands || true && \
+mv ${shlex.quote('/__w/' + parts.join('/') + '/')} /github/workspace && \
+cd /github/workspace && \
+exec ${environmentPrefix} ${shlex.quote(entryPoint)} ${
+ entryPointArgs?.length ? entryPointArgs.map(shlex.quote).join(' ') : ''
+ }
+`
+ const filename = `${uuidv4()}.sh`
+ const entryPointPath = `${dst}/${filename}`
+ core.debug(`Writing container step script to ${entryPointPath}`)
+ fs.writeFileSync(entryPointPath, content)
+ return {
+ containerPath: `/__w/_temp/${filename}`,
+ runnerPath: entryPointPath
+ }
+}
+
+function scriptEnv(envs?: { [key: string]: string }): string {
+ if (!envs || !Object.entries(envs).length) {
+ return ''
+ }
+ const envBuffer: string[] = []
+ for (const [key, value] of Object.entries(envs)) {
+ if (
+ key.includes(`=`) ||
+ key.includes(`'`) ||
+ key.includes(`"`) ||
+ key.includes(`$`)
+ ) {
+ throw new Error(
+ `environment key ${key} is invalid - the key must not contain =, $, ', or "`
+ )
+ }
+ envBuffer.push(
+ `"${key}=${value
+ .replace(/\\/g, '\\\\')
+ .replace(/"/g, '\\"')
+ .replace(/\$/g, '\\$')
+ .replace(/`/g, '\\`')}"`
+ )
+ }
+
+ if (!envBuffer?.length) {
+ return ''
+ }
+
+ return `env ${envBuffer.join(' ')} `
+}
+
+export function generateContainerName(image: string): string {
+ const nameWithTag = image.split('/').pop()
+ const name = nameWithTag?.split(':')[0]
+
+ if (!name) {
+ throw new Error(`Image definition '${image}' is invalid`)
+ }
+
+ return name
+}
+
+// Overwrite or append based on container options
+//
+// Keep in mind, envs and volumes could be passed as fields in container definition
+// so default volume mounts and envs are appended first, and then create options are used
+// to append more values
+//
+// Rest of the fields are just applied
+// For example, container.createOptions.container.image is going to overwrite container.image field
+export function mergeContainerWithOptions(
+ base: k8s.V1Container,
+ from: k8s.V1Container
+): void {
+ for (const [key, value] of Object.entries(from)) {
+ if (key === 'name') {
+ if (value !== CONTAINER_EXTENSION_PREFIX + base.name) {
+ core.warning("Skipping name override: name can't be overwritten")
+ }
+ continue
+ } else if (key === 'image') {
+ core.warning("Skipping image override: image can't be overwritten")
+ continue
+ } else if (key === 'env') {
+ const envs = value as k8s.V1EnvVar[]
+ base.env = mergeLists(base.env, envs)
+ } else Iif (key === 'volumeMounts' && value) {
+ const volumeMounts = value as k8s.V1VolumeMount[]
+ base.volumeMounts = mergeLists(base.volumeMounts, volumeMounts)
+ } else if (key === 'ports' && value) {
+ const ports = value as k8s.V1ContainerPort[]
+ base.ports = mergeLists(base.ports, ports)
+ } else {
+ base[key] = value
+ }
+ }
+}
+
+export function mergePodSpecWithOptions(
+ base: k8s.V1PodSpec,
+ from: k8s.V1PodSpec
+): void {
+ for (const [key, value] of Object.entries(from)) {
+ if (key === 'containers') {
+ base.containers.push(
+ ...from.containers.filter(
+ e => !e.name?.startsWith(CONTAINER_EXTENSION_PREFIX)
+ )
+ )
+ } else if (key === 'volumes' && value) {
+ const volumes = value as k8s.V1Volume[]
+ base.volumes = mergeLists(base.volumes, volumes)
+ } else {
+ base[key] = value
+ }
+ }
+}
+
+export function mergeObjectMeta(
+ base: { metadata?: k8s.V1ObjectMeta },
+ from: k8s.V1ObjectMeta
+): void {
+ if (!base.metadata?.labels || !base.metadata?.annotations) {
+ throw new Error(
+ "Can't merge metadata: base.metadata or base.annotations field is undefined"
+ )
+ }
+ if (from?.labels) {
+ for (const [key, value] of Object.entries(from.labels)) {
+ if (base.metadata?.labels?.[key]) {
+ core.warning(`Label ${key} is already defined and will be overwritten`)
+ }
+ base.metadata.labels[key] = value
+ }
+ }
+
+ if (from?.annotations) {
+ for (const [key, value] of Object.entries(from.annotations)) {
+ if (base.metadata?.annotations?.[key]) {
+ core.warning(
+ `Annotation ${key} is already defined and will be overwritten`
+ )
+ }
+ base.metadata.annotations[key] = value
+ }
+ }
+}
+
+export function readExtensionFromFile(): k8s.V1PodTemplateSpec | undefined {
+ const filePath = process.env[ENV_HOOK_TEMPLATE_PATH]
+ Eif (!filePath) {
+ return undefined
+ }
+ const doc = yaml.load(fs.readFileSync(filePath, 'utf8'))
+ if (!doc || typeof doc !== 'object') {
+ throw new Error(`Failed to parse ${filePath}`)
+ }
+ return doc as k8s.V1PodTemplateSpec
+}
+
+export function useKubeScheduler(): boolean {
+ return process.env[ENV_USE_KUBE_SCHEDULER] === 'true'
+}
+
+export enum PodPhase {
+ PENDING = 'Pending',
+ RUNNING = 'Running',
+ SUCCEEDED = 'Succeeded',
+ FAILED = 'Failed',
+ UNKNOWN = 'Unknown',
+ COMPLETED = 'Completed'
+}
+
+function mergeLists<T>(base?: T[], from?: T[]): T[] {
+ const b: T[] = base || []
+ Iif (!from?.length) {
+ return b
+ }
+ b.push(...from)
+ return b
+}
+
+export function fixArgs(args: string[]): string[] {
+ // Preserve shell command strings passed via `sh -c` without re-tokenizing.
+ // Retokenizing would split the script into multiple args, breaking `sh -c`.
+ if (args.length >= 2 && args[0] === 'sh' && args[1] === '-c') {
+ return args
+ }
+ return shlex.split(args.join(' '))
+}
+
+export async function sleep(ms: number): Promise<void> {
+ return new Promise(resolve => setTimeout(resolve, ms))
+}
+
+export function listDirAllCommand(dir: string): string {
+ return `cd ${shlex.quote(dir)} && find . -type f -not -path '*/_runner_hook_responses*' -exec stat -c '%s %n' {} \\;`
+}
+
+// Safely turn an unknown thrown value into a diagnostic string without
+// throwing. The previous `JSON.stringify(err)` pattern crashed with
+// `TypeError: Converting circular structure to JSON` when err was a
+// @kubernetes/client-node HTTP error (its response embeds a
+// TLSSocket <-> HTTPParser cycle). The thrown TypeError shadowed the
+// original failure in every catch block that used it (issue #329).
+export function formatError(err: unknown): string {
+ if (err === null || err === undefined) {
+ return String(err)
+ }
+
+ // @kubernetes/client-node API errors expose the actual server message
+ // under response.body — prefer that when available.
+ const body =
+ (err as { response?: { body?: unknown } })?.response?.body ??
+ (err as { body?: unknown })?.body
+ if (body && typeof body === 'object') {
+ const msg = (body as { message?: unknown }).message
+ const reason = (body as { reason?: unknown }).reason
+ Eif (typeof msg === 'string') {
+ return typeof reason === 'string' && reason.length > 0
+ ? `${msg} (reason: ${reason})`
+ : msg
+ }
+ }
+
+ if (err instanceof Error) {
+ return err.message
+ }
+
+ if (typeof err === 'object') {
+ // Non-Error objects sometimes carry a top-level message (axios-style
+ // errors, hand-rolled error-likes). Extract before the JSON.stringify
+ // branch so a circular ref doesn't reduce the diagnostic to
+ // "[object Object]".
+ const msg = (err as { message?: unknown }).message
+ if (typeof msg === 'string') {
+ return msg
+ }
+ try {
+ return JSON.stringify(err)
+ } catch {
+ return String(err)
+ }
+ }
+
+ // Primitives serialise more readably via String() than JSON.stringify
+ // (which would quote strings and refuse to handle symbols).
+ return String(err)
+}
+ |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 +33 +34 +35 +36 +37 +38 +39 +40 +41 +42 +43 +44 +45 +46 +47 +48 +49 +50 +51 +52 +53 +54 +55 +56 +57 +58 +59 +60 +61 +62 +63 +64 +65 +66 +67 +68 +69 +70 +71 +72 +73 +74 +75 +76 +77 +78 +79 +80 +81 +82 +83 +84 +85 +86 +87 +88 +89 +90 +91 +92 +93 +94 +95 +96 +97 +98 +99 +100 +101 +102 +103 +104 +105 +106 +107 +108 +109 +110 +111 +112 +113 +114 +115 +116 +117 +118 +119 +120 +121 +122 +123 +124 +125 +126 +127 +128 +129 +130 +131 +132 +133 +134 +135 +136 +137 +138 +139 +140 +141 +142 +143 +144 +145 +146 +147 +148 +149 +150 +151 +152 +153 +154 +155 +156 +157 +158 +159 +160 +161 +162 +163 +164 +165 +166 +167 +168 +169 +170 +171 +172 +173 +174 +175 +176 +177 +178 +179 +180 +181 +182 +183 +184 +185 +186 +187 +188 +189 +190 +191 +192 +193 +194 +195 +196 +197 +198 +199 +200 +201 +202 +203 +204 +205 +206 +207 +208 +209 +210 +211 +212 +213 +214 +215 +216 +217 +218 +219 +220 +221 +222 +223 +224 +225 +226 +227 +228 +229 +230 +231 +232 +233 +234 +235 +236 +237 +238 +239 +240 +241 +242 +243 +244 +245 +246 +247 +248 +249 +250 +251 +252 +253 +254 +255 +256 +257 +258 +259 +260 +261 +262 +263 +264 +265 +266 +267 +268 +269 +270 +271 +272 +273 +274 +275 +276 +277 +278 +279 +280 +281 +282 +283 +284 +285 +286 +287 +288 +289 +290 +291 +292 +293 +294 +295 +296 +297 +298 +299 +300 +301 +302 +303 +304 +305 +306 +307 +308 +309 +310 +311 +312 +313 +314 +315 +316 +317 +318 +319 +320 +321 +322 +323 +324 +325 +326 +327 +328 +329 +330 +331 +332 +333 +334 +335 +336 +337 +338 +339 +340 +341 +342 +343 +344 +345 +346 +347 +348 +349 +350 +351 +352 +353 +354 +355 +356 +357 +358 +359 +360 +361 +362 +363 +364 +365 +366 +367 +368 +369 +370 +371 +372 +373 +374 +375 +376 +377 +378 +379 +380 +381 +382 +383 +384 +385 +386 +387 +388 +389 +390 +391 +392 +393 +394 +395 +396 +397 +398 +399 +400 +401 +402 +403 +404 +405 +406 +407 +408 +409 +410 +411 +412 +413 +414 +415 +416 +417 +418 +419 +420 +421 +422 +423 +424 +425 +426 +427 +428 +429 +430 +431 +432 +433 +434 +435 +436 +437 +438 +439 +440 +441 +442 +443 +444 +445 +446 +447 +448 +449 +450 +451 +452 +453 +454 +455 +456 +457 +458 +459 +460 +461 +462 +463 +464 +465 +466 +467 +468 +469 +470 +471 +472 +473 +474 +475 +476 +477 +478 +479 +480 +481 +482 +483 +484 +485 +486 +487 +488 +489 +490 +491 +492 +493 +494 +495 +496 +497 +498 +499 +500 +501 +502 +503 +504 +505 +506 +507 +508 +509 +510 +511 +512 +513 +514 +515 +516 +517 +518 +519 +520 +521 +522 +523 +524 +525 +526 +527 +528 +529 +530 +531 +532 +533 +534 +535 +536 +537 +538 +539 +540 +541 +542 +543 +544 +545 +546 +547 +548 +549 +550 +551 +552 +553 +554 +555 +556 +557 +558 +559 +560 +561 +562 +563 +564 +565 +566 +567 +568 +569 +570 +571 +572 +573 +574 +575 +576 +577 +578 +579 +580 +581 +582 +583 +584 +585 +586 +587 +588 +589 +590 +591 +592 +593 +594 +595 +596 +597 +598 +599 +600 +601 +602 +603 +604 +605 +606 +607 +608 +609 +610 +611 +612 +613 +614 +615 +616 +617 +618 +619 +620 +621 +622 +623 +624 +625 +626 +627 +628 +629 +630 +631 +632 +633 +634 +635 +636 +637 +638 +639 +640 +641 +642 +643 +644 +645 +646 +647 +648 +649 +650 +651 +652 +653 +654 +655 +656 +657 +658 +659 +660 +661 +662 +663 +664 +665 +666 +667 +668 +669 +670 +671 +672 +673 +674 +675 +676 +677 +678 +679 +680 +681 +682 +683 +684 +685 +686 +687 +688 +689 +690 +691 +692 +693 +694 +695 +696 +697 +698 +699 +700 +701 +702 +703 +704 +705 +706 +707 +708 +709 +710 +711 +712 +713 +714 +715 +716 +717 +718 +719 +720 +721 +722 +723 +724 +725 +726 +727 +728 +729 +730 +731 +732 +733 +734 +735 +736 +737 +738 +739 +740 +741 +742 +743 +744 +745 +746 +747 +748 +749 +750 +751 +752 +753 +754 +755 +756 +757 +758 +759 | 9x + +9x +9x +9x +9x +9x +9x + + + + + + + +9x +9x +9x +9x + +9x +9x +9x + +9x +9x +9x +9x + +9x +9x +9x + +9x +9x + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +9x +128x + +64x + + + + +9x + + + + + + + + + + + + + + + + +9x + + + + + + + + + + + + + + + + +9x +22x +44x +44x + + + + + +22x + + +9x + + + + + + + +9x + + + + + + + + + + + + + + + + + + + + + + + + +9x + + + + + + + + + + + + + + + + + + + + + + + + + + + + +9x + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +9x + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +9x +22x + + +22x +22x + + + + +9x + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +9x + + + + + + + + + +9x + + + +9x + + + + + + +9x + + + + + + + + + + + + + +9x + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +9x + + + + + + +9x + + + + + + + + + +9x + + + +21x +21x +21x + +21x +21x + + + + + + + + + + + + + + + + + + + + +9x + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +9x + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +9x + + + + + + + + + + + + + + + + + + + + + + + + + + +9x + + + + + + + + + + + + + + + + + + + +9x + + + + + + + + + + + + + + + + + + + + + + + +9x + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +9x + + + + + + + + + + + + + +9x + + + + + + + + + + + + + + + + + + + + + + + + + +9x +21x + + +21x + +21x + +21x +21x + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + | import * as core from '@actions/core'
+import * as k8s from '@kubernetes/client-node'
+import * as https from 'https'
+import * as http from 'http'
+import { URL } from 'url'
+import { getRunnerPodName, JOB_CONTAINER_NAME } from '../../hooks/constants'
+import { formatError } from './index'
+import {
+ execPodStepWithOutput,
+ getPodByName,
+ getSecretByName,
+ listPodsByRunnerInstance,
+ namespace
+} from '../index'
+
+export const NPU_METRICS_MARKER = 'NPU_METRICS_V1'
+export const ENV_NPU_METRICS_ENABLED = 'ACTIONS_RUNNER_NPU_METRICS'
+export const ENV_NPU_SAMPLE_INTERVAL = 'NPU_SAMPLE_INTERVAL'
+export const ENV_NPU_IDLE_THRESHOLD = 'NPU_IDLE_THRESHOLD'
+
+export const DEFAULT_NPU_SAMPLE_INTERVAL_SECONDS = 1
+export const MIN_NPU_SAMPLE_INTERVAL_SECONDS = 0.2
+export const DEFAULT_NPU_IDLE_THRESHOLD_PERCENT = 5
+
+export const NPU_SAMPLES_FILE = '/tmp/npu-samples.jsonl'
+export const NPU_TERMINATION_LOG = '/dev/termination-log'
+export const NPU_SAMPLES_MAX_BYTES = 4 * 1024 * 1024
+export const NPU_AGGREGATE_MAX_BYTES = 4096
+
+export const NPU_METRICS_SECRET_NAME = 'npu-metrics-pushgw'
+export const NPU_METRICS_PUSH_JOB = 'npu-job-record'
+export const NPU_COLLECT_TIMEOUT_MS = 10_000
+
+const NPU_RESOURCE_REGEX = /^huawei\.com\/(ascend\S*)$/i
+const SCALE_SET_NAME_REGEX = /^linux-[a-z0-9]+-[a-z0-9]+-\d+-([a-z][a-z0-9-]*)$/
+
+export interface NpuCardAggregate {
+ card: string
+ samples: number
+ peakUtil: number
+ avgUtil: number
+ peakHbm: number
+ firstTs: number
+ lastTs: number
+}
+
+export interface NpuAggregatePayload {
+ version: string
+ ts: number
+ cards: NpuCardAggregate[]
+}
+
+export interface NpuPushConfig {
+ url: string
+ username?: string
+ password?: string
+}
+
+export interface NpuJobLabels {
+ cluster: string
+ namespace: string
+ pod: string
+ podType: string
+ repo: string
+ runId: string
+ npuType: string
+ cardsRequested: string
+ result: string
+}
+
+export function npuMetricsEnabled(
+ Eenv: Record<string, string | undefined> = process.env
+): boolean {
+ return (
+ (env[ENV_NPU_METRICS_ENABLED] ?? '').trim().toLowerCase() !== 'disabled'
+ )
+}
+
+export function getNpuSampleIntervalSeconds(
+ env: Record<string, string | undefined> = process.env
+): number {
+ const raw = env[ENV_NPU_SAMPLE_INTERVAL]
+ if (!raw) {
+ return DEFAULT_NPU_SAMPLE_INTERVAL_SECONDS
+ }
+ const parsed = Number(raw)
+ if (!Number.isFinite(parsed) || parsed <= 0) {
+ core.warning(
+ `NPU_SAMPLE_INTERVAL is invalid ("${raw}"): use ${DEFAULT_NPU_SAMPLE_INTERVAL_SECONDS}`
+ )
+ return DEFAULT_NPU_SAMPLE_INTERVAL_SECONDS
+ }
+ return Math.max(parsed, MIN_NPU_SAMPLE_INTERVAL_SECONDS)
+}
+
+export function getIdleThresholdPercent(
+ env: Record<string, string | undefined> = process.env
+): number {
+ const raw = env[ENV_NPU_IDLE_THRESHOLD]
+ if (!raw) {
+ return DEFAULT_NPU_IDLE_THRESHOLD_PERCENT
+ }
+ const parsed = Number(raw)
+ if (!Number.isFinite(parsed) || parsed < 0) {
+ core.warning(
+ `NPU_IDLE_THRESHOLD is invalid ("${raw}"): use ${DEFAULT_NPU_IDLE_THRESHOLD_PERCENT}`
+ )
+ return DEFAULT_NPU_IDLE_THRESHOLD_PERCENT
+ }
+ return parsed
+}
+
+export function containerHasNpuRequest(container: k8s.V1Container): boolean {
+ const resources = [container.resources?.requests, container.resources?.limits]
+ for (const entry of resources) {
+ for (const key of Object.keys(entry ?? {})) {
+ if (NPU_RESOURCE_REGEX.test(key)) {
+ return true
+ }
+ }
+ }
+ return false
+}
+
+export function hasNpuRequest(spec: k8s.V1PodSpec | undefined): boolean {
+ return (spec?.containers ?? []).some(c => containerHasNpuRequest(c))
+}
+
+function listNpuCardsCommand(): string {
+ return `ls /dev/davinci[0-9]* 2>/dev/null | sed 's|.*/davinci||' | sort -n | uniq | tr '\\n' ' '`
+}
+
+export function buildNpuSamplerScript(
+ intervalSeconds = DEFAULT_NPU_SAMPLE_INTERVAL_SECONDS
+): string {
+ return [
+ `# ${NPU_METRICS_MARKER} sampler (detached, best-effort)`,
+ `F=${NPU_SAMPLES_FILE}`,
+ `CARDS=$(${listNpuCardsCommand()})`,
+ `[ -z "$CARDS" ] && exit 0`,
+ `while :; do`,
+ ` NOW=$(date +%s)`,
+ ` for C in $CARDS; do`,
+ ` OUT=$(npu-smi info -t usages -i "$C" 2>/dev/null) || continue`,
+ ` U=$(printf '%s\\n' "$OUT" | awk '{l=$0; gsub(/[ \\t\\r]/, "", l)} l ~ /AICore/ { if (match(l, /[0-9.]+%?$/)) { v = substr(l, RSTART, RLENGTH); sub(/%$/, "", v); print v; exit } }')`,
+ ` H=$(printf '%s\\n' "$OUT" | awk '{l=$0; gsub(/[ \\t\\r]/, "", l)} l ~ /HBM/ { if (match(l, /[0-9.]+%?$/)) { v = substr(l, RSTART, RLENGTH); sub(/%$/, "", v); print v; exit } }')`,
+ ` printf '{"ts":%s,"card":%s,"util":%s,"hbm":%s}\\n' "$NOW" "$C" "\${U:-0}" "\${H:-0}" >> "$F" 2>/dev/null || true`,
+ ` done`,
+ ` SZ=$(wc -c < "$F" 2>/dev/null || echo 0)`,
+ ` if [ "$SZ" -gt ${NPU_SAMPLES_MAX_BYTES} ]; then`,
+ ` tail -n 2000 "$F" > "$F.r" 2>/dev/null && mv "$F.r" "$F" || true`,
+ ` fi`,
+ ` sleep ${intervalSeconds}`,
+ `done`
+ ].join('\n')
+}
+
+const AGGREGATE_AWK = [
+ `BEGIN {`,
+ ` n = split(cards, a, " ")`,
+ ` for (i = 1; i <= n; i++) { cnt[a[i]] = 0; pu[a[i]] = 0; ph[a[i]] = 0; su[a[i]] = 0; fs[a[i]] = 0; ls[a[i]] = 0 }`,
+ `}`,
+ `{`,
+ ` ts = $0; sub(/.*"ts":/, "", ts); sub(/[^0-9].*/, "", ts)`,
+ ` c = $0; sub(/.*"card":/, "", c); sub(/[^0-9].*/, "", c)`,
+ ` u = $0; sub(/.*"util":/, "", u); sub(/[^0-9.].*/, "", u)`,
+ ` h = $0; sub(/.*"hbm":/, "", h); sub(/[^0-9.].*/, "", h)`,
+ ` if (c == "") next`,
+ ` cnt[c]++`,
+ ` if (ts != "") { t = ts + 0; if (fs[c] == 0 || t < fs[c]) fs[c] = t; if (t > ls[c]) ls[c] = t }`,
+ ` if (u + 0 > pu[c]) pu[c] = u + 0`,
+ ` if (h + 0 > ph[c]) ph[c] = h + 0`,
+ ` su[c] += u + 0`,
+ `}`,
+ `END {`,
+ ` printf "{\\"v\\":\\"${NPU_METRICS_MARKER}\\",\\"ts\\":%d,\\"cards\\":[", now`,
+ ` sep = ""`,
+ ` for (c in cnt) {`,
+ ` au = cnt[c] > 0 ? su[c] / cnt[c] : 0`,
+ ` printf "%s{\\"c\\":\\"%s\\",\\"n\\":%d,\\"pu\\":%.1f,\\"au\\":%.1f,\\"ph\\":%.1f,\\"fs\\":%d,\\"ls\\":%d}", sep, c, cnt[c], pu[c], au, ph[c], fs[c], ls[c]`,
+ ` sep = ","`,
+ ` }`,
+ ` printf "]}\\n"`,
+ `}`
+].join('\n')
+
+export function buildNpuAggregateScript(): string {
+ return [
+ `# ${NPU_METRICS_MARKER} aggregate`,
+ `F=${NPU_SAMPLES_FILE}`,
+ `CARDS=$(${listNpuCardsCommand()})`,
+ `NOW=$(date +%s)`,
+ `touch "$F" 2>/dev/null || true`,
+ `awk -v now="$NOW" -v cards="$CARDS" '${AGGREGATE_AWK}' "$F" 2>/dev/null`
+ ].join('\n')
+}
+
+function commandHasMarker(command: string[] | undefined): boolean {
+ return (command ?? []).join(' ').includes(NPU_METRICS_MARKER)
+}
+
+function appendExecCommand(
+ existing: string[] | undefined,
+ appendLines: string[]
+): string[] {
+ if (
+ existing &&
+ existing.length >= 3 &&
+ /(^|\/)sh$/.test(existing[0]) &&
+ existing[1] === '-c'
+ ) {
+ return [
+ existing[0],
+ existing[1],
+ [...existing.slice(2), ...appendLines].join('\n')
+ ]
+ }
+ const prefix = (existing ?? []).map(
+ part => `'${part.replace(/'/g, "'\\''")}'`
+ )
+ return ['sh', '-c', [prefix.join(' '), ...appendLines].join('\n')]
+}
+
+export function injectNpuMetrics(
+ container: k8s.V1Container,
+ intervalSeconds = DEFAULT_NPU_SAMPLE_INTERVAL_SECONDS
+): boolean {
+ const lifecycle = container.lifecycle ?? {}
+ if (
+ commandHasMarker(lifecycle.postStart?.exec?.command) ||
+ commandHasMarker(lifecycle.preStop?.exec?.command)
+ ) {
+ return false
+ }
+
+ if (
+ (lifecycle.postStart && !lifecycle.postStart.exec) ||
+ (lifecycle.preStop && !lifecycle.preStop.exec)
+ ) {
+ core.warning(
+ 'npu-metrics: container already has a non-exec lifecycle handler, skip injection'
+ )
+ return false
+ }
+
+ const samplerLine = `(${buildNpuSamplerScript(intervalSeconds)}) >/dev/null 2>&1 &`
+ const aggregateLine = `{ ( ${buildNpuAggregateScript()} ) | head -c ${NPU_AGGREGATE_MAX_BYTES} > ${NPU_TERMINATION_LOG}; } 2>/dev/null || true`
+
+ lifecycle.postStart = {
+ exec: {
+ command: appendExecCommand(lifecycle.postStart?.exec?.command, [
+ `# ${NPU_METRICS_MARKER} postStart: start detached npu sampler`,
+ samplerLine
+ ])
+ }
+ }
+ lifecycle.preStop = {
+ exec: {
+ command: appendExecCommand(lifecycle.preStop?.exec?.command, [
+ `# ${NPU_METRICS_MARKER} preStop: write npu aggregate to termination log`,
+ aggregateLine
+ ])
+ }
+ }
+ container.lifecycle = lifecycle
+ return true
+}
+
+export function maybeInjectNpuMetrics(container: k8s.V1Container): boolean {
+ Iif (!npuMetricsEnabled()) {
+ return false
+ }
+ Eif (!containerHasNpuRequest(container)) {
+ return false
+ }
+ return injectNpuMetrics(container)
+}
+
+export function parseNpuAggregate(
+ raw: string | undefined
+): NpuAggregatePayload | undefined {
+ if (!raw) {
+ return undefined
+ }
+ const start = raw.indexOf('{"v":')
+ if (start === -1) {
+ return undefined
+ }
+ const candidates = [raw.slice(start)]
+ const end = raw.lastIndexOf('}')
+ if (end > start) {
+ candidates.push(raw.slice(start, end + 1))
+ }
+ for (const candidate of candidates) {
+ let parsed: any
+ try {
+ parsed = JSON.parse(candidate)
+ } catch {
+ continue
+ }
+ if (parsed?.v !== NPU_METRICS_MARKER || !Array.isArray(parsed.cards)) {
+ continue
+ }
+ const cards: NpuCardAggregate[] = []
+ for (const c of parsed.cards) {
+ if (c === null || typeof c !== 'object' || c.c === undefined) {
+ continue
+ }
+ cards.push({
+ card: String(c.c),
+ samples: Number(c.n) || 0,
+ peakUtil: Number(c.pu) || 0,
+ avgUtil: Number(c.au) || 0,
+ peakHbm: Number(c.ph) || 0,
+ firstTs: Number(c.fs) || 0,
+ lastTs: Number(c.ls) || 0
+ })
+ }
+ return { version: parsed.v, ts: Number(parsed.ts) || 0, cards }
+ }
+ return undefined
+}
+
+export function isIdleRecord(
+ cards: NpuCardAggregate[],
+ threshold = DEFAULT_NPU_IDLE_THRESHOLD_PERCENT
+): boolean {
+ if (!cards.length) {
+ return false
+ }
+ return cards.every(c => c.peakUtil < threshold && c.avgUtil < threshold)
+}
+
+export function escapePromLabelValue(value: string): string {
+ return value.replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/\n/g, '\\n')
+}
+
+export function formatPromLabels(labels: Record<string, string>): string {
+ return Object.entries(labels)
+ .filter(([, v]) => v !== undefined)
+ .map(([k, v]) => `${k}="${escapePromLabelValue(v)}"`)
+ .join(',')
+}
+
+const NPU_JOB_METRIC_LABEL_KEYS = [
+ 'cluster',
+ 'namespace',
+ 'repo',
+ 'run_id',
+ 'pod',
+ 'pod_type',
+ 'card',
+ 'npu_type',
+ 'cards_requested',
+ 'result',
+ 'idle'
+]
+
+export function buildNpuJobMetricsBody(
+ labels: NpuJobLabels,
+ cards: NpuCardAggregate[],
+ idleThreshold = DEFAULT_NPU_IDLE_THRESHOLD_PERCENT
+): string {
+ const base: Record<string, string> = {
+ cluster: labels.cluster,
+ namespace: labels.namespace,
+ repo: labels.repo,
+ run_id: labels.runId,
+ pod: labels.pod,
+ pod_type: labels.podType,
+ npu_type: labels.npuType,
+ cards_requested: labels.cardsRequested,
+ result: labels.result
+ }
+ const idle = isIdleRecord(cards, idleThreshold) ? 'true' : 'false'
+ const lines: string[] = []
+ lines.push('# TYPE custom_npu_job_peak_util_percent gauge')
+ lines.push('# TYPE custom_npu_job_avg_util_percent gauge')
+ lines.push('# TYPE custom_npu_job_duration_seconds gauge')
+ lines.push('# TYPE custom_npu_job_sample_count gauge')
+ for (const card of cards) {
+ const cardLabels = { ...base, card: card.card, idle }
+ const l = formatPromLabels(
+ Object.fromEntries(
+ NPU_JOB_METRIC_LABEL_KEYS.map(k => [k, cardLabels[k]])
+ ) as Record<string, string>
+ )
+ lines.push(`custom_npu_job_peak_util_percent{${l}} ${card.peakUtil}`)
+ lines.push(`custom_npu_job_avg_util_percent{${l}} ${card.avgUtil}`)
+ const duration = Math.max(card.lastTs - card.firstTs, 0)
+ lines.push(`custom_npu_job_duration_seconds{${l}} ${duration}`)
+ lines.push(`custom_npu_job_sample_count{${l}} ${card.samples}`)
+ }
+ lines.push('# TYPE custom_npu_job_push_total counter')
+ lines.push(
+ `custom_npu_job_push_total{cluster="${escapePromLabelValue(labels.cluster)}",namespace="${escapePromLabelValue(labels.namespace)}",status="success"} 1`
+ )
+ return lines.join('\n') + '\n'
+}
+
+export function buildNpuPushFailedBody(labels: NpuJobLabels): string {
+ return (
+ `# TYPE custom_npu_job_push_total counter\n` +
+ `custom_npu_job_push_total{cluster="${escapePromLabelValue(labels.cluster)}",namespace="${escapePromLabelValue(labels.namespace)}",status="failed"} 1\n`
+ )
+}
+
+export function buildPushgatewayPath(
+ cluster: string,
+ ns: string,
+ pod: string
+): string {
+ return `/metrics/job/${NPU_METRICS_PUSH_JOB}/cluster/${encodeURIComponent(
+ cluster
+ )}/ns/${encodeURIComponent(ns)}/pod/${encodeURIComponent(pod)}`
+}
+
+export async function withTimeout<T>(
+ promise: Promise<T>,
+ timeoutMs: number
+): Promise<T> {
+ return await new Promise<T>((resolve, reject) => {
+ const timer = setTimeout(() => reject(new Error('timeout')), timeoutMs)
+ promise.then(
+ value => {
+ clearTimeout(timer)
+ resolve(value)
+ },
+ err => {
+ clearTimeout(timer)
+ reject(err)
+ }
+ )
+ })
+}
+
+export interface PushRequest {
+ url: string
+ method: string
+ body: string
+ username?: string
+ password?: string
+ timeoutMs?: number
+}
+
+export type RequestFn = (req: PushRequest) => Promise<{ statusCode: number }>
+
+export async function nodeRequest(req: PushRequest): Promise<{
+ statusCode: number
+}> {
+ const target = new URL(req.url)
+ const client = target.protocol === 'http:' ? http : https
+ const headers: Record<string, string> = {
+ 'Content-Type': 'text/plain'
+ }
+ if (req.username !== undefined || req.password !== undefined) {
+ const basic = Buffer.from(
+ `${req.username ?? ''}:${req.password ?? ''}`
+ ).toString('base64')
+ headers['Authorization'] = `Basic ${basic}`
+ }
+ return await new Promise((resolve, reject) => {
+ const request = client.request(
+ target,
+ {
+ method: req.method,
+ headers,
+ timeout: req.timeoutMs ?? 5000,
+ rejectUnauthorized: false
+ },
+ response => {
+ response.resume()
+ response.on('end', () =>
+ resolve({ statusCode: response.statusCode ?? 0 })
+ )
+ }
+ )
+ request.on('timeout', () => request.destroy(new Error('request timeout')))
+ request.on('error', reject)
+ request.end(req.body)
+ })
+}
+
+export async function pushToPushgateway(
+ config: NpuPushConfig,
+ path: string,
+ body: string,
+ options?: { retries?: number; retryDelayMs?: number; requestFn?: RequestFn }
+): Promise<void> {
+ const retries = options?.retries ?? 2
+ const retryDelayMs = options?.retryDelayMs ?? 1000
+ const requestFn = options?.requestFn ?? nodeRequest
+ const url = `${config.url.replace(/\/+$/, '')}${path}`
+ let lastError: unknown
+ for (let attempt = 0; attempt <= retries; attempt++) {
+ if (attempt > 0) {
+ await new Promise(resolve => setTimeout(resolve, retryDelayMs))
+ }
+ try {
+ const response = await requestFn({
+ url,
+ method: 'POST',
+ body,
+ username: config.username,
+ password: config.password
+ })
+ if (response.statusCode >= 200 && response.statusCode < 300) {
+ return
+ }
+ lastError = new Error(`pushgateway returned ${response.statusCode}`)
+ } catch (err) {
+ lastError = err
+ }
+ }
+ throw lastError instanceof Error ? lastError : new Error(String(lastError))
+}
+
+export function decodePushSecret(
+ secret: k8s.V1Secret | undefined
+): NpuPushConfig | undefined {
+ const data = secret?.data ?? {}
+ const decode = (key: string): string | undefined => {
+ const encoded = data[key]
+ if (!encoded) {
+ return undefined
+ }
+ try {
+ const decoded = Buffer.from(encoded, 'base64').toString('utf8')
+ return decoded.length ? decoded : undefined
+ } catch {
+ return undefined
+ }
+ }
+ const url = decode('PUSHGATEWAY_URL')
+ if (!url) {
+ return undefined
+ }
+ return {
+ url,
+ username: decode('PUSHGATEWAY_USER'),
+ password: decode('PUSHGATEWAY_PASSWORD')
+ }
+}
+
+export async function readNpuPushConfig(): Promise<NpuPushConfig | undefined> {
+ try {
+ const secret = await getSecretByName(NPU_METRICS_SECRET_NAME)
+ return decodePushSecret(secret)
+ } catch (err) {
+ core.debug(
+ `npu-metrics: secret ${NPU_METRICS_SECRET_NAME} unavailable, skip push: ${formatError(err)}`
+ )
+ return undefined
+ }
+}
+
+function parseScaleSetSuffix(value: string | undefined): string | undefined {
+ if (!value) {
+ return undefined
+ }
+ const match = value.match(SCALE_SET_NAME_REGEX)
+ return match ? match[1] : undefined
+}
+
+export function extractClusterFromRunnerPod(pod: k8s.V1Pod): string {
+ const labels = pod.metadata?.labels ?? {}
+ const annotations = pod.metadata?.annotations ?? {}
+ const entries: [string, string][] = [
+ ...Object.entries(labels),
+ ...Object.entries(annotations)
+ ]
+ for (const [key, value] of entries) {
+ if (/scale[-_]?set|runner[-_]?deployment/i.test(key)) {
+ const suffix = parseScaleSetSuffix(value)
+ if (suffix) {
+ return suffix
+ }
+ }
+ }
+ for (const [, value] of entries) {
+ const suffix = parseScaleSetSuffix(value)
+ if (suffix) {
+ return suffix
+ }
+ }
+ return parseScaleSetSuffix(pod.metadata?.name) ?? 'unknown'
+}
+
+export function resolveNpuLabelsFromPod(
+ pod: k8s.V1Pod,
+ podName: string
+): {
+ podType: string
+ npuType: string
+ cardsRequested: string
+ result: string
+} {
+ const podType = podName.includes('-step-') ? 'step' : 'workflow'
+ const npuTypes = new Set<string>()
+ let cards = 0
+ for (const container of pod.spec?.containers ?? []) {
+ for (const resources of [
+ container.resources?.requests,
+ container.resources?.limits
+ ]) {
+ for (const [key, quantity] of Object.entries(resources ?? {})) {
+ const match = key.match(NPU_RESOURCE_REGEX)
+ if (match) {
+ npuTypes.add(match[1])
+ cards = Math.max(cards, parseInt(String(quantity), 10) || 0)
+ }
+ }
+ }
+ }
+ const mainStatus = (pod.status?.containerStatuses ?? []).find(
+ s => s.name === JOB_CONTAINER_NAME
+ )
+ const terminated = mainStatus?.state?.terminated
+ const result = terminated
+ ? String(terminated.exitCode)
+ : (pod.status?.phase ?? 'unknown')
+ return {
+ podType,
+ npuType: Array.from(npuTypes).sort().join(',') || 'unknown',
+ cardsRequested: String(cards || 0),
+ result
+ }
+}
+
+export function repoFromEnv(
+ env: Record<string, string | undefined> = process.env
+): string {
+ if (env.GITHUB_REPOSITORY) {
+ return env.GITHUB_REPOSITORY
+ }
+ const workspace = env.GITHUB_WORKSPACE ?? ''
+ const parts = workspace.split('/').filter(p => p.length > 0)
+ if (parts.length >= 2) {
+ return `${parts[parts.length - 2]}/${parts[parts.length - 1]}`
+ }
+ return 'unknown'
+}
+
+export async function collectNpuAggregateFromPod(
+ pod: k8s.V1Pod
+): Promise<NpuAggregatePayload | undefined> {
+ const podName = pod.metadata?.name
+ if (!podName) {
+ return undefined
+ }
+ const mainStatus = (pod.status?.containerStatuses ?? []).find(
+ s => s.name === JOB_CONTAINER_NAME
+ )
+ const terminatedMessage = mainStatus?.state?.terminated?.message
+ if (terminatedMessage) {
+ return parseNpuAggregate(terminatedMessage)
+ }
+ if (!mainStatus?.state?.running) {
+ return undefined
+ }
+ const { output } = await execPodStepWithOutput(
+ ['sh', '-c', buildNpuAggregateScript()],
+ podName,
+ JOB_CONTAINER_NAME,
+ 5
+ )
+ return parseNpuAggregate(output)
+}
+
+export async function collectAndPushNpuMetrics(): Promise<void> {
+ Iif (!npuMetricsEnabled()) {
+ return
+ }
+ let pods: k8s.V1Pod[] = []
+ try {
+ pods = await listPodsByRunnerInstance()
+ } catch (err) {
+ core.debug(`npu-metrics: pod list failed: ${formatError(err)}`)
+ return
+ }
+ const npuPods = pods.filter(
+ pod => pod.metadata?.name && hasNpuRequest(pod.spec)
+ )
+ if (!npuPods.length) {
+ return
+ }
+
+ let cluster = 'unknown'
+ try {
+ const runnerPod = await getPodByName(getRunnerPodName())
+ cluster = extractClusterFromRunnerPod(runnerPod)
+ } catch (err) {
+ core.debug(`npu-metrics: cluster label unresolved: ${formatError(err)}`)
+ }
+
+ const config = await readNpuPushConfig()
+ if (!config) {
+ core.debug(
+ `npu-metrics: push config missing (secret ${NPU_METRICS_SECRET_NAME}), skip push`
+ )
+ return
+ }
+
+ for (const pod of npuPods) {
+ const podName = pod.metadata?.name as string
+ try {
+ await collectAndPushForPod(pod, podName, cluster, config)
+ } catch (err) {
+ core.debug(
+ `npu-metrics: collection failed for ${podName}: ${formatError(err)}`
+ )
+ }
+ }
+}
+
+async function collectAndPushForPod(
+ pod: k8s.V1Pod,
+ podName: string,
+ cluster: string,
+ config: NpuPushConfig
+): Promise<void> {
+ const payload = await collectNpuAggregateFromPod(pod)
+ if (!payload || !payload.cards.length) {
+ core.debug(`npu-metrics: no aggregate payload for ${podName}, skip`)
+ return
+ }
+
+ const npuInfo = resolveNpuLabelsFromPod(pod, podName)
+ const labels: NpuJobLabels = {
+ cluster,
+ namespace: namespace(),
+ pod: podName,
+ podType: npuInfo.podType,
+ repo: repoFromEnv(),
+ runId: process.env.GITHUB_RUN_ID ?? '',
+ npuType: npuInfo.npuType,
+ cardsRequested: npuInfo.cardsRequested,
+ result: npuInfo.result
+ }
+
+ const body = buildNpuJobMetricsBody(
+ labels,
+ payload.cards,
+ getIdleThresholdPercent()
+ )
+ const path = buildPushgatewayPath(labels.cluster, labels.namespace, podName)
+ try {
+ await pushToPushgateway(config, path, body)
+ core.debug(`npu-metrics: pushed job record for ${podName}`)
+ } catch (err) {
+ core.warning(`npu-metrics: push failed for ${podName}: ${formatError(err)}`)
+ try {
+ await pushToPushgateway(config, path, buildNpuPushFailedBody(labels))
+ } catch {
+ // pushgateway unreachable, nothing else to record
+ }
+ }
+}
+ |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| File | ++ | Statements | ++ | Branches | ++ | Functions | ++ | Lines | ++ |
|---|---|---|---|---|---|---|---|---|---|
| test-setup.ts | +
+
+ |
+ 33.82% | +23/68 | +0% | +0/7 | +14.28% | +2/14 | +32.83% | +22/67 | +
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 +33 +34 +35 +36 +37 +38 +39 +40 +41 +42 +43 +44 +45 +46 +47 +48 +49 +50 +51 +52 +53 +54 +55 +56 +57 +58 +59 +60 +61 +62 +63 +64 +65 +66 +67 +68 +69 +70 +71 +72 +73 +74 +75 +76 +77 +78 +79 +80 +81 +82 +83 +84 +85 +86 +87 +88 +89 +90 +91 +92 +93 +94 +95 +96 +97 +98 +99 +100 +101 +102 +103 +104 +105 +106 +107 +108 +109 +110 +111 +112 +113 +114 +115 +116 +117 +118 +119 +120 +121 +122 +123 +124 +125 +126 +127 +128 +129 +130 +131 +132 +133 +134 +135 +136 +137 +138 +139 +140 +141 +142 +143 +144 +145 +146 +147 +148 +149 +150 +151 +152 +153 +154 +155 +156 +157 +158 +159 +160 +161 +162 +163 +164 +165 +166 | 6x +6x + +6x +6x + +6x + +6x + +6x + +6x + + + + + + +46x +46x +46x +46x + + +6x + + + + + + + + + + + + + + + + + + + + + + + + + + + + +6x + + + + + + + + +6x + + + + + + + + + + + + + + + + + + + + + + + +6x + + + + + +6x + + + + +6x + + + + + + + + + + + + + + + + + + + + + + + +6x + + + + + + + + + + + + + + + + +6x + + + + + + + + + + + +6x + + + + + + + + + + + + +6x + | import * as k8s from '@kubernetes/client-node'
+import * as fs from 'fs'
+import { HookData } from 'hooklib/lib'
+import * as path from 'path'
+import { v4 as uuidv4 } from 'uuid'
+
+const kc = new k8s.KubeConfig()
+
+kc.loadFromDefault()
+
+const k8sApi = kc.makeApiClient(k8s.CoreV1Api)
+
+export class TestHelper {
+ private tempDirPath: string
+ private podName: string
+ private runnerWorkdir: string
+ private runnerTemp: string
+
+ constructor() {
+ this.tempDirPath = `${__dirname}/_temp/runner`
+ this.runnerWorkdir = `${this.tempDirPath}/_work`
+ this.runnerTemp = `${this.tempDirPath}/_work/_temp`
+ this.podName = uuidv4().replace(/-/g, '')
+ }
+
+ async initialize(): Promise<void> {
+ process.env['ACTIONS_RUNNER_POD_NAME'] = `${this.podName}`
+ process.env['RUNNER_WORKSPACE'] = `${this.runnerWorkdir}/repo`
+ process.env['RUNNER_TEMP'] = `${this.runnerTemp}`
+ process.env['GITHUB_WORKSPACE'] = `${this.runnerWorkdir}/repo/repo`
+ process.env['ACTIONS_RUNNER_KUBERNETES_NAMESPACE'] = 'default'
+
+ fs.mkdirSync(`${this.runnerWorkdir}/repo/repo`, { recursive: true })
+ fs.mkdirSync(`${this.tempDirPath}/externals`, { recursive: true })
+ fs.mkdirSync(this.runnerTemp, { recursive: true })
+ fs.mkdirSync(`${this.runnerTemp}/_github_workflow`, { recursive: true })
+ fs.mkdirSync(`${this.runnerTemp}/_github_home`, { recursive: true })
+ fs.mkdirSync(`${this.runnerTemp}/_runner_file_commands`, {
+ recursive: true
+ })
+
+ fs.copyFileSync(
+ path.resolve(`${__dirname}/../../../examples/example-script.sh`),
+ `${this.runnerTemp}/example-script.sh`
+ )
+
+ await this.cleanupK8sResources()
+ try {
+ await this.createTestJobPod()
+ } catch (e) {
+ console.log(e)
+ }
+ }
+
+ async cleanup(): Promise<void> {
+ try {
+ await this.cleanupK8sResources()
+ fs.rmSync(this.tempDirPath, { recursive: true })
+ } catch {
+ // Ignore errors during cleanup
+ }
+ }
+
+ async cleanupK8sResources(): Promise<void> {
+ await k8sApi
+ .deleteNamespacedPod({
+ name: this.podName,
+ namespace: 'default',
+ gracePeriodSeconds: 0
+ })
+ .catch((e: k8s.ApiException<any>) => {
+ if (e.code !== 404) {
+ console.error(JSON.stringify(e))
+ }
+ })
+ await k8sApi
+ .deleteNamespacedPod({
+ name: `${this.podName}-workflow`,
+ namespace: 'default',
+ gracePeriodSeconds: 0
+ })
+ .catch((e: k8s.ApiException<any>) => {
+ if (e.code !== 404) {
+ console.error(JSON.stringify(e))
+ }
+ })
+ }
+ createFile(fileName?: string): string {
+ const filePath = `${this.tempDirPath}/${fileName || uuidv4()}`
+ fs.writeFileSync(filePath, '')
+ return filePath
+ }
+
+ removeFile(fileName: string): void {
+ const filePath = `${this.tempDirPath}/${fileName}`
+ fs.rmSync(filePath)
+ }
+
+ async createTestJobPod(): Promise<void> {
+ const container = {
+ name: 'runner',
+ image: 'ghcr.io/actions/actions-runner:latest',
+ imagePullPolicy: 'IfNotPresent'
+ } as k8s.V1Container
+
+ const pod: k8s.V1Pod = {
+ metadata: {
+ name: this.podName
+ },
+ spec: {
+ restartPolicy: 'Never',
+ containers: [container],
+ securityContext: {
+ runAsUser: 1001,
+ runAsGroup: 1001,
+ fsGroup: 1001
+ }
+ }
+ } as k8s.V1Pod
+ await k8sApi.createNamespacedPod({ namespace: 'default', body: pod })
+ }
+
+ getPrepareJobDefinition(): HookData {
+ const prepareJob = JSON.parse(
+ fs.readFileSync(
+ path.resolve(__dirname + '/../../../examples/prepare-job.json'),
+ 'utf8'
+ )
+ )
+
+ prepareJob.args.container.userMountVolumes = undefined
+ prepareJob.args.container.registry = null
+ prepareJob.args.services.forEach(s => {
+ s.registry = null
+ })
+
+ return prepareJob
+ }
+
+ getRunScriptStepDefinition(): HookData {
+ const runScriptStep = JSON.parse(
+ fs.readFileSync(
+ path.resolve(__dirname + '/../../../examples/run-script-step.json'),
+ 'utf8'
+ )
+ )
+
+ runScriptStep.args.entryPointArgs[1] = `/__w/_temp/example-script.sh`
+ return runScriptStep
+ }
+
+ getRunContainerStepDefinition(): HookData {
+ const runContainerStep = JSON.parse(
+ fs.readFileSync(
+ path.resolve(__dirname + '/../../../examples/run-container-step.json'),
+ 'utf8'
+ )
+ )
+
+ runContainerStep.args.entryPointArgs[1] = `/__w/_temp/example-script.sh`
+ runContainerStep.args.userMountVolumes = undefined
+ runContainerStep.args.registry = null
+ return runContainerStep
+ }
+}
+ |