From 4c151d719f1c48ba1a73a80ed5ce77cf95ed689a Mon Sep 17 00:00:00 2001 From: Zachary Boogher Date: Wed, 24 Jun 2026 16:31:55 -0400 Subject: [PATCH 1/4] feature: add NinjaRMM runaway script-log purge tool Reclaims disk from runaway custom-script output transcripts under the NinjaRMM agent scripting directory. Scans for *-output.txt over a configurable threshold (default 500MB), kills the writing process if still active, and hard-deletes the file. Dual-mode (interactive + RMM), runs silently with no required input. Self-protecting: excludes its own PID from process kills and treats a locked file as a counted failure rather than silent success. 64-bit relaunch guard for NinjaRMM's 32-bit default host. Tested both modes on a workstation: EXIT 0 nothing-to-clean. --- rmm-ninja/ninjarmm-purge-runaway-logs.ps1 | 127 ++++++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 rmm-ninja/ninjarmm-purge-runaway-logs.ps1 diff --git a/rmm-ninja/ninjarmm-purge-runaway-logs.ps1 b/rmm-ninja/ninjarmm-purge-runaway-logs.ps1 new file mode 100644 index 0000000..876b052 --- /dev/null +++ b/rmm-ninja/ninjarmm-purge-runaway-logs.ps1 @@ -0,0 +1,127 @@ +# --- Ensure 64-bit: NinjaRMM launches 32-bit PowerShell by default. Relaunch under +# native sysnative host so behavior is identical interactive vs RMM. --- +if ($env:PROCESSOR_ARCHITEW6432 -eq "AMD64" -and -not [Environment]::Is64BitProcess) { + $sysnative = "$env:WINDIR\sysnative\WindowsPowerShell\v1.0\powershell.exe" + if (Test-Path -LiteralPath $sysnative) { + & $sysnative -NoProfile -ExecutionPolicy Bypass -File $PSCommandPath + exit $LASTEXITCODE + } +} + +## PLEASE COMMENT YOUR VARIABLES DIRECTLY BELOW HERE IF YOU'RE RUNNING FROM A RMM +## THIS IS HOW WE EASILY LET PEOPLE KNOW WHAT VARIABLES NEED SET IN THE RMM +## $env:RMM - "1" when executed from the RMM (string). Anything else = interactive. +## $env:Description - optional audit label for the transcript (default set below). Ninja +## already captures operator/ticket at the platform level, so no prompt. +## $env:ThresholdMB - size floor in MB for a file to be considered runaway (optional, default 500) +## +## Purpose: Reclaim disk consumed by runaway NinjaRMM custom-script output transcripts +## under the agent scripting directory. Kills the writing process (if any) then +## hard-deletes the oversized "*-output.txt" file. Self-protecting: never matches, +## kills, or deletes the process or files belonging to THIS cleanup run. +## Runs silently with no required input in either mode. +## +## Exit codes: 0 = completed (zero or more files cleaned, no failures) +## 1 = completed with one or more deletion failures (see transcript) +## 2 = scripting directory not found / nothing to scan + +$ScriptLogName = "ninjarmm-purge-runaway-logs.log" + +# --- Resolve ThresholdMB from RMM env var; default 500, floor of 1 --- +$ThresholdMB = 500 +$parsedMB = 0 +if ($env:ThresholdMB -and [int]::TryParse($env:ThresholdMB, [ref]$parsedMB) -and $parsedMB -ge 1) { + $ThresholdMB = $parsedMB +} +$thresholdBytes = [int64]$ThresholdMB * 1MB + +# --- Input handling: no required user input. Default Description, set log path by context. --- +if ([string]::IsNullOrEmpty($env:Description)) { + $env:Description = "Automated runaway-log purge" +} + +if ($env:RMM -eq "1") { + if (-not [string]::IsNullOrEmpty($env:RMMScriptPath)) { + $LogPath = "$env:RMMScriptPath\logs\$ScriptLogName" + } else { + $LogPath = "$env:WINDIR\logs\$ScriptLogName" + } +} else { + $LogPath = "$env:WINDIR\logs\$ScriptLogName" +} + +Start-Transcript -Path $LogPath + +Write-Host "============ Purge Runaway NinjaRMM Script Logs ============" +Write-Host "Description : $env:Description" +Write-Host "Log path : $LogPath" +Write-Host "RMM mode : $env:RMM" +Write-Host "Threshold : $ThresholdMB MB ($thresholdBytes bytes)" +Write-Host "===========================================================" + +function Invoke-RunawayLogPurge { + param([int64]$ThresholdBytes) + + $scriptingDir = "C:\ProgramData\NinjaRMMAgent\scripting" + $myPid = $PID + + Write-Host "[1/4] Validating scripting directory: $scriptingDir" + if (-not (Test-Path -LiteralPath $scriptingDir)) { + Write-Host "RESULT: scripting directory not found -> nothing to do -> EXIT 2" + return 2 + } + + Write-Host "[2/4] Scanning for *-output.txt over threshold ..." + $candidates = Get-ChildItem -LiteralPath $scriptingDir -Filter *output.txt -File -ErrorAction SilentlyContinue | + Where-Object { $_.Length -gt $ThresholdBytes } + + if (-not $candidates) { + Write-Host "RESULT: no files over $([math]::Round($ThresholdBytes/1MB)) MB -> nothing to clean -> EXIT 0" + return 0 + } + + Write-Host " Found $($candidates.Count) candidate file(s):" + $candidates | ForEach-Object { Write-Host " $($_.Name) [$([math]::Round($_.Length/1GB,2)) GB]" } + + $failures = 0 + foreach ($f in $candidates) { + $genId = ($f.Name -split '\.ps1')[0] + Write-Host "[3/4] Processing $($f.Name) (gen id: $genId)" + + # Kill the writing process if one is still spinning. Anchor on the scripting + # path AND the gen id, and NEVER match our own PID. + $writers = Get-CimInstance Win32_Process -Filter "Name='powershell.exe' OR Name='cmd.exe'" -ErrorAction SilentlyContinue | + Where-Object { + $_.ProcessId -ne $myPid -and + $_.CommandLine -like "*$scriptingDir*" -and + $_.CommandLine -like "*$genId*" + } + foreach ($w in $writers) { + Write-Host " Killing writer PID $($w.ProcessId)" + try { Stop-Process -Id $w.ProcessId -Force -ErrorAction Stop } + catch { Write-Host " WARN: could not kill PID $($w.ProcessId): $($_.Exception.Message)" } + } + + Write-Host "[4/4] Deleting $($f.FullName) [$([math]::Round($f.Length/1GB,2)) GB]" + try { + Remove-Item -LiteralPath $f.FullName -Force -ErrorAction Stop + Write-Host " Deleted." + } catch { + Write-Host " ERROR: could not delete (file locked or in use by this run): $($_.Exception.Message)" + $failures++ + } + } + + if ($failures -gt 0) { + Write-Host "RESULT: completed with $failures deletion failure(s) -> EXIT 1" + return 1 + } + Write-Host "RESULT: all candidates cleaned -> EXIT 0" + return 0 +} + +$exitCode = Invoke-RunawayLogPurge -ThresholdBytes $thresholdBytes + +Write-Host "Final exit code: $exitCode" +Stop-Transcript +exit $exitCode \ No newline at end of file From 0d4c4483c9c832a11e75446046466ba51365a318 Mon Sep 17 00:00:00 2001 From: Zachary Boogher Date: Wed, 24 Jun 2026 17:26:53 -0400 Subject: [PATCH 2/4] refactor: replace string-match kill with Restart Manager lock detection The initial command-line substring match failed to identify the writer on a live runaway (multi-process cmd/powershell tree, GUID-style customscript_src_ filename). Replaced with Windows Restart Manager (rstrtmgr.dll) interop to query the exact PIDs holding each oversized output file, with a hard guard that never terminates the NinjaRMM agent or this script's own process. Added 3x delete retry for non-instant handle release. Fixes $hpid parse error in kill-failure path. Validated destructive path on a live 9.3GB runaway: holders identified, terminated, file deleted, EXIT 0. --- rmm-ninja/ninjarmm-purge-runaway-logs.ps1 | 132 +++++++++++++++++----- 1 file changed, 105 insertions(+), 27 deletions(-) diff --git a/rmm-ninja/ninjarmm-purge-runaway-logs.ps1 b/rmm-ninja/ninjarmm-purge-runaway-logs.ps1 index 876b052..57da2f3 100644 --- a/rmm-ninja/ninjarmm-purge-runaway-logs.ps1 +++ b/rmm-ninja/ninjarmm-purge-runaway-logs.ps1 @@ -15,14 +15,15 @@ if ($env:PROCESSOR_ARCHITEW6432 -eq "AMD64" -and -not [Environment]::Is64BitProc ## already captures operator/ticket at the platform level, so no prompt. ## $env:ThresholdMB - size floor in MB for a file to be considered runaway (optional, default 500) ## -## Purpose: Reclaim disk consumed by runaway NinjaRMM custom-script output transcripts -## under the agent scripting directory. Kills the writing process (if any) then -## hard-deletes the oversized "*-output.txt" file. Self-protecting: never matches, -## kills, or deletes the process or files belonging to THIS cleanup run. -## Runs silently with no required input in either mode. +## Purpose: Reclaim disk consumed by runaway NinjaRMM custom-script output transcripts under +## the agent scripting directory. Uses the Windows Restart Manager to identify the +## exact process(es) holding each oversized "*-output.txt" file open, terminates them +## (never the NinjaRMM agent, never this run), then deletes the file. +## Runs silently with no required input in either mode. Fleet-safe across workstations +## and servers: fixed agent path, OS-level lock detection, no filename-pattern assumptions. ## ## Exit codes: 0 = completed (zero or more files cleaned, no failures) -## 1 = completed with one or more deletion failures (see transcript) +## 1 = completed with one or more files that could not be freed/deleted ## 2 = scripting directory not found / nothing to scan $ScriptLogName = "ninjarmm-purge-runaway-logs.log" @@ -59,11 +60,66 @@ Write-Host "RMM mode : $env:RMM" Write-Host "Threshold : $ThresholdMB MB ($thresholdBytes bytes)" Write-Host "===========================================================" +# --- Restart Manager interop: ask Windows which PIDs hold a file open. Dependency-free, +# works on every Windows version (rstrtmgr.dll, XP+). Added once per session. --- +if (-not ([System.Management.Automation.PSTypeName]'DtcFileLock').Type) { + Add-Type -ErrorAction Stop -TypeDefinition @" +using System; +using System.Collections.Generic; +using System.Runtime.InteropServices; +public static class DtcFileLock { + [StructLayout(LayoutKind.Sequential)] + struct RM_UNIQUE_PROCESS { public int dwProcessId; public System.Runtime.InteropServices.ComTypes.FILETIME ProcessStartTime; } + const int CCH_RM_MAX_APP_NAME = 255; + const int CCH_RM_MAX_SVC_NAME = 63; + enum RM_APP_TYPE { RmUnknownApp=0, RmMainWindow=1, RmOtherWindow=2, RmService=3, RmExplorer=4, RmConsole=5, RmCritical=1000 } + [StructLayout(LayoutKind.Sequential, CharSet=CharSet.Unicode)] + struct RM_PROCESS_INFO { + public RM_UNIQUE_PROCESS Process; + [MarshalAs(UnmanagedType.ByValTStr, SizeConst=CCH_RM_MAX_APP_NAME+1)] public string strAppName; + [MarshalAs(UnmanagedType.ByValTStr, SizeConst=CCH_RM_MAX_SVC_NAME+1)] public string strServiceShortName; + public RM_APP_TYPE ApplicationType; + public uint AppStatus; + public uint TSSessionId; + [MarshalAs(UnmanagedType.Bool)] public bool bRestartable; + } + [DllImport("rstrtmgr.dll", CharSet=CharSet.Unicode)] + static extern int RmStartSession(out uint pSessionHandle, int dwSessionFlags, string strSessionKey); + [DllImport("rstrtmgr.dll")] + static extern int RmEndSession(uint pSessionHandle); + [DllImport("rstrtmgr.dll", CharSet=CharSet.Unicode)] + static extern int RmRegisterResources(uint pSessionHandle, uint nFiles, string[] rgsFilenames, uint nApplications, [In] RM_UNIQUE_PROCESS[] rgApplications, uint nServices, string[] rgsServiceNames); + [DllImport("rstrtmgr.dll")] + static extern int RmGetList(uint dwSessionHandle, out uint pnProcInfoNeeded, ref uint pnProcInfo, [In, Out] RM_PROCESS_INFO[] rgAffectedApps, ref uint lpdwRebReasons); + public static List WhoHas(string path) { + var pids = new List(); + uint session; string key = Guid.NewGuid().ToString(); + if (RmStartSession(out session, 0, key) != 0) return pids; + try { + string[] resources = { path }; + if (RmRegisterResources(session, 1, resources, 0, null, 0, null) != 0) return pids; + uint needed = 0, count = 0, reason = 0; + int r = RmGetList(session, out needed, ref count, null, ref reason); + if (r == 234 && needed > 0) { + var info = new RM_PROCESS_INFO[needed]; + count = needed; + if (RmGetList(session, out needed, ref count, info, ref reason) == 0) { + for (int i = 0; i < count; i++) pids.Add(info[i].Process.dwProcessId); + } + } + } finally { RmEndSession(session); } + return pids; + } +} +"@ +} + function Invoke-RunawayLogPurge { param([int64]$ThresholdBytes) $scriptingDir = "C:\ProgramData\NinjaRMMAgent\scripting" $myPid = $PID + $protectedNames = @('NinjaRMMAgent', 'ninjarmm-agent', 'ninjarmm-cli') Write-Host "[1/4] Validating scripting directory: $scriptingDir" if (-not (Test-Path -LiteralPath $scriptingDir)) { @@ -85,35 +141,57 @@ function Invoke-RunawayLogPurge { $failures = 0 foreach ($f in $candidates) { - $genId = ($f.Name -split '\.ps1')[0] - Write-Host "[3/4] Processing $($f.Name) (gen id: $genId)" - - # Kill the writing process if one is still spinning. Anchor on the scripting - # path AND the gen id, and NEVER match our own PID. - $writers = Get-CimInstance Win32_Process -Filter "Name='powershell.exe' OR Name='cmd.exe'" -ErrorAction SilentlyContinue | - Where-Object { - $_.ProcessId -ne $myPid -and - $_.CommandLine -like "*$scriptingDir*" -and - $_.CommandLine -like "*$genId*" + Write-Host "[3/4] Processing $($f.Name) [$([math]::Round($f.Length/1GB,2)) GB]" + + # Ask Windows directly which PIDs hold this file open. + $holders = @() + try { $holders = [DtcFileLock]::WhoHas($f.FullName) } catch { + Write-Host " WARN: Restart Manager query failed: $($_.Exception.Message)" + } + + if ($holders.Count -gt 0) { + Write-Host " Lock holders (PIDs): $($holders -join ', ')" + foreach ($hpid in $holders) { + if ($hpid -eq $myPid -or $hpid -le 4) { + Write-Host " Skipping PID $hpid (self or system)" + continue + } + $proc = Get-Process -Id $hpid -ErrorAction SilentlyContinue + if (-not $proc) { continue } + if ($protectedNames -contains $proc.Name) { + Write-Host " PROTECTED: PID $hpid is '$($proc.Name)' (NinjaRMM agent) - will NOT kill. File held by agent; stop the source automation in NinjaOne, then re-run." + continue + } + Write-Host " Killing PID $hpid ($($proc.Name))" + try { Stop-Process -Id $hpid -Force -ErrorAction Stop } + catch { Write-Host " WARN: could not kill PID ${hpid}: $($_.Exception.Message)" } } - foreach ($w in $writers) { - Write-Host " Killing writer PID $($w.ProcessId)" - try { Stop-Process -Id $w.ProcessId -Force -ErrorAction Stop } - catch { Write-Host " WARN: could not kill PID $($w.ProcessId): $($_.Exception.Message)" } + } else { + Write-Host " No lock holders reported (file may be unlocked, or held by a protected/system handle)." } - Write-Host "[4/4] Deleting $($f.FullName) [$([math]::Round($f.Length/1GB,2)) GB]" - try { - Remove-Item -LiteralPath $f.FullName -Force -ErrorAction Stop - Write-Host " Deleted." - } catch { - Write-Host " ERROR: could not delete (file locked or in use by this run): $($_.Exception.Message)" + # Delete with brief retry: handle release after a kill is not always instantaneous. + Write-Host "[4/4] Deleting $($f.FullName)" + $deleted = $false + for ($attempt = 1; $attempt -le 3; $attempt++) { + try { + Remove-Item -LiteralPath $f.FullName -Force -ErrorAction Stop + $deleted = $true + Write-Host " Deleted on attempt $attempt." + break + } catch { + Write-Host " Attempt $attempt failed: $($_.Exception.Message)" + Start-Sleep -Seconds 2 + } + } + if (-not $deleted) { + Write-Host " ERROR: could not delete after 3 attempts (still locked). Counted as failure." $failures++ } } if ($failures -gt 0) { - Write-Host "RESULT: completed with $failures deletion failure(s) -> EXIT 1" + Write-Host "RESULT: completed with $failures file(s) not freed -> EXIT 1" return 1 } Write-Host "RESULT: all candidates cleaned -> EXIT 0" From 3441352fa971890b89032f8990303425ad87ab87 Mon Sep 17 00:00:00 2001 From: Zachary Boogher Date: Wed, 24 Jun 2026 18:12:27 -0400 Subject: [PATCH 3/4] improvement: detect runaways by output growth and runtime ceiling The threshold-only scan missed freshly-respawned runaways still small at scan time (confirmed: tool reported nothing over 500MB while a NinjaRMM customscript spun a Read-Host NonInteractive loop into its output). Added a 3s growth-sampling pass: a file is a runaway if over ThresholdMB, OR growing while held by a NinjaRMM customscript process, OR its holder has run past RuntimeCeilingMin (default 30, backstop). Holder lookup factored into Get-ScriptHolder with proper error handling. Restart Manager kill, agent self-protection, and 3x delete retry unchanged. --- rmm-ninja/ninjarmm-purge-runaway-logs.ps1 | 149 ++++++++++++++++------ 1 file changed, 111 insertions(+), 38 deletions(-) diff --git a/rmm-ninja/ninjarmm-purge-runaway-logs.ps1 b/rmm-ninja/ninjarmm-purge-runaway-logs.ps1 index 57da2f3..593e127 100644 --- a/rmm-ninja/ninjarmm-purge-runaway-logs.ps1 +++ b/rmm-ninja/ninjarmm-purge-runaway-logs.ps1 @@ -10,25 +10,29 @@ if ($env:PROCESSOR_ARCHITEW6432 -eq "AMD64" -and -not [Environment]::Is64BitProc ## PLEASE COMMENT YOUR VARIABLES DIRECTLY BELOW HERE IF YOU'RE RUNNING FROM A RMM ## THIS IS HOW WE EASILY LET PEOPLE KNOW WHAT VARIABLES NEED SET IN THE RMM -## $env:RMM - "1" when executed from the RMM (string). Anything else = interactive. -## $env:Description - optional audit label for the transcript (default set below). Ninja -## already captures operator/ticket at the platform level, so no prompt. -## $env:ThresholdMB - size floor in MB for a file to be considered runaway (optional, default 500) +## $env:RMM - "1" when executed from the RMM (string). Anything else = interactive. +## $env:Description - optional audit label for the transcript (default set below). +## $env:ThresholdMB - size floor in MB for an output file to be a runaway (optional, default 500) +## $env:RuntimeCeilingMin - backstop: kill any NinjaRMM customscript process older than this many +## minutes regardless of file size (optional, default 30) ## -## Purpose: Reclaim disk consumed by runaway NinjaRMM custom-script output transcripts under -## the agent scripting directory. Uses the Windows Restart Manager to identify the -## exact process(es) holding each oversized "*-output.txt" file open, terminates them -## (never the NinjaRMM agent, never this run), then deletes the file. -## Runs silently with no required input in either mode. Fleet-safe across workstations -## and servers: fixed agent path, OS-level lock detection, no filename-pattern assumptions. +## Purpose: Reclaim disk consumed by runaway NinjaRMM custom-script output transcripts under the +## agent scripting directory, and stop the processes producing them. A file is a runaway if: +## (a) it is over ThresholdMB, OR +## (b) it is actively growing during a short sample window while a NinjaRMM customscript +## process holds it open, OR +## (c) the holding process has run longer than RuntimeCeilingMin (backstop). +## Uses the Windows Restart Manager to find the exact PIDs holding each file, terminates +## them (never the NinjaRMM agent, never this run), then deletes the file. Fleet-safe: +## fixed agent path, OS-level lock detection, no filename-pattern assumptions. ## -## Exit codes: 0 = completed (zero or more files cleaned, no failures) +## Exit codes: 0 = completed (zero or more cleaned, no failures) ## 1 = completed with one or more files that could not be freed/deleted ## 2 = scripting directory not found / nothing to scan $ScriptLogName = "ninjarmm-purge-runaway-logs.log" -# --- Resolve ThresholdMB from RMM env var; default 500, floor of 1 --- +# --- Resolve optional numeric RMM env vars with defaults and floors --- $ThresholdMB = 500 $parsedMB = 0 if ($env:ThresholdMB -and [int]::TryParse($env:ThresholdMB, [ref]$parsedMB) -and $parsedMB -ge 1) { @@ -36,6 +40,12 @@ if ($env:ThresholdMB -and [int]::TryParse($env:ThresholdMB, [ref]$parsedMB) -and } $thresholdBytes = [int64]$ThresholdMB * 1MB +$RuntimeCeilingMin = 30 +$parsedMin = 0 +if ($env:RuntimeCeilingMin -and [int]::TryParse($env:RuntimeCeilingMin, [ref]$parsedMin) -and $parsedMin -ge 1) { + $RuntimeCeilingMin = $parsedMin +} + # --- Input handling: no required user input. Default Description, set log path by context. --- if ([string]::IsNullOrEmpty($env:Description)) { $env:Description = "Automated runaway-log purge" @@ -54,10 +64,11 @@ if ($env:RMM -eq "1") { Start-Transcript -Path $LogPath Write-Host "============ Purge Runaway NinjaRMM Script Logs ============" -Write-Host "Description : $env:Description" -Write-Host "Log path : $LogPath" -Write-Host "RMM mode : $env:RMM" -Write-Host "Threshold : $ThresholdMB MB ($thresholdBytes bytes)" +Write-Host "Description : $env:Description" +Write-Host "Log path : $LogPath" +Write-Host "RMM mode : $env:RMM" +Write-Host "Threshold : $ThresholdMB MB ($thresholdBytes bytes)" +Write-Host "Runtime ceiling : $RuntimeCeilingMin min (backstop)" Write-Host "===========================================================" # --- Restart Manager interop: ask Windows which PIDs hold a file open. Dependency-free, @@ -114,40 +125,104 @@ public static class DtcFileLock { "@ } +function Get-ScriptHolder { + param([string]$Path, [string]$ScriptingDir) + $result = @() + try { + $holders = [DtcFileLock]::WhoHas($Path) + foreach ($hpid in $holders) { + $p = Get-CimInstance Win32_Process -Filter "ProcessId=$hpid" -ErrorAction SilentlyContinue + if ($p -and $p.CommandLine -like "*$ScriptingDir*") { + $result += $p + } + } + } catch { + Write-Host " WARN: RM query failed during detection: $($_.Exception.Message)" + } + return $result +} + function Invoke-RunawayLogPurge { - param([int64]$ThresholdBytes) + param([int64]$ThresholdBytes, [int]$RuntimeCeilingMin) $scriptingDir = "C:\ProgramData\NinjaRMMAgent\scripting" $myPid = $PID $protectedNames = @('NinjaRMMAgent', 'ninjarmm-agent', 'ninjarmm-cli') + $growthSampleSec = 3 - Write-Host "[1/4] Validating scripting directory: $scriptingDir" + Write-Host "[1/5] Validating scripting directory: $scriptingDir" if (-not (Test-Path -LiteralPath $scriptingDir)) { Write-Host "RESULT: scripting directory not found -> nothing to do -> EXIT 2" return 2 } - Write-Host "[2/4] Scanning for *-output.txt over threshold ..." - $candidates = Get-ChildItem -LiteralPath $scriptingDir -Filter *output.txt -File -ErrorAction SilentlyContinue | - Where-Object { $_.Length -gt $ThresholdBytes } + Write-Host "[2/5] First pass: snapshot all *output.txt sizes ..." + $first = @{} + Get-ChildItem -LiteralPath $scriptingDir -Filter *output.txt -File -ErrorAction SilentlyContinue | + ForEach-Object { $first[$_.FullName] = $_.Length } - if (-not $candidates) { - Write-Host "RESULT: no files over $([math]::Round($ThresholdBytes/1MB)) MB -> nothing to clean -> EXIT 0" + if ($first.Count -eq 0) { + Write-Host "RESULT: no output files present -> nothing to clean -> EXIT 0" return 0 } - Write-Host " Found $($candidates.Count) candidate file(s):" - $candidates | ForEach-Object { Write-Host " $($_.Name) [$([math]::Round($_.Length/1GB,2)) GB]" } + Write-Host "[3/5] Sampling growth over $growthSampleSec sec ..." + Start-Sleep -Seconds $growthSampleSec + $now = Get-Date + + $targets = @() + foreach ($path in $first.Keys) { + $item = Get-Item -LiteralPath $path -ErrorAction SilentlyContinue + if (-not $item) { continue } + $sizeNow = $item.Length + $sizeWas = $first[$path] + $reason = $null + + if ($sizeNow -gt $ThresholdBytes) { + $reason = "over threshold ($([math]::Round($sizeNow/1GB,2)) GB)" + } + elseif ($sizeNow -gt $sizeWas) { + $scriptHolders = Get-ScriptHolder -Path $path -ScriptingDir $scriptingDir + if ($scriptHolders.Count -gt 0) { + $reason = "actively growing (+$([math]::Round(($sizeNow-$sizeWas)/1KB,1)) KB in ${growthSampleSec}s) and held by a script process" + } + } + + if (-not $reason) { + $scriptHolders = Get-ScriptHolder -Path $path -ScriptingDir $scriptingDir + foreach ($p in $scriptHolders) { + $started = $null + if ($p.CreationDate) { + try { $started = [Management.ManagementDateTimeConverter]::ToDateTime($p.CreationDate) } catch { $started = $null } + } + if ($started -and ($now - $started).TotalMinutes -gt $RuntimeCeilingMin) { + $reason = "backstop: holder PID $($p.ProcessId) running $([math]::Round(($now-$started).TotalMinutes)) min (> $RuntimeCeilingMin)" + break + } + } + } + + if ($reason) { + $targets += [pscustomobject]@{ Path = $path; Reason = $reason } + } + } + + if ($targets.Count -eq 0) { + Write-Host "RESULT: no runaways (none over threshold, growing, or past runtime ceiling) -> EXIT 0" + return 0 + } + + Write-Host "[4/5] Identified $($targets.Count) runaway file(s):" + $targets | ForEach-Object { Write-Host " $([System.IO.Path]::GetFileName($_.Path)) -- $($_.Reason)" } $failures = 0 - foreach ($f in $candidates) { - Write-Host "[3/4] Processing $($f.Name) [$([math]::Round($f.Length/1GB,2)) GB]" + foreach ($t in $targets) { + $path = $t.Path + Write-Host "[5/5] Processing $([System.IO.Path]::GetFileName($path)) -- $($t.Reason)" - # Ask Windows directly which PIDs hold this file open. $holders = @() - try { $holders = [DtcFileLock]::WhoHas($f.FullName) } catch { - Write-Host " WARN: Restart Manager query failed: $($_.Exception.Message)" - } + try { $holders = [DtcFileLock]::WhoHas($path) } + catch { Write-Host " WARN: Restart Manager query failed: $($_.Exception.Message)" } if ($holders.Count -gt 0) { Write-Host " Lock holders (PIDs): $($holders -join ', ')" @@ -159,7 +234,7 @@ function Invoke-RunawayLogPurge { $proc = Get-Process -Id $hpid -ErrorAction SilentlyContinue if (-not $proc) { continue } if ($protectedNames -contains $proc.Name) { - Write-Host " PROTECTED: PID $hpid is '$($proc.Name)' (NinjaRMM agent) - will NOT kill. File held by agent; stop the source automation in NinjaOne, then re-run." + Write-Host " PROTECTED: PID $hpid is '$($proc.Name)' (NinjaRMM agent) - will NOT kill." continue } Write-Host " Killing PID $hpid ($($proc.Name))" @@ -167,15 +242,13 @@ function Invoke-RunawayLogPurge { catch { Write-Host " WARN: could not kill PID ${hpid}: $($_.Exception.Message)" } } } else { - Write-Host " No lock holders reported (file may be unlocked, or held by a protected/system handle)." + Write-Host " No lock holders reported." } - # Delete with brief retry: handle release after a kill is not always instantaneous. - Write-Host "[4/4] Deleting $($f.FullName)" $deleted = $false for ($attempt = 1; $attempt -le 3; $attempt++) { try { - Remove-Item -LiteralPath $f.FullName -Force -ErrorAction Stop + Remove-Item -LiteralPath $path -Force -ErrorAction Stop $deleted = $true Write-Host " Deleted on attempt $attempt." break @@ -194,11 +267,11 @@ function Invoke-RunawayLogPurge { Write-Host "RESULT: completed with $failures file(s) not freed -> EXIT 1" return 1 } - Write-Host "RESULT: all candidates cleaned -> EXIT 0" + Write-Host "RESULT: all runaways cleaned -> EXIT 0" return 0 } -$exitCode = Invoke-RunawayLogPurge -ThresholdBytes $thresholdBytes +$exitCode = Invoke-RunawayLogPurge -ThresholdBytes $thresholdBytes -RuntimeCeilingMin $RuntimeCeilingMin Write-Host "Final exit code: $exitCode" Stop-Transcript From c622d36d1673ef13471baa1cfa540c234135a469 Mon Sep 17 00:00:00 2001 From: Zachary Boogher Date: Wed, 24 Jun 2026 18:21:58 -0400 Subject: [PATCH 4/4] improvement: detect runaways by growth + runtime ceiling, exclude own output Threshold-only scan missed freshly-respawned runaways still small at scan time, and the tool false-failed trying to delete its own live output file when run via NinjaRMM. Now: snapshots sizes, samples 3s growth, flags a file as runaway if over ThresholdMB OR growing while held by a NinjaRMM customscript process OR holder past RuntimeCeilingMin (default 30). Captures own PID+parent up front and excludes files they hold open, so the tool never targets its own transcript. Restart Manager kill, agent self-protection, 3x delete retry unchanged. Validated via NinjaRMM against a live 11.48GB runaway: real file cleaned; prior false-positive on own output eliminated. --- rmm-ninja/ninjarmm-purge-runaway-logs.ps1 | 47 +++++++++++++++-------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/rmm-ninja/ninjarmm-purge-runaway-logs.ps1 b/rmm-ninja/ninjarmm-purge-runaway-logs.ps1 index 593e127..f7f47a2 100644 --- a/rmm-ninja/ninjarmm-purge-runaway-logs.ps1 +++ b/rmm-ninja/ninjarmm-purge-runaway-logs.ps1 @@ -17,14 +17,12 @@ if ($env:PROCESSOR_ARCHITEW6432 -eq "AMD64" -and -not [Environment]::Is64BitProc ## minutes regardless of file size (optional, default 30) ## ## Purpose: Reclaim disk consumed by runaway NinjaRMM custom-script output transcripts under the -## agent scripting directory, and stop the processes producing them. A file is a runaway if: -## (a) it is over ThresholdMB, OR -## (b) it is actively growing during a short sample window while a NinjaRMM customscript -## process holds it open, OR -## (c) the holding process has run longer than RuntimeCeilingMin (backstop). +## agent scripting directory, and stop the processes producing them. A file is a runaway if +## it is over ThresholdMB, OR actively growing while a NinjaRMM customscript process holds +## it open, OR the holding process has run longer than RuntimeCeilingMin (backstop). +## Excludes this run's own output file so the tool never false-flags what it is writing. ## Uses the Windows Restart Manager to find the exact PIDs holding each file, terminates -## them (never the NinjaRMM agent, never this run), then deletes the file. Fleet-safe: -## fixed agent path, OS-level lock detection, no filename-pattern assumptions. +## them (never the NinjaRMM agent, never this run), then deletes the file. ## ## Exit codes: 0 = completed (zero or more cleaned, no failures) ## 1 = completed with one or more files that could not be freed/deleted @@ -32,7 +30,6 @@ if ($env:PROCESSOR_ARCHITEW6432 -eq "AMD64" -and -not [Environment]::Is64BitProc $ScriptLogName = "ninjarmm-purge-runaway-logs.log" -# --- Resolve optional numeric RMM env vars with defaults and floors --- $ThresholdMB = 500 $parsedMB = 0 if ($env:ThresholdMB -and [int]::TryParse($env:ThresholdMB, [ref]$parsedMB) -and $parsedMB -ge 1) { @@ -46,7 +43,6 @@ if ($env:RuntimeCeilingMin -and [int]::TryParse($env:RuntimeCeilingMin, [ref]$pa $RuntimeCeilingMin = $parsedMin } -# --- Input handling: no required user input. Default Description, set log path by context. --- if ([string]::IsNullOrEmpty($env:Description)) { $env:Description = "Automated runaway-log purge" } @@ -71,8 +67,6 @@ Write-Host "Threshold : $ThresholdMB MB ($thresholdBytes bytes)" Write-Host "Runtime ceiling : $RuntimeCeilingMin min (backstop)" Write-Host "===========================================================" -# --- Restart Manager interop: ask Windows which PIDs hold a file open. Dependency-free, -# works on every Windows version (rstrtmgr.dll, XP+). Added once per session. --- if (-not ([System.Management.Automation.PSTypeName]'DtcFileLock').Type) { Add-Type -ErrorAction Stop -TypeDefinition @" using System; @@ -156,13 +150,36 @@ function Invoke-RunawayLogPurge { return 2 } + $ownPids = @($myPid) + try { + $parent = (Get-CimInstance Win32_Process -Filter "ProcessId=$myPid" -ErrorAction SilentlyContinue).ParentProcessId + if ($parent) { $ownPids += [int]$parent } + } catch { + Write-Host " WARN: could not resolve parent PID: $($_.Exception.Message)" + } + + $selfFiles = @() + Get-ChildItem -LiteralPath $scriptingDir -Filter *output.txt -File -ErrorAction SilentlyContinue | ForEach-Object { + $fp = $_.FullName + try { + $h = [DtcFileLock]::WhoHas($fp) + if ($h | Where-Object { $ownPids -contains $_ }) { $selfFiles += $fp } + } catch { + Write-Host " WARN: self-file check failed on $fp : $($_.Exception.Message)" + } + } + if ($selfFiles.Count -gt 0) { + Write-Host " Excluding this run's own output file(s): $($selfFiles -join ', ')" + } + Write-Host "[2/5] First pass: snapshot all *output.txt sizes ..." $first = @{} Get-ChildItem -LiteralPath $scriptingDir -Filter *output.txt -File -ErrorAction SilentlyContinue | + Where-Object { $selfFiles -notcontains $_.FullName } | ForEach-Object { $first[$_.FullName] = $_.Length } if ($first.Count -eq 0) { - Write-Host "RESULT: no output files present -> nothing to clean -> EXIT 0" + Write-Host "RESULT: no eligible output files present -> nothing to clean -> EXIT 0" return 0 } @@ -227,8 +244,8 @@ function Invoke-RunawayLogPurge { if ($holders.Count -gt 0) { Write-Host " Lock holders (PIDs): $($holders -join ', ')" foreach ($hpid in $holders) { - if ($hpid -eq $myPid -or $hpid -le 4) { - Write-Host " Skipping PID $hpid (self or system)" + if ($ownPids -contains $hpid -or $hpid -le 4) { + Write-Host " Skipping PID $hpid (self/parent or system)" continue } $proc = Get-Process -Id $hpid -ErrorAction SilentlyContinue @@ -275,4 +292,4 @@ $exitCode = Invoke-RunawayLogPurge -ThresholdBytes $thresholdBytes -RuntimeCeili Write-Host "Final exit code: $exitCode" Stop-Transcript -exit $exitCode \ No newline at end of file +exit $exitCode