Add zombie process tracking

Author: zachlatta

app/controllers/home_controller.rb

@@ -39,6 +39,8 @@ def index
       prepare_cpu_view
     when 'ram'
       prepare_ram_view
+    when 'zombies'
+      prepare_zombies_view
     end
 
     # Get all servers for filter dropdown
@@ -103,5 +105,36 @@ def prepare_ram_view
       .order('avg_mem_24h DESC NULLS LAST')
       .limit(100)
   end
+  
+  def prepare_zombies_view
+    # Get servers with zombie processes
+    server_query = Server.includes(:coolify_team)
+    server_query = server_query.where(id: @server_id) if @server_id.present?
+    
+    # Get latest server stats with zombie processes
+    @servers_by_zombies = server_query
+      .left_joins(:server_stats)
+      .select("servers.*, 
+               MAX(CASE WHEN server_stats.captured_at >= NOW() - INTERVAL '24 hours' THEN server_stats.zombie_processes END) as max_zombies_24h,
+               AVG(CASE WHEN server_stats.captured_at >= NOW() - INTERVAL '24 hours' THEN server_stats.zombie_processes END) as avg_zombies_24h")
+      .group('servers.id')
+      .having("MAX(CASE WHEN server_stats.captured_at >= NOW() - INTERVAL '24 hours' THEN server_stats.zombie_processes END) > 0")
+      .order(Arel.sql('max_zombies_24h DESC NULLS LAST'))
+    
+    # Get resources with zombie processes
+    resource_query = Resource.includes(:server, environment: :project)
+    resource_query = resource_query.where(server_id: @server_id) if @server_id.present?
+    
+    # Get resources with highest zombie counts
+    @resources_by_zombies = resource_query
+      .left_joins(:resource_stats)
+      .select("resources.*, 
+               MAX(CASE WHEN resource_stats.captured_at >= NOW() - INTERVAL '24 hours' THEN resource_stats.zombie_processes END) as max_zombies_24h,
+               AVG(CASE WHEN resource_stats.captured_at >= NOW() - INTERVAL '24 hours' THEN resource_stats.zombie_processes END) as avg_zombies_24h")
+      .group('resources.id')
+      .having("MAX(CASE WHEN resource_stats.captured_at >= NOW() - INTERVAL '24 hours' THEN resource_stats.zombie_processes END) > 0")
+      .order(Arel.sql('max_zombies_24h DESC NULLS LAST'))
+      .limit(100)
+  end
 end
 

app/controllers/resources_controller.rb

@@ -38,7 +38,8 @@ def prepare_chart_data(stats)
       memory_bytes: stats.map { |s| s.mem_used_bytes },
       disk_persistent: stats.map { |s| s.disk_persistent_bytes },
       disk_runtime: stats.map { |s| s.disk_runtime_bytes },
-      disk_total: stats.map { |s| (s.disk_persistent_bytes.to_i + s.disk_runtime_bytes.to_i) }
+      disk_total: stats.map { |s| (s.disk_persistent_bytes.to_i + s.disk_runtime_bytes.to_i) },
+      zombie_processes: stats.map { |s| s.zombie_processes }
     }
   end
 
@@ -48,6 +49,7 @@ def calculate_stats_summary(stats)
     cpu_values = stats.map(&:cpu_pct).compact
     mem_values = stats.map(&:mem_used_bytes).compact
     disk_values = stats.map { |s| (s.disk_persistent_bytes.to_i + s.disk_runtime_bytes.to_i) }
+    zombie_values = stats.map(&:zombie_processes).compact
 
     {
       cpu_avg: cpu_values.any? ? (cpu_values.sum / cpu_values.size).round(2) : nil,
@@ -59,6 +61,9 @@ def calculate_stats_summary(stats)
       disk_avg: disk_values.any? ? (disk_values.sum / disk_values.size).round(0) : nil,
       disk_max: disk_values.max,
       disk_min: disk_values.min,
+      zombie_avg: zombie_values.any? ? (zombie_values.sum.to_f / zombie_values.size).round(1) : nil,
+      zombie_max: zombie_values.max,
+      zombie_current: zombie_values.last,
       data_points: stats.size
     }
   end

app/controllers/servers_controller.rb

@@ -59,7 +59,8 @@ def prepare_chart_data(stats)
       iops_write: stats.map { |s| s.iops_w&.round(2) },
       load1: stats.map { |s| s.load1&.round(2) },
       load5: stats.map { |s| s.load5&.round(2) },
-      load15: stats.map { |s| s.load15&.round(2) }
+      load15: stats.map { |s| s.load15&.round(2) },
+      zombie_processes: stats.map { |s| s.zombie_processes }
     }
   end
 
@@ -75,6 +76,7 @@ def calculate_stats_summary(stats)
     load1_values = stats.map(&:load1).compact
     load5_values = stats.map(&:load5).compact
     load15_values = stats.map(&:load15).compact
+    zombie_values = stats.map(&:zombie_processes).compact
 
     {
       cpu_avg: cpu_values.any? ? (cpu_values.sum / cpu_values.size).round(2) : nil,
@@ -94,6 +96,9 @@ def calculate_stats_summary(stats)
       load1_max: load1_values.max,
       load5_avg: load5_values.any? ? (load5_values.sum / load5_values.size).round(2) : nil,
       load15_avg: load15_values.any? ? (load15_values.sum / load15_values.size).round(2) : nil,
+      zombie_avg: zombie_values.any? ? (zombie_values.sum.to_f / zombie_values.size).round(1) : nil,
+      zombie_max: zombie_values.max,
+      zombie_current: zombie_values.last,
       data_points: stats.size
     }
   end

app/jobs/resource_metrics_collector_job.rb

@@ -137,7 +137,19 @@ def perform(server_id)
 
     Rails.logger.info "[ResourceMetrics] #{server.name}:   Found #{volume_paths.count} unique volume paths"
 
-    # 4. Bulk calculate all volume sizes (simple du approach)
+    # 4. Collect zombie processes for all containers (single SSH call)
+    Rails.logger.info "[ResourceMetrics] #{server.name}: → Counting zombie processes per container..."
+    zombie_start = Time.current
+    zombies_by_container = collect_container_zombies(client, id_to_resource.keys)
+    zombie_elapsed = (Time.current - zombie_start).round(2)
+    total_zombies = zombies_by_container.values.sum
+    if total_zombies > 0
+      Rails.logger.info "[ResourceMetrics] #{server.name}:   Found #{total_zombies} zombie processes across #{zombies_by_container.count { |_, v| v > 0 }} containers in #{zombie_elapsed}s"
+    else
+      Rails.logger.info "[ResourceMetrics] #{server.name}:   No zombie processes found (#{zombie_elapsed}s)"
+    end
+    
+    # 5. Bulk calculate all volume sizes (simple du approach)
     volume_sizes = {}
     if volume_paths.any?
       Rails.logger.info "[ResourceMetrics] #{server.name}: → Calculating volume sizes (#{volume_paths.count} paths)..."
@@ -212,6 +224,9 @@ def perform(server_id)
       # Use container limit if set, otherwise use system memory from stats
       final_mem_limit = container_mem_limit > 0 ? container_mem_limit : mem_limit
 
+      # Get zombie process count for this container
+      zombie_count = zombies_by_container[cid].to_i
+      
       # Log detailed info for first few containers
       if collected < 3
         limit_source = container_mem_limit > 0 ? "container limit" : "system memory"
@@ -224,6 +239,7 @@ def perform(server_id)
         Rails.logger.info "[ResourceMetrics] #{server.name}:   #{resource.name}:"
         Rails.logger.info "[ResourceMetrics] #{server.name}:     MEM: #{used_mb}MB / #{limit_gb}GB (#{limit_source})"
         Rails.logger.info "[ResourceMetrics] #{server.name}:     DISK: RW=#{rw_mb}MB, Volumes=#{vol_mb}MB (#{mounts.count} mounts)"
+        Rails.logger.info "[ResourceMetrics] #{server.name}:     ZOMBIES: #{zombie_count}" if zombie_count > 0
       end
 
       ResourceStat.create!(
@@ -235,7 +251,8 @@ def perform(server_id)
         mem_used_bytes: mem_used,
         mem_limit_bytes: final_mem_limit,
         disk_runtime_bytes: size_rw,
-        disk_persistent_bytes: container_volume_size
+        disk_persistent_bytes: container_volume_size,
+        zombie_processes: zombie_count
       )
       collected += 1
 
@@ -304,6 +321,95 @@ def parse_size_to_bytes(size_str)
     else value
     end.to_i
   end
+
+  # Collect zombie process counts per container using efficient single SSH call
+  def collect_container_zombies(client, container_ids)
+    return {} if container_ids.empty?
+    
+    # Single compound command to get all data we need
+    compound_cmd = <<~SHELL.strip
+      echo "=ZOMBIES=";
+      ps -eo ppid,state 2>/dev/null | awk '$2=="Z" {print $1}' | sort | uniq -c || echo "";
+      echo "=CONTAINERS=";
+      docker inspect --format '{{.State.Pid}} {{.Id}}' #{container_ids.join(' ')} 2>/dev/null || echo "";
+      echo "=TREE=";
+      ps -eo pid,ppid 2>/dev/null || echo ""
+    SHELL
+    
+    _code, output, _err = client.exec!(compound_cmd, timeout: 30)
+    
+    # Parse output sections
+    sections = output.split(/^=/m)
+    zombies_section = sections.find { |s| s.start_with?('ZOMBIES=') }&.sub('ZOMBIES=', '')&.strip || ''
+    containers_section = sections.find { |s| s.start_with?('CONTAINERS=') }&.sub('CONTAINERS=', '')&.strip || ''
+    tree_section = sections.find { |s| s.start_with?('TREE=') }&.sub('TREE=', '')&.strip || ''
+    
+    # Parse zombie parent PIDs with counts: "  5 1234" -> {1234 => 5}
+    zombie_parents = {}
+    zombies_section.each_line do |line|
+      parts = line.strip.split
+      next unless parts.length == 2
+      count = parts[0].to_i
+      ppid = parts[1].to_i
+      zombie_parents[ppid] = count if ppid > 0
+    end
+    
+    # Parse container PIDs: "12345 abc123def456..." -> {12345 => "abc123..."}
+    container_pids = {}
+    pid_to_container = {}
+    containers_section.each_line do |line|
+      parts = line.strip.split
+      next unless parts.length == 2
+      pid = parts[0].to_i
+      container_id = parts[1]
+      if pid > 0 && container_id.present?
+        container_pids[pid] = container_id
+        pid_to_container[pid] = container_id
+      end
+    end
+    
+    # Build process tree: {child_pid => parent_pid}
+    process_tree = {}
+    tree_section.each_line do |line|
+      parts = line.strip.split
+      next unless parts.length == 2
+      pid = parts[0].to_i
+      ppid = parts[1].to_i
+      process_tree[pid] = ppid if pid > 0
+    end
+    
+    # Map zombie parents to containers by walking up the tree
+    zombies_by_container = Hash.new(0)
+    
+    zombie_parents.each do |ppid, count|
+      # Walk up the process tree to find a container PID
+      current_pid = ppid
+      visited = {}
+      max_depth = 50
+      depth = 0
+      
+      while current_pid && current_pid > 1 && depth < max_depth
+        # Prevent infinite loops
+        break if visited[current_pid]
+        visited[current_pid] = true
+        
+        # Check if this PID is a container's main process
+        if (container_id = pid_to_container[current_pid])
+          zombies_by_container[container_id] += count
+          break
+        end
+        
+        # Move to parent
+        current_pid = process_tree[current_pid]
+        depth += 1
+      end
+    end
+    
+    zombies_by_container
+  rescue => e
+    Rails.logger.warn "[ResourceMetrics] Failed to collect zombie processes: #{e.message}"
+    {}
+  end
 end
 
 

app/jobs/server_metrics_collector_job.rb

@@ -37,6 +37,9 @@ def perform(server_id)
 
     Rails.logger.info "[ServerMetrics] #{server.name}: → Collecting filesystem info..."
     filesystems = collect_filesystems(client)
+    
+    Rails.logger.info "[ServerMetrics] #{server.name}: → Counting zombie processes..."
+    zombie_count = collect_zombie_processes(client)
 
     ServerStat.create!(
       server: server,
@@ -51,7 +54,8 @@ def perform(server_id)
       load1: load1,
       load5: load5,
       load15: load15,
-      filesystems: filesystems
+      filesystems: filesystems,
+      zombie_processes: zombie_count
     )
 
     elapsed = (Time.current - start_time).round(1)
@@ -237,6 +241,18 @@ def collect_cpu_cores(client)
     code, out, _ = client.exec!("nproc 2>/dev/null || grep -c processor /proc/cpuinfo")
     out.to_i if code == 0 && out.to_i > 0
   end
+
+  def collect_zombie_processes(client)
+    # Count processes in zombie state (Z)
+    code, out, _err = client.exec!("ps axo state | grep -c '^Z' 2>/dev/null || echo 0")
+    return out.to_i if code == 0 && out.to_s.strip.match?(/^\d+$/)
+    
+    # Fallback: count from /proc
+    code, out, _ = client.exec!("find /proc -maxdepth 1 -type d -name '[0-9]*' -exec cat {}/stat 2>/dev/null \\; | awk '{if ($3 == \"Z\") count++} END {print count+0}'")
+    code == 0 ? out.to_i : nil
+  rescue
+    nil
+  end
 end