From 57b4e7fd995dc97f2742db80dfae56246b2e717d Mon Sep 17 00:00:00 2001 From: Basem Mohammed <95645899+Basemism@users.noreply.github.com> Date: Thu, 6 Aug 2026 14:12:20 +0200 Subject: [PATCH 1/2] mem-ruby: add option to disable GPU coalescer deadlock watchdog The GPU coalescer watchdog aborts the simulation when a request stays outstanding longer than deadlock_threshold. This false-triggers in slow-memory sensitivity experiments where DRAM timings are intentionally stretched and long-lived requests are expected. Add a disable_deadlock_check parameter to GPUCoalescer (default False, so the watchdog stays on by default) and gate the wakeup() deadlock scan on it. Expose it through GPU_VIPER's --disable-watchdog flag, wired to both VIPER coalescers. --- configs/ruby/GPU_VIPER.py | 11 +++++++++- src/mem/ruby/system/GPUCoalescer.cc | 33 ++++++++++++++++++----------- src/mem/ruby/system/GPUCoalescer.hh | 1 + src/mem/ruby/system/GPUCoalescer.py | 7 ++++++ 4 files changed, 39 insertions(+), 13 deletions(-) diff --git a/configs/ruby/GPU_VIPER.py b/configs/ruby/GPU_VIPER.py index 56d01b6d52e..06adec3b514 100644 --- a/configs/ruby/GPU_VIPER.py +++ b/configs/ruby/GPU_VIPER.py @@ -178,6 +178,7 @@ def create(self, options, ruby_system, system): self.coalescer.is_cpu_sequencer = False if options.tcp_deadlock_threshold: self.coalescer.deadlock_threshold = options.tcp_deadlock_threshold + self.coalescer.disable_deadlock_check = options.disable_watchdog self.coalescer.max_coalesces_per_cycle = ( options.max_coalesces_per_cycle ) @@ -218,13 +219,13 @@ def createCP(self, options, ruby_system, system): self.coalescer.ruby_system = ruby_system self.coalescer.support_inst_reqs = False self.coalescer.is_cpu_sequencer = False + self.coalescer.disable_deadlock_check = options.disable_watchdog self.sequencer = RubySequencer(ruby_system=ruby_system) self.sequencer.version = self.seqCount() self.sequencer.dcache = self.L1cache self.sequencer.ruby_system = ruby_system self.sequencer.is_cpu_sequencer = True - self.use_seq_not_coal = True self.ruby_system = ruby_system @@ -504,6 +505,14 @@ def define_options(parser): type=int, help="Set the TCP deadlock threshold to some value", ) + parser.add_argument( + "--disable-watchdog", + action="store_true", + default=False, + help="Disable the GPU coalescer deadlock watchdog (which otherwise " + "aborts on long-lived outstanding requests). Useful for " + "slow-memory sensitivity experiments.", + ) parser.add_argument( "--max-coalesces-per-cycle", type=int, diff --git a/src/mem/ruby/system/GPUCoalescer.cc b/src/mem/ruby/system/GPUCoalescer.cc index 893b2d794d3..ab1c14b3eee 100644 --- a/src/mem/ruby/system/GPUCoalescer.cc +++ b/src/mem/ruby/system/GPUCoalescer.cc @@ -231,6 +231,7 @@ GPUCoalescer::GPUCoalescer(const Params &p) m_dataCache_ptr = p.dcache; m_max_outstanding_requests = p.max_outstanding_requests; m_deadlock_threshold = p.deadlock_threshold; + m_disable_deadlock_check = p.disable_deadlock_check; assert(m_max_outstanding_requests > 0); assert(m_deadlock_threshold > 0); @@ -305,21 +306,29 @@ GPUCoalescer::getPort(const std::string &if_name, PortID idx) void GPUCoalescer::wakeup() { - Cycles current_time = curCycle(); - for (auto& requestList : coalescedTable) { - for (auto& req : requestList.second) { - if (current_time - req->getIssueTime() > m_deadlock_threshold) { - std::stringstream ss; - printRequestTable(ss); - warn("GPUCoalescer %d Possible deadlock detected!\n%s\n", - m_version, ss.str()); - panic("Aborting due to deadlock!\n"); + // The deadlock watchdog treats a request that stays outstanding longer + // than m_deadlock_threshold as a fatal deadlock. This can false-trigger + // when DRAM timings are intentionally stretched (slow-memory sensitivity + // experiments), so it can be turned off via the disable_deadlock_check + // parameter (--disable-watchdog). + if (!m_disable_deadlock_check) { + Cycles current_time = curCycle(); + for (auto& requestList : coalescedTable) { + for (auto& req : requestList.second) { + if (current_time - req->getIssueTime() + > m_deadlock_threshold) { + std::stringstream ss; + printRequestTable(ss); + warn("GPUCoalescer %d Possible deadlock detected!\n%s\n", + m_version, ss.str()); + panic("Aborting due to deadlock!\n"); + } } } - } - Tick tick_threshold = cyclesToTicks(m_deadlock_threshold); - uncoalescedTable.checkDeadlock(tick_threshold); + Tick tick_threshold = cyclesToTicks(m_deadlock_threshold); + uncoalescedTable.checkDeadlock(tick_threshold); + } if (m_outstanding_count > 0) { schedule(deadlockCheckEvent, diff --git a/src/mem/ruby/system/GPUCoalescer.hh b/src/mem/ruby/system/GPUCoalescer.hh index 45b7b25dd0e..b4dbf7f5bcf 100644 --- a/src/mem/ruby/system/GPUCoalescer.hh +++ b/src/mem/ruby/system/GPUCoalescer.hh @@ -427,6 +427,7 @@ class GPUCoalescer : public RubyPort protected: int m_max_outstanding_requests; Cycles m_deadlock_threshold; + bool m_disable_deadlock_check; CacheMemory* m_dataCache_ptr; CacheMemory* m_instCache_ptr; diff --git a/src/mem/ruby/system/GPUCoalescer.py b/src/mem/ruby/system/GPUCoalescer.py index d69314eb54a..2b99ae1d515 100644 --- a/src/mem/ruby/system/GPUCoalescer.py +++ b/src/mem/ruby/system/GPUCoalescer.py @@ -53,6 +53,13 @@ class RubyGPUCoalescer(RubyPort): "max outstanding cycles for a request before " "deadlock/livelock declared", ) + disable_deadlock_check = Param.Bool( + False, + "Disable the coalescer deadlock watchdog. The watchdog aborts the " + "simulation when a request stays outstanding longer than " + "deadlock_threshold; disable it for slow-memory sensitivity " + "experiments where long-lived requests are expected.", + ) garnet_standalone = Param.Bool(False, "") gmTokenPort = ResponsePort("Port to the CU for sharing tokens") From f3f71f7d62317f5437069d66cec6edb3482a7935 Mon Sep 17 00:00:00 2001 From: Basem Mohammed <95645899+Basemism@users.noreply.github.com> Date: Thu, 10 Sep 2026 16:21:10 +0200 Subject: [PATCH 2/2] configs: disable guest watchdog with GPU watchdog --- configs/example/gpufs/system/system.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/configs/example/gpufs/system/system.py b/configs/example/gpufs/system/system.py index 31e1e5eb914..c02efbb1803 100644 --- a/configs/example/gpufs/system/system.py +++ b/configs/example/gpufs/system/system.py @@ -57,6 +57,8 @@ def makeGpuFSSystem(args): # Tell linux to use MP table for PCI IRQs and not ACPI. "pci=noacpi", ] + if args.disable_watchdog: + boot_options.extend(["nowatchdog", "nosoftlockup", "tsc=unstable"]) cmdline = " ".join(boot_options) if MemorySize(args.mem_size) < MemorySize("2GiB"):