mem-ruby: add option to disable GPU coalescer deadlock watchdog

configs/example/gpufs/Disjoint_VIPER.py

@@ -50,6 +50,7 @@ def __init__(self):
         super().__init__()
 
     def create(self, options, system, piobus, dma_devices):
+        self.clk_domain = system.fabric_clk
         # Disjoint network topology
         if "garnet" in options.network:
             self.network_cpu = DisjointGarnet(self)

configs/example/gpufs/mi300.py

@@ -71,17 +71,18 @@
     sh /home/gem5/load_amdgpu.sh
 elif [ ! -f /lib/modules/`uname -r`/updates/dkms/amdgpu.ko ]; then
     echo "ERROR: Missing DKMS package for kernel `uname -r`. Exiting gem5."
-    /sbin/m5 exit
+    # m5 exit
 else
     # Backward compatibility with old disk images (ROCm 6.1)
     modprobe -v amdgpu ip_block_mask=0x6f ppfeaturemask=0 dpm=0 audio=0 ras_enable=0 discovery=2
 fi
+modprobe -v amdgpu ip_block_mask=0x6f ppfeaturemask=0 dpm=0 audio=0 ras_enable=0 discovery=2
 
 echo "Running {} {}"
 echo "{}" | base64 -d > myapp
 chmod +x myapp
 ./myapp {}
-/sbin/m5 exit
+m5 exit
 """
 
 demo_runscript_with_checkpoint = """\

configs/example/gpufs/system/amdgpu.py

@@ -36,7 +36,10 @@ def createGPU(system, args):
         n_wf=args.wfs_per_simd,
         cu_per_sqc=args.cu_per_sqc,
         timing=True,
-        clk_domain=system.clk_domain,
+        clk_domain=SrcClockDomain(
+            clock=args.gpu_clock,
+            voltage_domain=VoltageDomain(voltage=args.gpu_voltage),
+        ),
         progress_interval=args.gpu_progress_interval,
     )
 

configs/example/gpufs/system/system.py

@@ -54,6 +54,8 @@ def makeGpuFSSystem(args):
         "drm_kms_helper.fbdev_emulation=0",
         "modprobe.blacklist=amdgpu",
         "modprobe.blacklist=psmouse",
+        # Tell linux to use MP table for PCI IRQs and not ACPI.
+        "pci=noacpi",
     ]
     cmdline = " ".join(boot_options)
 
@@ -73,6 +75,13 @@ def makeGpuFSSystem(args):
     )
     system.workload.object_file = binary(args.kernel)
 
+    # FADT pointing at a minimal DSDT. This prevents Linux from disabling
+    # ACPI which is needed by the WMI module which is a dependency for the
+    # amdgpu module.
+    fadt = X86ACPIFADT(dsdt=X86ACPIDSDT(), oem_id="gem5")
+    system.workload.acpi_description_table_pointer.rsdt.entries.append(fadt)
+    system.workload.acpi_description_table_pointer.xsdt.entries.append(fadt)
+
     # Set the cache line size for the entire system.
     system.cache_line_size = args.cacheline_size
 
@@ -302,6 +311,13 @@ def makeGpuFSSystem(args):
         pm4_proc.pio = system.iobus.mem_side_ports
     system_hub.pio = system.iobus.mem_side_ports
 
+    system.fabric_clk = SrcClockDomain(
+        clock=args.fabric_clock, voltage_domain=system.voltage_domain
+    )
+    system.memory_clk = SrcClockDomain(
+        clock=args.memory_clock, voltage_domain=system.voltage_domain
+    )
+
     # Full system needs special TLBs for SQC, Scalar, and vector data ports
     args.full_system = True
     GPUTLBConfig.config_tlb_hierarchy(
@@ -313,9 +329,10 @@ def makeGpuFSSystem(args):
     system.ruby.create(args, system, system.iobus, system._dma_ports)
 
     # Create a seperate clock domain for Ruby
-    system.ruby.clk_domain = SrcClockDomain(
-        clock=args.ruby_clock, voltage_domain=system.voltage_domain
-    )
+    # system.ruby.clk_domain = SrcClockDomain(
+    #     clock=args.ruby_clock, voltage_domain=system.voltage_domain
+    # )
+    system.ruby.clk_domain = system.fabric_clk
 
     # If we are using KVM cpu, enable AVX. AVX is used in some ROCm libraries
     # such as rocBLAS which is used in higher level libraries like PyTorch.

configs/ruby/GPU_VIPER.py

@@ -165,7 +165,7 @@ def create(self, options, ruby_system, system):
         self.L1cache.dataArrayBanks = options.tcp_num_banks
         self.L1cache.tagArrayBanks = options.tcp_num_banks
         self.L1cache.create(options)
-        self.issue_latency = 1
+        self.issue_latency = options.tcp_issue_latency
         # TCP_Controller inherits this from RubyController
         self.mandatory_queue_latency = options.mandatory_queue_latency
 
@@ -178,6 +178,7 @@ def create(self, options, ruby_system, system):
         self.coalescer.is_cpu_sequencer = False
         if options.tcp_deadlock_threshold:
             self.coalescer.deadlock_threshold = options.tcp_deadlock_threshold
+        self.coalescer.disable_deadlock_check = options.disable_watchdog
         self.coalescer.max_coalesces_per_cycle = (
             options.max_coalesces_per_cycle
         )
@@ -218,13 +219,13 @@ def createCP(self, options, ruby_system, system):
         self.coalescer.ruby_system = ruby_system
         self.coalescer.support_inst_reqs = False
         self.coalescer.is_cpu_sequencer = False
+        self.coalescer.disable_deadlock_check = options.disable_watchdog
 
         self.sequencer = RubySequencer(ruby_system=ruby_system)
         self.sequencer.version = self.seqCount()
         self.sequencer.dcache = self.L1cache
         self.sequencer.ruby_system = ruby_system
         self.sequencer.is_cpu_sequencer = True
-
         self.use_seq_not_coal = True
 
         self.ruby_system = ruby_system
@@ -320,9 +321,11 @@ def create(self, options, ruby_system, system):
         self.L2cache.resourceStalls = options.no_tcc_resource_stalls
 
         self.ruby_system = ruby_system
-        if hasattr(options, "gpu_clock") and hasattr(options, "gpu_voltage"):
+        if hasattr(options, "fabric_clock") and hasattr(
+            options, "gpu_voltage"
+        ):
             self.clk_domain = SrcClockDomain(
-                clock=options.gpu_clock,
+                clock=options.fabric_clock,
                 voltage_domain=VoltageDomain(voltage=options.gpu_voltage),
             )
 
@@ -424,6 +427,7 @@ def connectWireBuffers(
 
 def define_options(parser):
     parser.add_argument("--num-subcaches", type=int, default=4)
+    parser.add_argument("--tcp-issue-latency", type=int, default=1)
     parser.add_argument("--l3-data-latency", type=int, default=20)
     parser.add_argument("--l3-tag-latency", type=int, default=15)
     parser.add_argument("--cpu-to-dir-latency", type=int, default=120)
@@ -465,7 +469,17 @@ def define_options(parser):
         "--TCP_latency",
         type=int,
         default=4,
-        help="In combination with the number of banks for the "
+        help="Set tcp tag access latency. "
+        "In combination with the number of banks for the "
+        "TCP, this determines how many requests can happen "
+        "per cycle (i.e., the bandwidth)",
+    )
+    parser.add_argument(
+        "--TCP_latency_data",
+        type=int,
+        default=4,
+        help="Set tcp data access latency. "
+        "In combination with the number of banks for the "
         "TCP, this determines how many requests can happen "
         "per cycle (i.e., the bandwidth)",
     )
@@ -491,6 +505,14 @@ def define_options(parser):
         type=int,
         help="Set the TCP deadlock threshold to some value",
     )
+    parser.add_argument(
+        "--disable-watchdog",
+        action="store_true",
+        default=False,
+        help="Disable the GPU coalescer deadlock watchdog (which otherwise "
+             "aborts on long-lived outstanding requests). Useful for "
+             "slow-memory sensitivity experiments.",
+    )
     parser.add_argument(
         "--max-coalesces-per-cycle",
         type=int,
@@ -537,6 +559,8 @@ def define_options(parser):
         default="8",
         help="Data access latency in L2 cache",
     )
+    parser.add_argument("--fabric-clock", type=str, default="1080MHz")
+    parser.add_argument("--memory-clock", type=str, default="1000MHz")
 
 
 def construct_dirs(options, system, ruby_system, network):
@@ -631,7 +655,11 @@ def construct_gpudirs(options, system, ruby_system, network):
             xorHighBit=xor_low_bit,
         )
 
-        dir_cntrl = DirCntrl(noTCCdir=True, TCC_select_num_bits=TCC_bits)
+        dir_cntrl = DirCntrl(
+            noTCCdir=True,
+            TCC_select_num_bits=TCC_bits,
+            clk_domain=system.fabric_clk,
+        )
         dir_cntrl.create(options, [addr_range], ruby_system, system)
         dir_cntrl.number_of_TBEs = options.num_tbes
         dir_cntrl.useL3OnWT = False
@@ -702,7 +730,9 @@ def construct_gpudirs(options, system, ruby_system, network):
         if issubclass(mem_type, DRAMInterface):
             if options.hbm_ctrl:
                 mem_ctrl = m5.objects.HBMCtrl(
-                    dram=dram_intf, dram_2=dram_intf_2
+                    dram=dram_intf,
+                    dram_2=dram_intf_2,
+                    clk_domain=system.memory_clk,
                 )
             else:
                 mem_ctrl = m5.objects.MemCtrl(dram=dram_intf)
@@ -780,7 +810,7 @@ def construct_tcps(options, system, ruby_system, network):
         tcp_cntrl.WB = options.WB_L1
         tcp_cntrl.disableL1 = options.noL1
         tcp_cntrl.L1cache.tagAccessLatency = options.TCP_latency
-        tcp_cntrl.L1cache.dataAccessLatency = options.TCP_latency
+        tcp_cntrl.L1cache.dataAccessLatency = options.TCP_latency_data
 
         exec("ruby_system.tcp_cntrl%d = tcp_cntrl" % i)
         #
@@ -936,7 +966,10 @@ def construct_tccs(options, system, ruby_system, network):
     tcc_cntrl_nodes = []
 
     for i in range(options.num_tccs):
-        tcc_cntrl = TCCCntrl(l2_response_latency=options.TCC_latency)
+        tcc_cntrl = TCCCntrl(
+            l2_response_latency=options.TCC_latency,
+            clk_domain=system.fabric_clk,
+        )
         tcc_cntrl.create(options, ruby_system, system)
         tcc_cntrl.l2_request_latency = options.gpu_to_dir_latency
         tcc_cntrl.l2_response_latency = options.TCC_latency