configs: model the GPU scalar cache separately from the SQC

configs/example/gpufs/Disjoint_VIPER.py

@@ -50,6 +50,7 @@ def __init__(self):
         super().__init__()
 
     def create(self, options, system, piobus, dma_devices):
+        self.clk_domain = system.fabric_clk
         # Disjoint network topology
         if "garnet" in options.network:
             self.network_cpu = DisjointGarnet(self)

configs/example/gpufs/mi300.py

@@ -71,17 +71,18 @@
     sh /home/gem5/load_amdgpu.sh
 elif [ ! -f /lib/modules/`uname -r`/updates/dkms/amdgpu.ko ]; then
     echo "ERROR: Missing DKMS package for kernel `uname -r`. Exiting gem5."
-    /sbin/m5 exit
+    # m5 exit
 else
     # Backward compatibility with old disk images (ROCm 6.1)
     modprobe -v amdgpu ip_block_mask=0x6f ppfeaturemask=0 dpm=0 audio=0 ras_enable=0 discovery=2
 fi
+modprobe -v amdgpu ip_block_mask=0x6f ppfeaturemask=0 dpm=0 audio=0 ras_enable=0 discovery=2
 
 echo "Running {} {}"
 echo "{}" | base64 -d > myapp
 chmod +x myapp
 ./myapp {}
-/sbin/m5 exit
+m5 exit
 """
 
 demo_runscript_with_checkpoint = """\

configs/example/gpufs/system/amdgpu.py

@@ -36,7 +36,10 @@ def createGPU(system, args):
         n_wf=args.wfs_per_simd,
         cu_per_sqc=args.cu_per_sqc,
         timing=True,
-        clk_domain=system.clk_domain,
+        clk_domain=SrcClockDomain(
+            clock=args.gpu_clock,
+            voltage_domain=VoltageDomain(voltage=args.gpu_voltage),
+        ),
         progress_interval=args.gpu_progress_interval,
     )
 

configs/example/gpufs/system/system.py

@@ -54,6 +54,8 @@ def makeGpuFSSystem(args):
         "drm_kms_helper.fbdev_emulation=0",
         "modprobe.blacklist=amdgpu",
         "modprobe.blacklist=psmouse",
+        # Tell linux to use MP table for PCI IRQs and not ACPI.
+        "pci=noacpi",
     ]
     cmdline = " ".join(boot_options)
 
@@ -73,6 +75,13 @@ def makeGpuFSSystem(args):
     )
     system.workload.object_file = binary(args.kernel)
 
+    # FADT pointing at a minimal DSDT. This prevents Linux from disabling
+    # ACPI which is needed by the WMI module which is a dependency for the
+    # amdgpu module.
+    fadt = X86ACPIFADT(dsdt=X86ACPIDSDT(), oem_id="gem5")
+    system.workload.acpi_description_table_pointer.rsdt.entries.append(fadt)
+    system.workload.acpi_description_table_pointer.xsdt.entries.append(fadt)
+
     # Set the cache line size for the entire system.
     system.cache_line_size = args.cacheline_size
 
@@ -302,6 +311,13 @@ def makeGpuFSSystem(args):
         pm4_proc.pio = system.iobus.mem_side_ports
     system_hub.pio = system.iobus.mem_side_ports
 
+    system.fabric_clk = SrcClockDomain(
+        clock=args.fabric_clock, voltage_domain=system.voltage_domain
+    )
+    system.memory_clk = SrcClockDomain(
+        clock=args.memory_clock, voltage_domain=system.voltage_domain
+    )
+
     # Full system needs special TLBs for SQC, Scalar, and vector data ports
     args.full_system = True
     GPUTLBConfig.config_tlb_hierarchy(
@@ -313,9 +329,10 @@ def makeGpuFSSystem(args):
     system.ruby.create(args, system, system.iobus, system._dma_ports)
 
     # Create a seperate clock domain for Ruby
-    system.ruby.clk_domain = SrcClockDomain(
-        clock=args.ruby_clock, voltage_domain=system.voltage_domain
-    )
+    # system.ruby.clk_domain = SrcClockDomain(
+    #     clock=args.ruby_clock, voltage_domain=system.voltage_domain
+    # )
+    system.ruby.clk_domain = system.fabric_clk
 
     # If we are using KVM cpu, enable AVX. AVX is used in some ROCm libraries
     # such as rocBLAS which is used in higher level libraries like PyTorch.

configs/ruby/GPU_VIPER.py

@@ -165,7 +165,7 @@ def create(self, options, ruby_system, system):
         self.L1cache.dataArrayBanks = options.tcp_num_banks
         self.L1cache.tagArrayBanks = options.tcp_num_banks
         self.L1cache.create(options)
-        self.issue_latency = 1
+        self.issue_latency = options.tcp_issue_latency
         # TCP_Controller inherits this from RubyController
         self.mandatory_queue_latency = options.mandatory_queue_latency
 
@@ -275,6 +275,50 @@ def create(self, options, ruby_system, system):
             self.recycle_latency = options.recycle_latency
 
 
+class ScalarCache(RubyCache):
+    dataArrayBanks = 8
+    tagArrayBanks = 8
+    dataAccessLatency = 1
+    tagAccessLatency = 1
+
+    def create(self, options):
+        self.size = MemorySize(options.scalar_size)
+        self.assoc = options.scalar_assoc
+        if hasattr(options, "sqc_rp"):
+            self.replacement_policy = ObjectList.rp_list.get(options.sqc_rp)()
+
+
+class ScalarCntrl(GPU_VIPER_SQC_Controller, CntrlBase):
+    def create(self, options, ruby_system, system):
+        # Scalar and SQC controllers share MachineType:SQC, so their
+        # controller version IDs must come from the same counter.
+        self.version = SQCCntrl.versionCount()
+
+        self.L1cache = ScalarCache()
+        self.L1cache.create(options)
+        self.L1cache.resourceStalls = options.no_resource_stalls
+
+        self.sequencer = VIPERSequencer()
+
+        self.sequencer.version = self.seqCount()
+        self.sequencer.dcache = self.L1cache
+        self.sequencer.ruby_system = ruby_system
+        self.sequencer.support_data_reqs = False
+        self.sequencer.is_cpu_sequencer = False
+        if options.sqc_deadlock_threshold:
+            self.sequencer.deadlock_threshold = options.sqc_deadlock_threshold
+
+        self.ruby_system = ruby_system
+        if hasattr(options, "gpu_clock") and hasattr(options, "gpu_voltage"):
+            self.clk_domain = SrcClockDomain(
+                clock=options.gpu_clock,
+                voltage_domain=VoltageDomain(voltage=options.gpu_voltage),
+            )
+
+        if options.recycle_latency:
+            self.recycle_latency = options.recycle_latency
+
+
 class TCC(RubyCache):
     size = MemorySize("256KiB")
     assoc = 16
@@ -320,9 +364,11 @@ def create(self, options, ruby_system, system):
         self.L2cache.resourceStalls = options.no_tcc_resource_stalls
 
         self.ruby_system = ruby_system
-        if hasattr(options, "gpu_clock") and hasattr(options, "gpu_voltage"):
+        if hasattr(options, "fabric_clock") and hasattr(
+            options, "gpu_voltage"
+        ):
             self.clk_domain = SrcClockDomain(
-                clock=options.gpu_clock,
+                clock=options.fabric_clock,
                 voltage_domain=VoltageDomain(voltage=options.gpu_voltage),
             )
 
@@ -424,6 +470,7 @@ def connectWireBuffers(
 
 def define_options(parser):
     parser.add_argument("--num-subcaches", type=int, default=4)
+    parser.add_argument("--tcp-issue-latency", type=int, default=1)
     parser.add_argument("--l3-data-latency", type=int, default=20)
     parser.add_argument("--l3-tag-latency", type=int, default=15)
     parser.add_argument("--cpu-to-dir-latency", type=int, default=120)
@@ -449,6 +496,12 @@ def define_options(parser):
     parser.add_argument(
         "--sqc-assoc", type=int, default=8, help="SQC cache assoc"
     )
+    parser.add_argument(
+        "--scalar-size", type=str, default="32KiB", help="Scalar cache size"
+    )
+    parser.add_argument(
+        "--scalar-assoc", type=int, default=8, help="Scalar cache assoc"
+    )
     parser.add_argument(
         "--sqc-deadlock-threshold",
         type=int,
@@ -465,7 +518,17 @@ def define_options(parser):
         "--TCP_latency",
         type=int,
         default=4,
-        help="In combination with the number of banks for the "
+        help="Set tcp tag access latency. "
+        "In combination with the number of banks for the "
+        "TCP, this determines how many requests can happen "
+        "per cycle (i.e., the bandwidth)",
+    )
+    parser.add_argument(
+        "--TCP_latency_data",
+        type=int,
+        default=4,
+        help="Set tcp data access latency. "
+        "In combination with the number of banks for the "
         "TCP, this determines how many requests can happen "
         "per cycle (i.e., the bandwidth)",
     )
@@ -537,6 +600,8 @@ def define_options(parser):
         default="8",
         help="Data access latency in L2 cache",
     )
+    parser.add_argument("--fabric-clock", type=str, default="1080MHz")
+    parser.add_argument("--memory-clock", type=str, default="1000MHz")
 
 
 def construct_dirs(options, system, ruby_system, network):
@@ -631,7 +696,11 @@ def construct_gpudirs(options, system, ruby_system, network):
             xorHighBit=xor_low_bit,
         )
 
-        dir_cntrl = DirCntrl(noTCCdir=True, TCC_select_num_bits=TCC_bits)
+        dir_cntrl = DirCntrl(
+            noTCCdir=True,
+            TCC_select_num_bits=TCC_bits,
+            clk_domain=system.fabric_clk,
+        )
         dir_cntrl.create(options, [addr_range], ruby_system, system)
         dir_cntrl.number_of_TBEs = options.num_tbes
         dir_cntrl.useL3OnWT = False
@@ -702,7 +771,9 @@ def construct_gpudirs(options, system, ruby_system, network):
         if issubclass(mem_type, DRAMInterface):
             if options.hbm_ctrl:
                 mem_ctrl = m5.objects.HBMCtrl(
-                    dram=dram_intf, dram_2=dram_intf_2
+                    dram=dram_intf,
+                    dram_2=dram_intf_2,
+                    clk_domain=system.memory_clk,
                 )
             else:
                 mem_ctrl = m5.objects.MemCtrl(dram=dram_intf)
@@ -780,7 +851,7 @@ def construct_tcps(options, system, ruby_system, network):
         tcp_cntrl.WB = options.WB_L1
         tcp_cntrl.disableL1 = options.noL1
         tcp_cntrl.L1cache.tagAccessLatency = options.TCP_latency
-        tcp_cntrl.L1cache.dataAccessLatency = options.TCP_latency
+        tcp_cntrl.L1cache.dataAccessLatency = options.TCP_latency_data
 
         exec("ruby_system.tcp_cntrl%d = tcp_cntrl" % i)
         #
@@ -851,7 +922,7 @@ def construct_scalars(options, system, ruby_system, network):
     TCC_bits = int(math.log(options.num_tccs, 2))
 
     for i in range(options.num_scalar_cache):
-        scalar_cntrl = SQCCntrl(TCC_select_num_bits=TCC_bits)
+        scalar_cntrl = ScalarCntrl(TCC_select_num_bits=TCC_bits)
         scalar_cntrl.create(options, ruby_system, system)
 
         exec("ruby_system.scalar_cntrl%d = scalar_cntrl" % i)
@@ -936,7 +1007,10 @@ def construct_tccs(options, system, ruby_system, network):
     tcc_cntrl_nodes = []
 
     for i in range(options.num_tccs):
-        tcc_cntrl = TCCCntrl(l2_response_latency=options.TCC_latency)
+        tcc_cntrl = TCCCntrl(
+            l2_response_latency=options.TCC_latency,
+            clk_domain=system.fabric_clk,
+        )
         tcc_cntrl.create(options, ruby_system, system)
         tcc_cntrl.l2_request_latency = options.gpu_to_dir_latency
         tcc_cntrl.l2_response_latency = options.TCC_latency