mem-ruby,configs: honor configurable Ruby link width

configs/example/gpufs/Disjoint_VIPER.py

@@ -50,6 +50,7 @@ def __init__(self):
         super().__init__()
 
     def create(self, options, system, piobus, dma_devices):
+        self.clk_domain = system.fabric_clk
         # Disjoint network topology
         if "garnet" in options.network:
             self.network_cpu = DisjointGarnet(self)

configs/example/gpufs/mi300.py

@@ -71,17 +71,18 @@
     sh /home/gem5/load_amdgpu.sh
 elif [ ! -f /lib/modules/`uname -r`/updates/dkms/amdgpu.ko ]; then
     echo "ERROR: Missing DKMS package for kernel `uname -r`. Exiting gem5."
-    /sbin/m5 exit
+    # m5 exit
 else
     # Backward compatibility with old disk images (ROCm 6.1)
     modprobe -v amdgpu ip_block_mask=0x6f ppfeaturemask=0 dpm=0 audio=0 ras_enable=0 discovery=2
 fi
+modprobe -v amdgpu ip_block_mask=0x6f ppfeaturemask=0 dpm=0 audio=0 ras_enable=0 discovery=2
 
 echo "Running {} {}"
 echo "{}" | base64 -d > myapp
 chmod +x myapp
 ./myapp {}
-/sbin/m5 exit
+m5 exit
 """
 
 demo_runscript_with_checkpoint = """\

configs/example/gpufs/system/amdgpu.py

@@ -36,7 +36,10 @@ def createGPU(system, args):
         n_wf=args.wfs_per_simd,
         cu_per_sqc=args.cu_per_sqc,
         timing=True,
-        clk_domain=system.clk_domain,
+        clk_domain=SrcClockDomain(
+            clock=args.gpu_clock,
+            voltage_domain=VoltageDomain(voltage=args.gpu_voltage),
+        ),
         progress_interval=args.gpu_progress_interval,
     )
 

configs/example/gpufs/system/system.py

@@ -54,6 +54,8 @@ def makeGpuFSSystem(args):
         "drm_kms_helper.fbdev_emulation=0",
         "modprobe.blacklist=amdgpu",
         "modprobe.blacklist=psmouse",
+        # Tell linux to use MP table for PCI IRQs and not ACPI.
+        "pci=noacpi",
     ]
     cmdline = " ".join(boot_options)
 
@@ -73,6 +75,13 @@ def makeGpuFSSystem(args):
     )
     system.workload.object_file = binary(args.kernel)
 
+    # FADT pointing at a minimal DSDT. This prevents Linux from disabling
+    # ACPI which is needed by the WMI module which is a dependency for the
+    # amdgpu module.
+    fadt = X86ACPIFADT(dsdt=X86ACPIDSDT(), oem_id="gem5")
+    system.workload.acpi_description_table_pointer.rsdt.entries.append(fadt)
+    system.workload.acpi_description_table_pointer.xsdt.entries.append(fadt)
+
     # Set the cache line size for the entire system.
     system.cache_line_size = args.cacheline_size
 
@@ -302,6 +311,13 @@ def makeGpuFSSystem(args):
         pm4_proc.pio = system.iobus.mem_side_ports
     system_hub.pio = system.iobus.mem_side_ports
 
+    system.fabric_clk = SrcClockDomain(
+        clock=args.fabric_clock, voltage_domain=system.voltage_domain
+    )
+    system.memory_clk = SrcClockDomain(
+        clock=args.memory_clock, voltage_domain=system.voltage_domain
+    )
+
     # Full system needs special TLBs for SQC, Scalar, and vector data ports
     args.full_system = True
     GPUTLBConfig.config_tlb_hierarchy(
@@ -313,9 +329,10 @@ def makeGpuFSSystem(args):
     system.ruby.create(args, system, system.iobus, system._dma_ports)
 
     # Create a seperate clock domain for Ruby
-    system.ruby.clk_domain = SrcClockDomain(
-        clock=args.ruby_clock, voltage_domain=system.voltage_domain
-    )
+    # system.ruby.clk_domain = SrcClockDomain(
+    #     clock=args.ruby_clock, voltage_domain=system.voltage_domain
+    # )
+    system.ruby.clk_domain = system.fabric_clk
 
     # If we are using KVM cpu, enable AVX. AVX is used in some ROCm libraries
     # such as rocBLAS which is used in higher level libraries like PyTorch.

configs/ruby/GPU_VIPER.py

@@ -165,7 +165,7 @@ def create(self, options, ruby_system, system):
         self.L1cache.dataArrayBanks = options.tcp_num_banks
         self.L1cache.tagArrayBanks = options.tcp_num_banks
         self.L1cache.create(options)
-        self.issue_latency = 1
+        self.issue_latency = options.tcp_issue_latency
         # TCP_Controller inherits this from RubyController
         self.mandatory_queue_latency = options.mandatory_queue_latency
 
@@ -320,9 +320,11 @@ def create(self, options, ruby_system, system):
         self.L2cache.resourceStalls = options.no_tcc_resource_stalls
 
         self.ruby_system = ruby_system
-        if hasattr(options, "gpu_clock") and hasattr(options, "gpu_voltage"):
+        if hasattr(options, "fabric_clock") and hasattr(
+            options, "gpu_voltage"
+        ):
             self.clk_domain = SrcClockDomain(
-                clock=options.gpu_clock,
+                clock=options.fabric_clock,
                 voltage_domain=VoltageDomain(voltage=options.gpu_voltage),
             )
 
@@ -424,6 +426,7 @@ def connectWireBuffers(
 
 def define_options(parser):
     parser.add_argument("--num-subcaches", type=int, default=4)
+    parser.add_argument("--tcp-issue-latency", type=int, default=1)
     parser.add_argument("--l3-data-latency", type=int, default=20)
     parser.add_argument("--l3-tag-latency", type=int, default=15)
     parser.add_argument("--cpu-to-dir-latency", type=int, default=120)
@@ -465,7 +468,17 @@ def define_options(parser):
         "--TCP_latency",
         type=int,
         default=4,
-        help="In combination with the number of banks for the "
+        help="Set tcp tag access latency. "
+        "In combination with the number of banks for the "
+        "TCP, this determines how many requests can happen "
+        "per cycle (i.e., the bandwidth)",
+    )
+    parser.add_argument(
+        "--TCP_latency_data",
+        type=int,
+        default=4,
+        help="Set tcp data access latency. "
+        "In combination with the number of banks for the "
         "TCP, this determines how many requests can happen "
         "per cycle (i.e., the bandwidth)",
     )
@@ -537,6 +550,8 @@ def define_options(parser):
         default="8",
         help="Data access latency in L2 cache",
     )
+    parser.add_argument("--fabric-clock", type=str, default="1080MHz")
+    parser.add_argument("--memory-clock", type=str, default="1000MHz")
 
 
 def construct_dirs(options, system, ruby_system, network):
@@ -631,7 +646,11 @@ def construct_gpudirs(options, system, ruby_system, network):
             xorHighBit=xor_low_bit,
         )
 
-        dir_cntrl = DirCntrl(noTCCdir=True, TCC_select_num_bits=TCC_bits)
+        dir_cntrl = DirCntrl(
+            noTCCdir=True,
+            TCC_select_num_bits=TCC_bits,
+            clk_domain=system.fabric_clk,
+        )
         dir_cntrl.create(options, [addr_range], ruby_system, system)
         dir_cntrl.number_of_TBEs = options.num_tbes
         dir_cntrl.useL3OnWT = False
@@ -702,7 +721,9 @@ def construct_gpudirs(options, system, ruby_system, network):
         if issubclass(mem_type, DRAMInterface):
             if options.hbm_ctrl:
                 mem_ctrl = m5.objects.HBMCtrl(
-                    dram=dram_intf, dram_2=dram_intf_2
+                    dram=dram_intf,
+                    dram_2=dram_intf_2,
+                    clk_domain=system.memory_clk,
                 )
             else:
                 mem_ctrl = m5.objects.MemCtrl(dram=dram_intf)
@@ -780,7 +801,7 @@ def construct_tcps(options, system, ruby_system, network):
         tcp_cntrl.WB = options.WB_L1
         tcp_cntrl.disableL1 = options.noL1
         tcp_cntrl.L1cache.tagAccessLatency = options.TCP_latency
-        tcp_cntrl.L1cache.dataAccessLatency = options.TCP_latency
+        tcp_cntrl.L1cache.dataAccessLatency = options.TCP_latency_data
 
         exec("ruby_system.tcp_cntrl%d = tcp_cntrl" % i)
         #
@@ -936,7 +957,10 @@ def construct_tccs(options, system, ruby_system, network):
     tcc_cntrl_nodes = []
 
     for i in range(options.num_tccs):
-        tcc_cntrl = TCCCntrl(l2_response_latency=options.TCC_latency)
+        tcc_cntrl = TCCCntrl(
+            l2_response_latency=options.TCC_latency,
+            clk_domain=system.fabric_clk,
+        )
         tcc_cntrl.create(options, ruby_system, system)
         tcc_cntrl.l2_request_latency = options.gpu_to_dir_latency
         tcc_cntrl.l2_response_latency = options.TCC_latency

configs/topologies/Crossbar.py

@@ -39,6 +39,10 @@ def makeTopology(self, options, network, IntLink, ExtLink, Router):
         link_latency = options.link_latency  # used by simple and garnet
         router_latency = options.router_latency  # only used by garnet
 
+        # bandwidth_factor is used by SimpleNetwork (in bytes);
+        # --link-width-bits converts to bytes for consistency.
+        bw = getattr(options, "link_width_bits", 128) // 8
+
         # Create an individual router for each controller plus one more for
         # the centralized crossbar.  The large numbers of routers are needed
         # because external links do not model outgoing bandwidth in the
@@ -57,6 +61,7 @@ def makeTopology(self, options, network, IntLink, ExtLink, Router):
                 ext_node=n,
                 int_node=routers[i],
                 latency=link_latency,
+                bandwidth_factor=bw,
             )
             for (i, n) in enumerate(self.nodes)
         ]
@@ -72,6 +77,7 @@ def makeTopology(self, options, network, IntLink, ExtLink, Router):
                     src_node=routers[i],
                     dst_node=xbar,
                     latency=link_latency,
+                    bandwidth_factor=bw,
                 )
             )
 
@@ -84,6 +90,7 @@ def makeTopology(self, options, network, IntLink, ExtLink, Router):
                     src_node=xbar,
                     dst_node=routers[i],
                     latency=link_latency,
+                    bandwidth_factor=bw,
                 )
             )
 

configs/topologies/CrossbarGarnet.py

@@ -38,11 +38,13 @@ def makeTopology(self, options, network, IntLink, ExtLink, Router):
         # the associated allocator.
         # For simple network, use Crossbar.py
 
+        bw = getattr(options, "link_width_bits", 128) // 8
+
         xbar = Router(router_id=0)
         network.routers = xbar
 
         ext_links = [
-            ExtLink(link_id=i, ext_node=n, int_node=xbar)
+            ExtLink(link_id=i, ext_node=n, int_node=xbar, bandwidth_factor=bw)
             for (i, n) in enumerate(self.nodes)
         ]
         network.ext_links = ext_links

configs/topologies/Mesh_XY.py

@@ -56,6 +56,7 @@ def makeTopology(self, options, network, IntLink, ExtLink, Router):
         # Can be over-ridden on a per link/router basis
         link_latency = options.link_latency  # used by simple and garnet
         router_latency = options.router_latency  # only used by garnet
+        bw = getattr(options, "link_width_bits", 128) // 8
 
         # There must be an evenly divisible number of cntrls to routers
         # Also, obviously the number or rows must be <= the number of routers
@@ -95,6 +96,7 @@ def makeTopology(self, options, network, IntLink, ExtLink, Router):
                     ext_node=n,
                     int_node=routers[router_id],
                     latency=link_latency,
+                    bandwidth_factor=bw,
                 )
             )
             link_count += 1
@@ -110,6 +112,7 @@ def makeTopology(self, options, network, IntLink, ExtLink, Router):
                     ext_node=node,
                     int_node=routers[0],
                     latency=link_latency,
+                    bandwidth_factor=bw,
                 )
             )
             link_count += 1
@@ -147,6 +150,7 @@ def makeTopology(self, options, network, IntLink, ExtLink, Router):
                                 dst_inport="West",
                                 latency=link_latency,
                                 weight=1,
+                                bandwidth_factor=bw,
                                 supported_vnets=[v] if v is not None else [],
                             )
                         )
@@ -168,6 +172,7 @@ def makeTopology(self, options, network, IntLink, ExtLink, Router):
                                 dst_inport="East",
                                 latency=link_latency,
                                 weight=1,
+                                bandwidth_factor=bw,
                                 supported_vnets=[v] if v is not None else [],
                             )
                         )
@@ -189,6 +194,7 @@ def makeTopology(self, options, network, IntLink, ExtLink, Router):
                                 dst_inport="South",
                                 latency=link_latency,
                                 weight=2,
+                                bandwidth_factor=bw,
                                 supported_vnets=[v] if v is not None else [],
                             )
                         )
@@ -210,6 +216,7 @@ def makeTopology(self, options, network, IntLink, ExtLink, Router):
                                 dst_inport="North",
                                 latency=link_latency,
                                 weight=2,
+                                bandwidth_factor=bw,
                                 supported_vnets=[v] if v is not None else [],
                             )
                         )