diff --git a/Documentation/ABI/testing/debugfs-cxl b/Documentation/ABI/testing/debugfs-cxl index fe61d372e3fac..c61f9b8139730 100644 --- a/Documentation/ABI/testing/debugfs-cxl +++ b/Documentation/ABI/testing/debugfs-cxl @@ -33,3 +33,37 @@ Description: device cannot clear poison from the address, -ENXIO is returned. The clear_poison attribute is only visible for devices supporting the capability. + +What: /sys/kernel/debug/cxl/einj_types +Date: January, 2024 +KernelVersion: v6.9 +Contact: linux-cxl@vger.kernel.org +Description: + (RO) Prints the CXL protocol error types made available by + the platform in the format: + + 0x + + The possible error types are (as of ACPI v6.5): + + 0x1000 CXL.cache Protocol Correctable + 0x2000 CXL.cache Protocol Uncorrectable non-fatal + 0x4000 CXL.cache Protocol Uncorrectable fatal + 0x8000 CXL.mem Protocol Correctable + 0x10000 CXL.mem Protocol Uncorrectable non-fatal + 0x20000 CXL.mem Protocol Uncorrectable fatal + + The can be written to einj_inject to inject + into a chosen dport. + +What: /sys/kernel/debug/cxl/$dport_dev/einj_inject +Date: January, 2024 +KernelVersion: v6.9 +Contact: linux-cxl@vger.kernel.org +Description: + (WO) Writing an integer to this file injects the corresponding + CXL protocol error into $dport_dev ($dport_dev will be a device + name from /sys/bus/pci/devices). The integer to type mapping for + injection can be found by reading from einj_types. If the dport + was enumerated in RCH mode, a CXL 1.1 error is injected, otherwise + a CXL 2.0 error is injected. diff --git a/Documentation/ABI/testing/sysfs-bus-cxl b/Documentation/ABI/testing/sysfs-bus-cxl index 087f762ebfd53..3f5627a1210a1 100644 --- a/Documentation/ABI/testing/sysfs-bus-cxl +++ b/Documentation/ABI/testing/sysfs-bus-cxl @@ -28,6 +28,23 @@ Description: Payload in the CXL-2.0 specification. +What: /sys/bus/cxl/devices/memX/ram/qos_class +Date: May, 2023 +KernelVersion: v6.8 +Contact: linux-cxl@vger.kernel.org +Description: + (RO) For CXL host platforms that support "QoS Telemmetry" + this attribute conveys a comma delimited list of platform + specific cookies that identifies a QoS performance class + for the volatile partition of the CXL mem device. These + class-ids can be compared against a similar "qos_class" + published for a root decoder. While it is not required + that the endpoints map their local memory-class to a + matching platform class, mismatches are not recommended + and there are platform specific performance related + side-effects that may result. First class-id is displayed. + + What: /sys/bus/cxl/devices/memX/pmem/size Date: December, 2020 KernelVersion: v5.12 @@ -38,6 +55,23 @@ Description: Payload in the CXL-2.0 specification. +What: /sys/bus/cxl/devices/memX/pmem/qos_class +Date: May, 2023 +KernelVersion: v6.8 +Contact: linux-cxl@vger.kernel.org +Description: + (RO) For CXL host platforms that support "QoS Telemmetry" + this attribute conveys a comma delimited list of platform + specific cookies that identifies a QoS performance class + for the persistent partition of the CXL mem device. These + class-ids can be compared against a similar "qos_class" + published for a root decoder. While it is not required + that the endpoints map their local memory-class to a + matching platform class, mismatches are not recommended + and there are platform specific performance related + side-effects that may result. First class-id is displayed. + + What: /sys/bus/cxl/devices/memX/serial Date: January, 2022 KernelVersion: v5.18 @@ -178,6 +212,21 @@ Description: hardware decoder target list. +What: /sys/bus/cxl/devices/portX/decoders_committed +Date: October, 2023 +KernelVersion: v6.7 +Contact: linux-cxl@vger.kernel.org +Description: + (RO) A memory device is considered active when any of its + decoders are in the "committed" state (See CXL 3.0 8.2.4.19.7 + CXL HDM Decoder n Control Register). Hotplug and destructive + operations like "sanitize" are blocked while device is actively + decoding a Host Physical Address range. Note that this number + may be elevated without any regionX objects active or even + enumerated, as this may be due to decoders established by + platform firwmare or a previous kernel (kexec). + + What: /sys/bus/cxl/devices/decoderX.Y Date: June, 2021 KernelVersion: v5.14 @@ -369,6 +418,21 @@ Description: provided it is currently idle / not bound to a driver. +What: /sys/bus/cxl/devices/decoderX.Y/qos_class +Date: May, 2023 +KernelVersion: v6.5 +Contact: linux-cxl@vger.kernel.org +Description: + (RO) For CXL host platforms that support "QoS Telemmetry" this + root-decoder-only attribute conveys a platform specific cookie + that identifies a QoS performance class for the CXL Window. + This class-id can be compared against a similar "qos_class" + published for each memory-type that an endpoint supports. While + it is not required that endpoints map their local memory-class + to a matching platform class, mismatches are not recommended and + there are platform specific side-effects that may result. + + What: /sys/bus/cxl/devices/regionZ/uuid Date: May, 2022 KernelVersion: v6.0 @@ -488,3 +552,37 @@ Description: attribute is only visible for devices supporting the capability. The retrieved errors are logged as kernel events when cxl_poison event tracing is enabled. + + +What: /sys/bus/cxl/devices/regionZ/accessY/read_bandwidth + /sys/bus/cxl/devices/regionZ/accessY/write_banwidth +Date: Jan, 2024 +KernelVersion: v6.9 +Contact: linux-cxl@vger.kernel.org +Description: + (RO) The aggregated read or write bandwidth of the region. The + number is the accumulated read or write bandwidth of all CXL memory + devices that contributes to the region in MB/s. It is + identical data that should appear in + /sys/devices/system/node/nodeX/accessY/initiators/read_bandwidth or + /sys/devices/system/node/nodeX/accessY/initiators/write_bandwidth. + See Documentation/ABI/stable/sysfs-devices-node. access0 provides + the number to the closest initiator and access1 provides the + number to the closest CPU. + + +What: /sys/bus/cxl/devices/regionZ/accessY/read_latency + /sys/bus/cxl/devices/regionZ/accessY/write_latency +Date: Jan, 2024 +KernelVersion: v6.9 +Contact: linux-cxl@vger.kernel.org +Description: + (RO) The read or write latency of the region. The number is + the worst read or write latency of all CXL memory devices that + contributes to the region in nanoseconds. It is identical data + that should appear in + /sys/devices/system/node/nodeX/accessY/initiators/read_latency or + /sys/devices/system/node/nodeX/accessY/initiators/write_latency. + See Documentation/ABI/stable/sysfs-devices-node. access0 provides + the number to the closest initiator and access1 provides the + number to the closest CPU. diff --git a/Documentation/firmware-guide/acpi/apei/einj.rst b/Documentation/firmware-guide/acpi/apei/einj.rst index d6b61d22f5258..c52b9da08fa9a 100644 --- a/Documentation/firmware-guide/acpi/apei/einj.rst +++ b/Documentation/firmware-guide/acpi/apei/einj.rst @@ -32,6 +32,10 @@ configuration:: CONFIG_ACPI_APEI CONFIG_ACPI_APEI_EINJ +...and to (optionally) enable CXL protocol error injection set:: + + CONFIG_ACPI_APEI_EINJ_CXL + The EINJ user interface is in /apei/einj. The following files belong to it: @@ -118,6 +122,24 @@ The following files belong to it: this actually works depends on what operations the BIOS actually includes in the trigger phase. +CXL error types are supported from ACPI 6.5 onwards (given a CXL port +is present). The EINJ user interface for CXL error types is at +/cxl. The following files belong to it: + +- einj_types: + + Provides the same functionality as available_error_types above, but + for CXL error types + +- $dport_dev/einj_inject: + + Injects a CXL error type into the CXL port represented by $dport_dev, + where $dport_dev is the name of the CXL port (usually a PCIe device name). + Error injections targeting a CXL 2.0+ port can use the legacy interface + under /apei/einj, while CXL 1.1/1.0 port injections + must use this file. + + BIOS versions based on the ACPI 4.0 specification have limited options in controlling where the errors are injected. Your BIOS may support an extension (enabled with the param_extension=1 module parameter, or boot @@ -181,6 +203,18 @@ You should see something like this in dmesg:: [22715.834759] EDAC sbridge MC3: PROCESSOR 0:306e7 TIME 1422553404 SOCKET 0 APIC 0 [22716.616173] EDAC MC3: 1 CE memory read error on CPU_SrcID#0_Channel#0_DIMM#0 (channel:0 slot:0 page:0x12345 offset:0x0 grain:32 syndrome:0x0 - area:DRAM err_code:0001:0090 socket:0 channel_mask:1 rank:0) +A CXL error injection example with $dport_dev=0000:e0:01.1:: + + # cd /sys/kernel/debug/cxl/ + # ls + 0000:e0:01.1 0000:0c:00.0 + # cat einj_types # See which errors can be injected + 0x00008000 CXL.mem Protocol Correctable + 0x00010000 CXL.mem Protocol Uncorrectable non-fatal + 0x00020000 CXL.mem Protocol Uncorrectable fatal + # cd 0000:e0:01.1 # Navigate to dport to inject into + # echo 0x8000 > einj_inject # Inject error + Special notes for injection into SGX enclaves: There may be a separate BIOS setup option to enable SGX injection. diff --git a/MAINTAINERS b/MAINTAINERS index b413d5c97d413..12d1ef1a37503 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -294,6 +294,8 @@ F: drivers/pnp/pnpacpi/ F: include/acpi/ F: include/linux/acpi.h F: include/linux/fwnode.h +F: include/linux/fw_table.h +F: lib/fw_table.c F: tools/power/acpi/ ACPI APEI @@ -5222,7 +5224,10 @@ M: Dan Williams L: linux-cxl@vger.kernel.org S: Maintained F: drivers/cxl/ +F: include/linux/cxl-einj.h +F: include/linux/cxl-event.h F: include/uapi/linux/cxl_mem.h +F: tools/testing/cxl/ COMPUTE EXPRESS LINK PMU (CPMU) M: Jonathan Cameron diff --git a/drivers/acpi/Kconfig b/drivers/acpi/Kconfig index 648228831f5e8..9d5e09a79fbd4 100644 --- a/drivers/acpi/Kconfig +++ b/drivers/acpi/Kconfig @@ -12,6 +12,7 @@ menuconfig ACPI select PNP select NLS select CRC32 + select FIRMWARE_TABLE default y if X86 help Advanced Configuration and Power Interface (ACPI) support for diff --git a/drivers/acpi/apei/Kconfig b/drivers/acpi/apei/Kconfig index 71e0d64a7792e..070c07d68dfb2 100644 --- a/drivers/acpi/apei/Kconfig +++ b/drivers/acpi/apei/Kconfig @@ -61,6 +61,19 @@ config ACPI_APEI_EINJ mainly used for debugging and testing the other parts of APEI and some other RAS features. +config ACPI_APEI_EINJ_CXL + bool "CXL Error INJection Support" + default ACPI_APEI_EINJ + depends on ACPI_APEI_EINJ + depends on CXL_BUS && CXL_BUS <= ACPI_APEI_EINJ + help + Support for CXL protocol Error INJection through debugfs/cxl. + Availability and which errors are supported is dependent on + the host platform. Look to ACPI v6.5 section 18.6.4 and kernel + EINJ documentation for more information. + + If unsure say 'n' + config ACPI_APEI_ERST_DEBUG tristate "APEI Error Record Serialization Table (ERST) Debug Support" depends on ACPI_APEI diff --git a/drivers/acpi/apei/Makefile b/drivers/acpi/apei/Makefile index 4dfac2128737c..2c474e6477e12 100644 --- a/drivers/acpi/apei/Makefile +++ b/drivers/acpi/apei/Makefile @@ -2,6 +2,8 @@ obj-$(CONFIG_ACPI_APEI) += apei.o obj-$(CONFIG_ACPI_APEI_GHES) += ghes.o obj-$(CONFIG_ACPI_APEI_EINJ) += einj.o +einj-y := einj-core.o +einj-$(CONFIG_ACPI_APEI_EINJ_CXL) += einj-cxl.o obj-$(CONFIG_ACPI_APEI_ERST_DEBUG) += erst-dbg.o apei-y := apei-base.o hest.o erst.o bert.o diff --git a/drivers/acpi/apei/apei-internal.h b/drivers/acpi/apei/apei-internal.h index 67c2c3b959e15..cd2766c69d78d 100644 --- a/drivers/acpi/apei/apei-internal.h +++ b/drivers/acpi/apei/apei-internal.h @@ -130,4 +130,22 @@ static inline u32 cper_estatus_len(struct acpi_hest_generic_status *estatus) } int apei_osc_setup(void); + +int einj_get_available_error_type(u32 *type); +int einj_error_inject(u32 type, u32 flags, u64 param1, u64 param2, u64 param3, + u64 param4); +int einj_cxl_rch_error_inject(u32 type, u32 flags, u64 param1, u64 param2, + u64 param3, u64 param4); +bool einj_is_cxl_error_type(u64 type); +int einj_validate_error_type(u64 type); + +#ifndef ACPI_EINJ_CXL_CACHE_CORRECTABLE +#define ACPI_EINJ_CXL_CACHE_CORRECTABLE BIT(12) +#define ACPI_EINJ_CXL_CACHE_UNCORRECTABLE BIT(13) +#define ACPI_EINJ_CXL_CACHE_FATAL BIT(14) +#define ACPI_EINJ_CXL_MEM_CORRECTABLE BIT(15) +#define ACPI_EINJ_CXL_MEM_UNCORRECTABLE BIT(16) +#define ACPI_EINJ_CXL_MEM_FATAL BIT(17) +#endif + #endif diff --git a/drivers/acpi/apei/einj.c b/drivers/acpi/apei/einj-core.c similarity index 90% rename from drivers/acpi/apei/einj.c rename to drivers/acpi/apei/einj-core.c index 89fb9331c611e..bb9f8475ce594 100644 --- a/drivers/acpi/apei/einj.c +++ b/drivers/acpi/apei/einj-core.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include "apei-internal.h" @@ -36,6 +37,12 @@ #define MEM_ERROR_MASK (ACPI_EINJ_MEMORY_CORRECTABLE | \ ACPI_EINJ_MEMORY_UNCORRECTABLE | \ ACPI_EINJ_MEMORY_FATAL) +#define CXL_ERROR_MASK (ACPI_EINJ_CXL_CACHE_CORRECTABLE | \ + ACPI_EINJ_CXL_CACHE_UNCORRECTABLE | \ + ACPI_EINJ_CXL_CACHE_FATAL | \ + ACPI_EINJ_CXL_MEM_CORRECTABLE | \ + ACPI_EINJ_CXL_MEM_UNCORRECTABLE | \ + ACPI_EINJ_CXL_MEM_FATAL) /* * ACPI version 5 provides a SET_ERROR_TYPE_WITH_ADDRESS action. @@ -137,6 +144,11 @@ static struct apei_exec_ins_type einj_ins_type[] = { */ static DEFINE_MUTEX(einj_mutex); +/* + * Exported APIs use this flag to exit early if einj_probe() failed. + */ +bool einj_initialized __ro_after_init; + static void *einj_param; static void einj_exec_ctx_init(struct apei_exec_context *ctx) @@ -160,7 +172,7 @@ static int __einj_get_available_error_type(u32 *type) } /* Get error injection capabilities of the platform */ -static int einj_get_available_error_type(u32 *type) +int einj_get_available_error_type(u32 *type) { int rc; @@ -530,8 +542,8 @@ static int __einj_error_inject(u32 type, u32 flags, u64 param1, u64 param2, } /* Inject the specified hardware error */ -static int einj_error_inject(u32 type, u32 flags, u64 param1, u64 param2, - u64 param3, u64 param4) +int einj_error_inject(u32 type, u32 flags, u64 param1, u64 param2, u64 param3, + u64 param4) { int rc; u64 base_addr, size; @@ -554,8 +566,17 @@ static int einj_error_inject(u32 type, u32 flags, u64 param1, u64 param2, if (type & ACPI5_VENDOR_BIT) { if (vendor_flags != SETWA_FLAGS_MEM) goto inject; - } else if (!(type & MEM_ERROR_MASK) && !(flags & SETWA_FLAGS_MEM)) + } else if (!(type & MEM_ERROR_MASK) && !(flags & SETWA_FLAGS_MEM)) { goto inject; + } + + /* + * Injections targeting a CXL 1.0/1.1 port have to be injected + * via the einj_cxl_rch_error_inject() path as that does the proper + * validation of the given RCRB base (MMIO) address. + */ + if (einj_is_cxl_error_type(type) && (flags & SETWA_FLAGS_MEM)) + return -EINVAL; /* * Disallow crazy address masks that give BIOS leeway to pick @@ -587,6 +608,21 @@ static int einj_error_inject(u32 type, u32 flags, u64 param1, u64 param2, return rc; } +int einj_cxl_rch_error_inject(u32 type, u32 flags, u64 param1, u64 param2, + u64 param3, u64 param4) +{ + int rc; + + if (!(einj_is_cxl_error_type(type) && (flags & SETWA_FLAGS_MEM))) + return -EINVAL; + + mutex_lock(&einj_mutex); + rc = __einj_error_inject(type, flags, param1, param2, param3, param4); + mutex_unlock(&einj_mutex); + + return rc; +} + static u32 error_type; static u32 error_flags; static u64 error_param1; @@ -607,12 +643,6 @@ static struct { u32 mask; const char *str; } const einj_error_type_string[] = { { BIT(9), "Platform Correctable" }, { BIT(10), "Platform Uncorrectable non-fatal" }, { BIT(11), "Platform Uncorrectable fatal"}, - { BIT(12), "CXL.cache Protocol Correctable" }, - { BIT(13), "CXL.cache Protocol Uncorrectable non-fatal" }, - { BIT(14), "CXL.cache Protocol Uncorrectable fatal" }, - { BIT(15), "CXL.mem Protocol Correctable" }, - { BIT(16), "CXL.mem Protocol Uncorrectable non-fatal" }, - { BIT(17), "CXL.mem Protocol Uncorrectable fatal" }, { BIT(31), "Vendor Defined Error Types" }, }; @@ -641,22 +671,26 @@ static int error_type_get(void *data, u64 *val) return 0; } -static int error_type_set(void *data, u64 val) +bool einj_is_cxl_error_type(u64 type) { + return (type & CXL_ERROR_MASK) && (!(type & ACPI5_VENDOR_BIT)); +} + +int einj_validate_error_type(u64 type) +{ + u32 tval, vendor, available_error_type = 0; int rc; - u32 available_error_type = 0; - u32 tval, vendor; /* Only low 32 bits for error type are valid */ - if (val & GENMASK_ULL(63, 32)) + if (type & GENMASK_ULL(63, 32)) return -EINVAL; /* * Vendor defined types have 0x80000000 bit set, and * are not enumerated by ACPI_EINJ_GET_ERROR_TYPE */ - vendor = val & ACPI5_VENDOR_BIT; - tval = val & 0x7fffffff; + vendor = type & ACPI5_VENDOR_BIT; + tval = type & GENMASK(30, 0); /* Only one error type can be specified */ if (tval & (tval - 1)) @@ -665,9 +699,21 @@ static int error_type_set(void *data, u64 val) rc = einj_get_available_error_type(&available_error_type); if (rc) return rc; - if (!(val & available_error_type)) + if (!(type & available_error_type)) return -EINVAL; } + + return 0; +} + +static int error_type_set(void *data, u64 val) +{ + int rc; + + rc = einj_validate_error_type(val); + if (rc) + return rc; + error_type = val; return 0; @@ -703,21 +749,21 @@ static int einj_check_table(struct acpi_table_einj *einj_tab) return 0; } -static int __init einj_init(void) +static int __init einj_probe(struct platform_device *pdev) { int rc; acpi_status status; struct apei_exec_context ctx; if (acpi_disabled) { - pr_info("ACPI disabled.\n"); + pr_debug("ACPI disabled.\n"); return -ENODEV; } status = acpi_get_table(ACPI_SIG_EINJ, 0, (struct acpi_table_header **)&einj_tab); if (status == AE_NOT_FOUND) { - pr_warn("EINJ table not found.\n"); + pr_debug("EINJ table not found.\n"); return -ENODEV; } else if (ACPI_FAILURE(status)) { pr_err("Failed to get EINJ table: %s\n", @@ -805,7 +851,7 @@ static int __init einj_init(void) return rc; } -static void __exit einj_exit(void) +static void einj_remove(struct platform_device *pdev) { struct apei_exec_context ctx; @@ -826,6 +872,40 @@ static void __exit einj_exit(void) acpi_put_table((struct acpi_table_header *)einj_tab); } +static struct platform_device *einj_dev; +static struct platform_driver einj_driver = { + .remove_new = einj_remove, + .driver = { + .name = "acpi-einj", + }, +}; + +static int __init einj_init(void) +{ + struct platform_device_info einj_dev_info = { + .name = "acpi-einj", + .id = -1, + }; + int rc; + + einj_dev = platform_device_register_full(&einj_dev_info); + if (IS_ERR(einj_dev)) + return PTR_ERR(einj_dev); + + rc = platform_driver_probe(&einj_driver, einj_probe); + einj_initialized = rc == 0; + + return 0; +} + +static void __exit einj_exit(void) +{ + if (einj_initialized) + platform_driver_unregister(&einj_driver); + + platform_device_unregister(einj_dev); +} + module_init(einj_init); module_exit(einj_exit); diff --git a/drivers/acpi/apei/einj-cxl.c b/drivers/acpi/apei/einj-cxl.c new file mode 100644 index 0000000000000..d64e2713aae4b --- /dev/null +++ b/drivers/acpi/apei/einj-cxl.c @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * CXL Error INJection support. Used by CXL core to inject + * protocol errors into CXL ports. + * + * Copyright (C) 2023 Advanced Micro Devices, Inc. + * + * Author: Ben Cheatham + */ +#include +#include +#include + +#include "apei-internal.h" + +/* Defined in einj-core.c */ +extern bool einj_initialized; + +static struct { u32 mask; const char *str; } const einj_cxl_error_type_string[] = { + { ACPI_EINJ_CXL_CACHE_CORRECTABLE, "CXL.cache Protocol Correctable" }, + { ACPI_EINJ_CXL_CACHE_UNCORRECTABLE, "CXL.cache Protocol Uncorrectable non-fatal" }, + { ACPI_EINJ_CXL_CACHE_FATAL, "CXL.cache Protocol Uncorrectable fatal" }, + { ACPI_EINJ_CXL_MEM_CORRECTABLE, "CXL.mem Protocol Correctable" }, + { ACPI_EINJ_CXL_MEM_UNCORRECTABLE, "CXL.mem Protocol Uncorrectable non-fatal" }, + { ACPI_EINJ_CXL_MEM_FATAL, "CXL.mem Protocol Uncorrectable fatal" }, +}; + +int einj_cxl_available_error_type_show(struct seq_file *m, void *v) +{ + int cxl_err, rc; + u32 available_error_type = 0; + + rc = einj_get_available_error_type(&available_error_type); + if (rc) + return rc; + + for (int pos = 0; pos < ARRAY_SIZE(einj_cxl_error_type_string); pos++) { + cxl_err = ACPI_EINJ_CXL_CACHE_CORRECTABLE << pos; + + if (available_error_type & cxl_err) + seq_printf(m, "0x%08x\t%s\n", + einj_cxl_error_type_string[pos].mask, + einj_cxl_error_type_string[pos].str); + } + + return 0; +} +EXPORT_SYMBOL_NS_GPL(einj_cxl_available_error_type_show, CXL); + +static int cxl_dport_get_sbdf(struct pci_dev *dport_dev, u64 *sbdf) +{ + struct pci_bus *pbus; + struct pci_host_bridge *bridge; + u64 seg = 0, bus; + + pbus = dport_dev->bus; + bridge = pci_find_host_bridge(pbus); + + if (!bridge) + return -ENODEV; + + if (bridge->domain_nr != PCI_DOMAIN_NR_NOT_SET) + seg = bridge->domain_nr; + + bus = pbus->number; + *sbdf = (seg << 24) | (bus << 16) | (dport_dev->devfn << 8); + + return 0; +} + +int einj_cxl_inject_rch_error(u64 rcrb, u64 type) +{ + int rc; + + /* Only CXL error types can be specified */ + if (!einj_is_cxl_error_type(type)) + return -EINVAL; + + rc = einj_validate_error_type(type); + if (rc) + return rc; + + return einj_cxl_rch_error_inject(type, 0x2, rcrb, GENMASK_ULL(63, 0), + 0, 0); +} +EXPORT_SYMBOL_NS_GPL(einj_cxl_inject_rch_error, CXL); + +int einj_cxl_inject_error(struct pci_dev *dport, u64 type) +{ + u64 param4 = 0; + int rc; + + /* Only CXL error types can be specified */ + if (!einj_is_cxl_error_type(type)) + return -EINVAL; + + rc = einj_validate_error_type(type); + if (rc) + return rc; + + rc = cxl_dport_get_sbdf(dport, ¶m4); + if (rc) + return rc; + + return einj_error_inject(type, 0x4, 0, 0, 0, param4); +} +EXPORT_SYMBOL_NS_GPL(einj_cxl_inject_error, CXL); + +bool einj_cxl_is_initialized(void) +{ + return einj_initialized; +} +EXPORT_SYMBOL_NS_GPL(einj_cxl_is_initialized, CXL); diff --git a/drivers/acpi/apei/ghes.c b/drivers/acpi/apei/ghes.c index e768dfd345fb2..98c5e309445bb 100644 --- a/drivers/acpi/apei/ghes.c +++ b/drivers/acpi/apei/ghes.c @@ -27,6 +27,8 @@ #include #include #include +#include +#include #include #include #include @@ -34,6 +36,7 @@ #include #include #include +#include #include #include #include @@ -675,6 +678,75 @@ static void ghes_defer_non_standard_event(struct acpi_hest_generic_data *gdata, schedule_work(&entry->work); } +/* Room for 8 entries for each of the 4 event log queues */ +#define CXL_CPER_FIFO_DEPTH 32 +DEFINE_KFIFO(cxl_cper_fifo, struct cxl_cper_work_data, CXL_CPER_FIFO_DEPTH); + +/* Synchronize schedule_work() with cxl_cper_work changes */ +static DEFINE_SPINLOCK(cxl_cper_work_lock); +struct work_struct *cxl_cper_work; + +static void cxl_cper_post_event(enum cxl_event_type event_type, + struct cxl_cper_event_rec *rec) +{ + struct cxl_cper_work_data wd; + + if (rec->hdr.length <= sizeof(rec->hdr) || + rec->hdr.length > sizeof(*rec)) { + pr_err(FW_WARN "CXL CPER Invalid section length (%u)\n", + rec->hdr.length); + return; + } + + if (!(rec->hdr.validation_bits & CPER_CXL_COMP_EVENT_LOG_VALID)) { + pr_err(FW_WARN "CXL CPER invalid event\n"); + return; + } + + guard(spinlock_irqsave)(&cxl_cper_work_lock); + + if (!cxl_cper_work) + return; + + wd.event_type = event_type; + memcpy(&wd.rec, rec, sizeof(wd.rec)); + + if (!kfifo_put(&cxl_cper_fifo, wd)) { + pr_err_ratelimited("CXL CPER kfifo overflow\n"); + return; + } + + schedule_work(cxl_cper_work); +} + +int cxl_cper_register_work(struct work_struct *work) +{ + if (cxl_cper_work) + return -EINVAL; + + guard(spinlock)(&cxl_cper_work_lock); + cxl_cper_work = work; + return 0; +} +EXPORT_SYMBOL_NS_GPL(cxl_cper_register_work, CXL); + +int cxl_cper_unregister_work(struct work_struct *work) +{ + if (cxl_cper_work != work) + return -EINVAL; + + guard(spinlock)(&cxl_cper_work_lock); + cxl_cper_work = NULL; + return 0; +} +EXPORT_SYMBOL_NS_GPL(cxl_cper_unregister_work, CXL); + +int cxl_cper_kfifo_get(struct cxl_cper_work_data *wd) +{ + return kfifo_get(&cxl_cper_fifo, wd); +} +EXPORT_SYMBOL_NS_GPL(cxl_cper_kfifo_get, CXL); + static bool ghes_do_proc(struct ghes *ghes, const struct acpi_hest_generic_status *estatus) { @@ -709,6 +781,18 @@ static bool ghes_do_proc(struct ghes *ghes, } else if (guid_equal(sec_type, &CPER_SEC_PROC_ARM)) { queued = ghes_handle_arm_hw_error(gdata, sev, sync); + } else if (guid_equal(sec_type, &CPER_SEC_CXL_GEN_MEDIA_GUID)) { + struct cxl_cper_event_rec *rec = acpi_hest_get_payload(gdata); + + cxl_cper_post_event(CXL_CPER_EVENT_GEN_MEDIA, rec); + } else if (guid_equal(sec_type, &CPER_SEC_CXL_DRAM_GUID)) { + struct cxl_cper_event_rec *rec = acpi_hest_get_payload(gdata); + + cxl_cper_post_event(CXL_CPER_EVENT_DRAM, rec); + } else if (guid_equal(sec_type, &CPER_SEC_CXL_MEM_MODULE_GUID)) { + struct cxl_cper_event_rec *rec = acpi_hest_get_payload(gdata); + + cxl_cper_post_event(CXL_CPER_EVENT_MEM_MODULE, rec); } else { void *err = acpi_hest_get_payload(gdata); diff --git a/drivers/acpi/numa/hmat.c b/drivers/acpi/numa/hmat.c index 1d6450c139225..83be8938462e0 100644 --- a/drivers/acpi/numa/hmat.c +++ b/drivers/acpi/numa/hmat.c @@ -59,7 +59,8 @@ struct target_cache { }; enum { - NODE_ACCESS_CLASS_GENPORT_SINK = ACCESS_COORDINATE_MAX, + NODE_ACCESS_CLASS_GENPORT_SINK_LOCAL = ACCESS_COORDINATE_MAX, + NODE_ACCESS_CLASS_GENPORT_SINK_CPU, NODE_ACCESS_CLASS_MAX, }; @@ -106,6 +107,51 @@ static struct memory_target *find_mem_target(unsigned int mem_pxm) return NULL; } +static struct memory_target *acpi_find_genport_target(u32 uid) +{ + struct memory_target *target; + u32 target_uid; + u8 *uid_ptr; + + list_for_each_entry(target, &targets, node) { + uid_ptr = target->gen_port_device_handle + 8; + target_uid = *(u32 *)uid_ptr; + if (uid == target_uid) + return target; + } + + return NULL; +} + +/** + * acpi_get_genport_coordinates - Retrieve the access coordinates for a generic port + * @uid: ACPI unique id + * @coord: The access coordinates written back out for the generic port. + * Expect 2 levels array. + * + * Return: 0 on success. Errno on failure. + * + * Only supports device handles that are ACPI. Assume ACPI0016 HID for CXL. + */ +int acpi_get_genport_coordinates(u32 uid, + struct access_coordinate *coord) +{ + struct memory_target *target; + + guard(mutex)(&target_lock); + target = acpi_find_genport_target(uid); + if (!target) + return -ENOENT; + + coord[ACCESS_COORDINATE_LOCAL] = + target->coord[NODE_ACCESS_CLASS_GENPORT_SINK_LOCAL]; + coord[ACCESS_COORDINATE_CPU] = + target->coord[NODE_ACCESS_CLASS_GENPORT_SINK_CPU]; + + return 0; +} +EXPORT_SYMBOL_NS_GPL(acpi_get_genport_coordinates, CXL); + static __init void alloc_memory_initiator(unsigned int cpu_pxm) { struct memory_initiator *initiator; @@ -282,6 +328,34 @@ static void hmat_update_target_access(struct memory_target *target, } } +int hmat_update_target_coordinates(int nid, struct access_coordinate *coord, + enum access_coordinate_class access) +{ + struct memory_target *target; + int pxm; + + if (nid == NUMA_NO_NODE) + return -EINVAL; + + pxm = node_to_pxm(nid); + guard(mutex)(&target_lock); + target = find_mem_target(pxm); + if (!target) + return -ENODEV; + + hmat_update_target_access(target, ACPI_HMAT_READ_LATENCY, + coord->read_latency, access); + hmat_update_target_access(target, ACPI_HMAT_WRITE_LATENCY, + coord->write_latency, access); + hmat_update_target_access(target, ACPI_HMAT_READ_BANDWIDTH, + coord->read_bandwidth, access); + hmat_update_target_access(target, ACPI_HMAT_WRITE_BANDWIDTH, + coord->write_bandwidth, access); + + return 0; +} +EXPORT_SYMBOL_GPL(hmat_update_target_coordinates); + static __init void hmat_add_locality(struct acpi_hmat_locality *hmat_loc) { struct memory_locality *loc; @@ -654,7 +728,8 @@ static void hmat_update_target_attrs(struct memory_target *target, int i; /* Don't update for generic port if there's no device handle */ - if (access == NODE_ACCESS_CLASS_GENPORT_SINK && + if ((access == NODE_ACCESS_CLASS_GENPORT_SINK_LOCAL || + access == NODE_ACCESS_CLASS_GENPORT_SINK_CPU) && !(*(u16 *)target->gen_port_device_handle)) return; @@ -695,7 +770,8 @@ static void hmat_update_target_attrs(struct memory_target *target, list_for_each_entry(initiator, &initiators, node) { u32 value; - if (access == ACCESS_COORDINATE_CPU && + if ((access == ACCESS_COORDINATE_CPU || + access == NODE_ACCESS_CLASS_GENPORT_SINK_CPU) && !initiator->has_cpu) { clear_bit(initiator->processor_pxm, p_nodes); continue; @@ -734,7 +810,9 @@ static void hmat_update_generic_target(struct memory_target *target) static DECLARE_BITMAP(p_nodes, MAX_NUMNODES); hmat_update_target_attrs(target, p_nodes, - NODE_ACCESS_CLASS_GENPORT_SINK); + NODE_ACCESS_CLASS_GENPORT_SINK_LOCAL); + hmat_update_target_attrs(target, p_nodes, + NODE_ACCESS_CLASS_GENPORT_SINK_CPU); } static void hmat_register_target_initiators(struct memory_target *target) diff --git a/drivers/acpi/numa/srat.c b/drivers/acpi/numa/srat.c index 848942bf883cb..f830d7c499e11 100644 --- a/drivers/acpi/numa/srat.c +++ b/drivers/acpi/numa/srat.c @@ -29,6 +29,8 @@ static int node_to_pxm_map[MAX_NUMNODES] unsigned char acpi_srat_revision __initdata; static int acpi_numa __initdata; +static int last_real_pxm; + void __init disable_srat(void) { acpi_numa = -1; @@ -541,6 +543,7 @@ int __init acpi_numa_init(void) if (node_to_pxm_map[i] > fake_pxm) fake_pxm = node_to_pxm_map[i]; } + last_real_pxm = fake_pxm; fake_pxm++; acpi_table_parse_cedt(ACPI_CEDT_TYPE_CFMWS, acpi_parse_cfmws, &fake_pxm); @@ -552,6 +555,14 @@ int __init acpi_numa_init(void) return 0; } +bool acpi_node_backed_by_real_pxm(int nid) +{ + int pxm = node_to_pxm(nid); + + return pxm <= last_real_pxm; +} +EXPORT_SYMBOL_GPL(acpi_node_backed_by_real_pxm); + static int acpi_get_pxm(acpi_handle h) { unsigned long long pxm; diff --git a/drivers/acpi/tables.c b/drivers/acpi/tables.c index ffa1888844fe6..9e1b01c350702 100644 --- a/drivers/acpi/tables.c +++ b/drivers/acpi/tables.c @@ -37,18 +37,6 @@ static struct acpi_table_desc initial_tables[ACPI_MAX_TABLES] __initdata; static int acpi_apic_instance __initdata_or_acpilib; -enum acpi_subtable_type { - ACPI_SUBTABLE_COMMON, - ACPI_SUBTABLE_HMAT, - ACPI_SUBTABLE_PRMT, - ACPI_SUBTABLE_CEDT, -}; - -struct acpi_subtable_entry { - union acpi_subtable_headers *hdr; - enum acpi_subtable_type type; -}; - /* * Disable table checksum verification for the early stage due to the size * limitation of the current x86 early mapping implementation. @@ -251,167 +239,6 @@ void acpi_table_print_madt_entry(struct acpi_subtable_header *header) } } -static unsigned long __init_or_acpilib -acpi_get_entry_type(struct acpi_subtable_entry *entry) -{ - switch (entry->type) { - case ACPI_SUBTABLE_COMMON: - return entry->hdr->common.type; - case ACPI_SUBTABLE_HMAT: - return entry->hdr->hmat.type; - case ACPI_SUBTABLE_PRMT: - return 0; - case ACPI_SUBTABLE_CEDT: - return entry->hdr->cedt.type; - } - return 0; -} - -static unsigned long __init_or_acpilib -acpi_get_entry_length(struct acpi_subtable_entry *entry) -{ - switch (entry->type) { - case ACPI_SUBTABLE_COMMON: - return entry->hdr->common.length; - case ACPI_SUBTABLE_HMAT: - return entry->hdr->hmat.length; - case ACPI_SUBTABLE_PRMT: - return entry->hdr->prmt.length; - case ACPI_SUBTABLE_CEDT: - return entry->hdr->cedt.length; - } - return 0; -} - -static unsigned long __init_or_acpilib -acpi_get_subtable_header_length(struct acpi_subtable_entry *entry) -{ - switch (entry->type) { - case ACPI_SUBTABLE_COMMON: - return sizeof(entry->hdr->common); - case ACPI_SUBTABLE_HMAT: - return sizeof(entry->hdr->hmat); - case ACPI_SUBTABLE_PRMT: - return sizeof(entry->hdr->prmt); - case ACPI_SUBTABLE_CEDT: - return sizeof(entry->hdr->cedt); - } - return 0; -} - -static enum acpi_subtable_type __init_or_acpilib -acpi_get_subtable_type(char *id) -{ - if (strncmp(id, ACPI_SIG_HMAT, 4) == 0) - return ACPI_SUBTABLE_HMAT; - if (strncmp(id, ACPI_SIG_PRMT, 4) == 0) - return ACPI_SUBTABLE_PRMT; - if (strncmp(id, ACPI_SIG_CEDT, 4) == 0) - return ACPI_SUBTABLE_CEDT; - return ACPI_SUBTABLE_COMMON; -} - -static __init_or_acpilib bool has_handler(struct acpi_subtable_proc *proc) -{ - return proc->handler || proc->handler_arg; -} - -static __init_or_acpilib int call_handler(struct acpi_subtable_proc *proc, - union acpi_subtable_headers *hdr, - unsigned long end) -{ - if (proc->handler) - return proc->handler(hdr, end); - if (proc->handler_arg) - return proc->handler_arg(hdr, proc->arg, end); - return -EINVAL; -} - -/** - * acpi_parse_entries_array - for each proc_num find a suitable subtable - * - * @id: table id (for debugging purposes) - * @table_size: size of the root table - * @table_header: where does the table start? - * @proc: array of acpi_subtable_proc struct containing entry id - * and associated handler with it - * @proc_num: how big proc is? - * @max_entries: how many entries can we process? - * - * For each proc_num find a subtable with proc->id and run proc->handler - * on it. Assumption is that there's only single handler for particular - * entry id. - * - * The table_size is not the size of the complete ACPI table (the length - * field in the header struct), but only the size of the root table; i.e., - * the offset from the very first byte of the complete ACPI table, to the - * first byte of the very first subtable. - * - * On success returns sum of all matching entries for all proc handlers. - * Otherwise, -ENODEV or -EINVAL is returned. - */ -static int __init_or_acpilib acpi_parse_entries_array( - char *id, unsigned long table_size, - struct acpi_table_header *table_header, struct acpi_subtable_proc *proc, - int proc_num, unsigned int max_entries) -{ - struct acpi_subtable_entry entry; - unsigned long table_end, subtable_len, entry_len; - int count = 0; - int errs = 0; - int i; - - table_end = (unsigned long)table_header + table_header->length; - - /* Parse all entries looking for a match. */ - - entry.type = acpi_get_subtable_type(id); - entry.hdr = (union acpi_subtable_headers *) - ((unsigned long)table_header + table_size); - subtable_len = acpi_get_subtable_header_length(&entry); - - while (((unsigned long)entry.hdr) + subtable_len < table_end) { - if (max_entries && count >= max_entries) - break; - - for (i = 0; i < proc_num; i++) { - if (acpi_get_entry_type(&entry) != proc[i].id) - continue; - if (!has_handler(&proc[i]) || - (!errs && - call_handler(&proc[i], entry.hdr, table_end))) { - errs++; - continue; - } - - proc[i].count++; - break; - } - if (i != proc_num) - count++; - - /* - * If entry->length is 0, break from this loop to avoid - * infinite loop. - */ - entry_len = acpi_get_entry_length(&entry); - if (entry_len == 0) { - pr_err("[%4.4s:0x%02x] Invalid zero length\n", id, proc->id); - return -EINVAL; - } - - entry.hdr = (union acpi_subtable_headers *) - ((unsigned long)entry.hdr + entry_len); - } - - if (max_entries && count > max_entries) { - pr_warn("[%4.4s:0x%02x] found the maximum %i entries\n", - id, proc->id, count); - } - - return errs ? -EINVAL : count; -} - int __init_or_acpilib acpi_table_parse_entries_array( char *id, unsigned long table_size, struct acpi_subtable_proc *proc, int proc_num, unsigned int max_entries) @@ -438,8 +265,9 @@ int __init_or_acpilib acpi_table_parse_entries_array( return -ENODEV; } - count = acpi_parse_entries_array(id, table_size, table_header, - proc, proc_num, max_entries); + count = acpi_parse_entries_array(id, table_size, + (union fw_table_header *)table_header, + 0, proc, proc_num, max_entries); acpi_put_table(table_header); return count; diff --git a/drivers/base/node.c b/drivers/base/node.c index 2b398c8a0f06c..2d2c6dc421d97 100644 --- a/drivers/base/node.c +++ b/drivers/base/node.c @@ -215,6 +215,38 @@ void node_set_perf_attrs(unsigned int nid, struct access_coordinate *coord, } } } +EXPORT_SYMBOL_GPL(node_set_perf_attrs); + +/** + * node_update_perf_attrs - Update the performance values for given access class + * @nid: Node identifier to be updated + * @coord: Heterogeneous memory performance coordinates + * @access: The access class for the given attributes + */ +void node_update_perf_attrs(unsigned int nid, struct access_coordinate *coord, + enum access_coordinate_class access) +{ + struct node_access_nodes *access_node; + struct node *node; + int i; + + if (WARN_ON_ONCE(!node_online(nid))) + return; + + node = node_devices[nid]; + list_for_each_entry(access_node, &node->access_list, list_node) { + if (access_node->access != access) + continue; + + access_node->coord = *coord; + for (i = 0; access_attrs[i]; i++) { + sysfs_notify(&access_node->dev.kobj, + NULL, access_attrs[i]->name); + } + break; + } +} +EXPORT_SYMBOL_GPL(node_update_perf_attrs); /** * struct node_cache_info - Internal tracking for memory node caches diff --git a/drivers/cxl/Kconfig b/drivers/cxl/Kconfig index 8ea1d340e4385..67998dbd1d46b 100644 --- a/drivers/cxl/Kconfig +++ b/drivers/cxl/Kconfig @@ -5,6 +5,7 @@ menuconfig CXL_BUS select FW_LOADER select FW_UPLOAD select PCI_DOE + select FIRMWARE_TABLE help CXL is a bus that is electrically compatible with PCI Express, but layers three protocols on that signalling (CXL.io, CXL.cache, and @@ -54,8 +55,10 @@ config CXL_MEM_RAW_COMMANDS config CXL_ACPI tristate "CXL ACPI: Platform Support" depends on ACPI + depends on ACPI_NUMA default CXL_BUS select ACPI_TABLE_LIB + select ACPI_HMAT help Enable support for host managed device memory (HDM) resources published by a platform's ACPI CXL memory layout description. See diff --git a/drivers/cxl/acpi.c b/drivers/cxl/acpi.c index 9a881a764cf35..3f9040b2b27fe 100644 --- a/drivers/cxl/acpi.c +++ b/drivers/cxl/acpi.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include "cxlpci.h" #include "cxl.h" @@ -17,6 +18,10 @@ struct cxl_cxims_data { u64 xormaps[] __counted_by(nr_maps); }; +static const guid_t acpi_cxl_qtg_id_guid = + GUID_INIT(0xF365F9A6, 0xA7DE, 0x4071, + 0xA6, 0x6A, 0xB4, 0x0C, 0x0B, 0x4F, 0x8E, 0x52); + /* * Find a targets entry (n) in the host bridge interleave list. * CXL Specification 3.0 Table 9-22 @@ -194,6 +199,123 @@ struct cxl_cfmws_context { int id; }; +/** + * cxl_acpi_evaluate_qtg_dsm - Retrieve QTG ids via ACPI _DSM + * @handle: ACPI handle + * @coord: performance access coordinates + * @entries: number of QTG IDs to return + * @qos_class: int array provided by caller to return QTG IDs + * + * Return: number of QTG IDs returned, or -errno for errors + * + * Issue QTG _DSM with accompanied bandwidth and latency data in order to get + * the QTG IDs that are suitable for the performance point in order of most + * suitable to least suitable. Write back array of QTG IDs and return the + * actual number of QTG IDs written back. + */ +static int +cxl_acpi_evaluate_qtg_dsm(acpi_handle handle, struct access_coordinate *coord, + int entries, int *qos_class) +{ + union acpi_object *out_obj, *out_buf, *obj; + union acpi_object in_array[4] = { + [0].integer = { ACPI_TYPE_INTEGER, coord->read_latency }, + [1].integer = { ACPI_TYPE_INTEGER, coord->write_latency }, + [2].integer = { ACPI_TYPE_INTEGER, coord->read_bandwidth }, + [3].integer = { ACPI_TYPE_INTEGER, coord->write_bandwidth }, + }; + union acpi_object in_obj = { + .package = { + .type = ACPI_TYPE_PACKAGE, + .count = 4, + .elements = in_array, + }, + }; + int count, pkg_entries, i; + u16 max_qtg; + int rc; + + if (!entries) + return -EINVAL; + + out_obj = acpi_evaluate_dsm(handle, &acpi_cxl_qtg_id_guid, 1, 1, &in_obj); + if (!out_obj) + return -ENXIO; + + if (out_obj->type != ACPI_TYPE_PACKAGE) { + rc = -ENXIO; + goto out; + } + + /* Check Max QTG ID */ + obj = &out_obj->package.elements[0]; + if (obj->type != ACPI_TYPE_INTEGER) { + rc = -ENXIO; + goto out; + } + + max_qtg = obj->integer.value; + + /* It's legal to have 0 QTG entries */ + pkg_entries = out_obj->package.count; + if (pkg_entries <= 1) { + rc = 0; + goto out; + } + + /* Retrieve QTG IDs package */ + obj = &out_obj->package.elements[1]; + if (obj->type != ACPI_TYPE_PACKAGE) { + rc = -ENXIO; + goto out; + } + + pkg_entries = obj->package.count; + count = min(entries, pkg_entries); + for (i = 0; i < count; i++) { + u16 qtg_id; + + out_buf = &obj->package.elements[i]; + if (out_buf->type != ACPI_TYPE_INTEGER) { + rc = -ENXIO; + goto out; + } + + qtg_id = out_buf->integer.value; + if (qtg_id > max_qtg) + pr_warn("QTG ID %u greater than MAX %u\n", + qtg_id, max_qtg); + + qos_class[i] = qtg_id; + } + rc = count; + +out: + ACPI_FREE(out_obj); + return rc; +} + +static int cxl_acpi_qos_class(struct cxl_root *cxl_root, + struct access_coordinate *coord, int entries, + int *qos_class) +{ + struct device *dev = cxl_root->port.uport_dev; + acpi_handle handle; + + if (!dev_is_platform(dev)) + return -ENODEV; + + handle = ACPI_HANDLE(dev); + if (!handle) + return -ENODEV; + + return cxl_acpi_evaluate_qtg_dsm(handle, coord, entries, qos_class); +} + +static const struct cxl_root_ops acpi_root_ops = { + .qos_class = cxl_acpi_qos_class, +}; + static int __cxl_parse_cfmws(struct acpi_cedt_cfmws *cfmws, struct cxl_cfmws_context *ctx) { @@ -285,6 +407,9 @@ static int __cxl_parse_cfmws(struct acpi_cedt_cfmws *cfmws, } } } + + cxlrd->qos_class = cfmws->qtg_id; + rc = cxl_decoder_add(cxld, target_map); err_xormap: if (rc) @@ -396,8 +521,20 @@ static int cxl_get_chbs(struct device *dev, struct acpi_device *hb, return 0; } +static int get_genport_coordinates(struct device *dev, struct cxl_dport *dport) +{ + struct acpi_device *hb = to_cxl_host_bridge(NULL, dev); + u32 uid; + + if (kstrtou32(acpi_device_uid(hb), 0, &uid)) + return -EINVAL; + + return acpi_get_genport_coordinates(uid, dport->coord); +} + static int add_host_bridge_dport(struct device *match, void *arg) { + int ret; acpi_status rc; struct device *bridge; struct cxl_dport *dport; @@ -447,6 +584,10 @@ static int add_host_bridge_dport(struct device *match, void *arg) if (IS_ERR(dport)) return PTR_ERR(dport); + ret = get_genport_coordinates(match, dport); + if (ret) + dev_dbg(match, "Failed to get generic port perf coordinates.\n"); + return 0; } @@ -663,6 +804,7 @@ static int cxl_acpi_probe(struct platform_device *pdev) { int rc; struct resource *cxl_res; + struct cxl_root *cxl_root; struct cxl_port *root_port; struct device *host = &pdev->dev; struct acpi_device *adev = ACPI_COMPANION(host); @@ -682,9 +824,10 @@ static int cxl_acpi_probe(struct platform_device *pdev) cxl_res->end = -1; cxl_res->flags = IORESOURCE_MEM; - root_port = devm_cxl_add_port(host, host, CXL_RESOURCE_NONE, NULL); - if (IS_ERR(root_port)) - return PTR_ERR(root_port); + cxl_root = devm_cxl_add_root(host, &acpi_root_ops); + if (IS_ERR(cxl_root)) + return PTR_ERR(cxl_root); + root_port = &cxl_root->port; rc = bus_for_each_dev(adev->dev.bus, NULL, root_port, add_host_bridge_dport); diff --git a/drivers/cxl/core/Makefile b/drivers/cxl/core/Makefile index 1f66b5d4d9355..9259bcc6773c8 100644 --- a/drivers/cxl/core/Makefile +++ b/drivers/cxl/core/Makefile @@ -13,5 +13,6 @@ cxl_core-y += mbox.o cxl_core-y += pci.o cxl_core-y += hdm.o cxl_core-y += pmu.o +cxl_core-y += cdat.o cxl_core-$(CONFIG_TRACING) += trace.o cxl_core-$(CONFIG_CXL_REGION) += region.o diff --git a/drivers/cxl/core/cdat.c b/drivers/cxl/core/cdat.c new file mode 100644 index 0000000000000..295d4487b2693 --- /dev/null +++ b/drivers/cxl/core/cdat.c @@ -0,0 +1,593 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2023 Intel Corporation. All rights reserved. */ +#include +#include +#include +#include +#include +#include "cxlpci.h" +#include "cxlmem.h" +#include "cxl.h" +#include "core.h" + +struct dsmas_entry { + struct range dpa_range; + u8 handle; + struct access_coordinate coord[ACCESS_COORDINATE_MAX]; + + int entries; + int qos_class; +}; + +static u32 cdat_normalize(u16 entry, u64 base, u8 type) +{ + u32 value; + + /* + * Check for invalid and overflow values + */ + if (entry == 0xffff || !entry) + return 0; + else if (base > (UINT_MAX / (entry))) + return 0; + + /* + * CDAT fields follow the format of HMAT fields. See table 5 Device + * Scoped Latency and Bandwidth Information Structure in Coherent Device + * Attribute Table (CDAT) Specification v1.01. + */ + value = entry * base; + switch (type) { + case ACPI_HMAT_ACCESS_LATENCY: + case ACPI_HMAT_READ_LATENCY: + case ACPI_HMAT_WRITE_LATENCY: + value = DIV_ROUND_UP(value, 1000); + break; + default: + break; + } + return value; +} + +static int cdat_dsmas_handler(union acpi_subtable_headers *header, void *arg, + const unsigned long end) +{ + struct acpi_cdat_header *hdr = &header->cdat; + struct acpi_cdat_dsmas *dsmas; + int size = sizeof(*hdr) + sizeof(*dsmas); + struct xarray *dsmas_xa = arg; + struct dsmas_entry *dent; + u16 len; + int rc; + + len = le16_to_cpu((__force __le16)hdr->length); + if (len != size || (unsigned long)hdr + len > end) { + pr_warn("Malformed DSMAS table length: (%u:%u)\n", size, len); + return -EINVAL; + } + + /* Skip common header */ + dsmas = (struct acpi_cdat_dsmas *)(hdr + 1); + + dent = kzalloc(sizeof(*dent), GFP_KERNEL); + if (!dent) + return -ENOMEM; + + dent->handle = dsmas->dsmad_handle; + dent->dpa_range.start = le64_to_cpu((__force __le64)dsmas->dpa_base_address); + dent->dpa_range.end = le64_to_cpu((__force __le64)dsmas->dpa_base_address) + + le64_to_cpu((__force __le64)dsmas->dpa_length) - 1; + + rc = xa_insert(dsmas_xa, dent->handle, dent, GFP_KERNEL); + if (rc) { + kfree(dent); + return rc; + } + + return 0; +} + +static void __cxl_access_coordinate_set(struct access_coordinate *coord, + int access, unsigned int val) +{ + switch (access) { + case ACPI_HMAT_ACCESS_LATENCY: + coord->read_latency = val; + coord->write_latency = val; + break; + case ACPI_HMAT_READ_LATENCY: + coord->read_latency = val; + break; + case ACPI_HMAT_WRITE_LATENCY: + coord->write_latency = val; + break; + case ACPI_HMAT_ACCESS_BANDWIDTH: + coord->read_bandwidth = val; + coord->write_bandwidth = val; + break; + case ACPI_HMAT_READ_BANDWIDTH: + coord->read_bandwidth = val; + break; + case ACPI_HMAT_WRITE_BANDWIDTH: + coord->write_bandwidth = val; + break; + } +} + +static void cxl_access_coordinate_set(struct access_coordinate *coord, + int access, unsigned int val) +{ + for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) + __cxl_access_coordinate_set(&coord[i], access, val); +} + +static int cdat_dslbis_handler(union acpi_subtable_headers *header, void *arg, + const unsigned long end) +{ + struct acpi_cdat_header *hdr = &header->cdat; + struct acpi_cdat_dslbis *dslbis; + int size = sizeof(*hdr) + sizeof(*dslbis); + struct xarray *dsmas_xa = arg; + struct dsmas_entry *dent; + __le64 le_base; + __le16 le_val; + u64 val; + u16 len; + + len = le16_to_cpu((__force __le16)hdr->length); + if (len != size || (unsigned long)hdr + len > end) { + pr_warn("Malformed DSLBIS table length: (%u:%u)\n", size, len); + return -EINVAL; + } + + /* Skip common header */ + dslbis = (struct acpi_cdat_dslbis *)(hdr + 1); + + /* Skip unrecognized data type */ + if (dslbis->data_type > ACPI_HMAT_WRITE_BANDWIDTH) + return 0; + + /* Not a memory type, skip */ + if ((dslbis->flags & ACPI_HMAT_MEMORY_HIERARCHY) != ACPI_HMAT_MEMORY) + return 0; + + dent = xa_load(dsmas_xa, dslbis->handle); + if (!dent) { + pr_warn("No matching DSMAS entry for DSLBIS entry.\n"); + return 0; + } + + le_base = (__force __le64)dslbis->entry_base_unit; + le_val = (__force __le16)dslbis->entry[0]; + val = cdat_normalize(le16_to_cpu(le_val), le64_to_cpu(le_base), + dslbis->data_type); + + cxl_access_coordinate_set(dent->coord, dslbis->data_type, val); + + return 0; +} + +static int cdat_table_parse_output(int rc) +{ + if (rc < 0) + return rc; + if (rc == 0) + return -ENOENT; + + return 0; +} + +static int cxl_cdat_endpoint_process(struct cxl_port *port, + struct xarray *dsmas_xa) +{ + int rc; + + rc = cdat_table_parse(ACPI_CDAT_TYPE_DSMAS, cdat_dsmas_handler, + dsmas_xa, port->cdat.table, port->cdat.length); + rc = cdat_table_parse_output(rc); + if (rc) + return rc; + + rc = cdat_table_parse(ACPI_CDAT_TYPE_DSLBIS, cdat_dslbis_handler, + dsmas_xa, port->cdat.table, port->cdat.length); + return cdat_table_parse_output(rc); +} + +static int cxl_port_perf_data_calculate(struct cxl_port *port, + struct xarray *dsmas_xa) +{ + struct access_coordinate ep_c[ACCESS_COORDINATE_MAX]; + struct dsmas_entry *dent; + int valid_entries = 0; + unsigned long index; + int rc; + + rc = cxl_endpoint_get_perf_coordinates(port, ep_c); + if (rc) { + dev_dbg(&port->dev, "Failed to retrieve ep perf coordinates.\n"); + return rc; + } + + struct cxl_root *cxl_root __free(put_cxl_root) = find_cxl_root(port); + + if (!cxl_root) + return -ENODEV; + + if (!cxl_root->ops || !cxl_root->ops->qos_class) + return -EOPNOTSUPP; + + xa_for_each(dsmas_xa, index, dent) { + int qos_class; + + cxl_coordinates_combine(dent->coord, dent->coord, ep_c); + dent->entries = 1; + rc = cxl_root->ops->qos_class(cxl_root, + &dent->coord[ACCESS_COORDINATE_CPU], + 1, &qos_class); + if (rc != 1) + continue; + + valid_entries++; + dent->qos_class = qos_class; + } + + if (!valid_entries) + return -ENOENT; + + return 0; +} + +static void update_perf_entry(struct device *dev, struct dsmas_entry *dent, + struct cxl_dpa_perf *dpa_perf) +{ + for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) + dpa_perf->coord[i] = dent->coord[i]; + dpa_perf->dpa_range = dent->dpa_range; + dpa_perf->qos_class = dent->qos_class; + dev_dbg(dev, + "DSMAS: dpa: %#llx qos: %d read_bw: %d write_bw %d read_lat: %d write_lat: %d\n", + dent->dpa_range.start, dpa_perf->qos_class, + dent->coord[ACCESS_COORDINATE_CPU].read_bandwidth, + dent->coord[ACCESS_COORDINATE_CPU].write_bandwidth, + dent->coord[ACCESS_COORDINATE_CPU].read_latency, + dent->coord[ACCESS_COORDINATE_CPU].write_latency); +} + +static void cxl_memdev_set_qos_class(struct cxl_dev_state *cxlds, + struct xarray *dsmas_xa) +{ + struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); + struct device *dev = cxlds->dev; + struct range pmem_range = { + .start = cxlds->pmem_res.start, + .end = cxlds->pmem_res.end, + }; + struct range ram_range = { + .start = cxlds->ram_res.start, + .end = cxlds->ram_res.end, + }; + struct dsmas_entry *dent; + unsigned long index; + + xa_for_each(dsmas_xa, index, dent) { + if (resource_size(&cxlds->ram_res) && + range_contains(&ram_range, &dent->dpa_range)) + update_perf_entry(dev, dent, &mds->ram_perf); + else if (resource_size(&cxlds->pmem_res) && + range_contains(&pmem_range, &dent->dpa_range)) + update_perf_entry(dev, dent, &mds->pmem_perf); + else + dev_dbg(dev, "no partition for dsmas dpa: %#llx\n", + dent->dpa_range.start); + } +} + +static int match_cxlrd_qos_class(struct device *dev, void *data) +{ + int dev_qos_class = *(int *)data; + struct cxl_root_decoder *cxlrd; + + if (!is_root_decoder(dev)) + return 0; + + cxlrd = to_cxl_root_decoder(dev); + if (cxlrd->qos_class == CXL_QOS_CLASS_INVALID) + return 0; + + if (cxlrd->qos_class == dev_qos_class) + return 1; + + return 0; +} + +static void reset_dpa_perf(struct cxl_dpa_perf *dpa_perf) +{ + *dpa_perf = (struct cxl_dpa_perf) { + .qos_class = CXL_QOS_CLASS_INVALID, + }; +} + +static bool cxl_qos_match(struct cxl_port *root_port, + struct cxl_dpa_perf *dpa_perf) +{ + if (dpa_perf->qos_class == CXL_QOS_CLASS_INVALID) + return false; + + if (!device_for_each_child(&root_port->dev, &dpa_perf->qos_class, + match_cxlrd_qos_class)) + return false; + + return true; +} + +static int match_cxlrd_hb(struct device *dev, void *data) +{ + struct device *host_bridge = data; + struct cxl_switch_decoder *cxlsd; + struct cxl_root_decoder *cxlrd; + + if (!is_root_decoder(dev)) + return 0; + + cxlrd = to_cxl_root_decoder(dev); + cxlsd = &cxlrd->cxlsd; + + guard(rwsem_read)(&cxl_region_rwsem); + for (int i = 0; i < cxlsd->nr_targets; i++) { + if (host_bridge == cxlsd->target[i]->dport_dev) + return 1; + } + + return 0; +} + +static int cxl_qos_class_verify(struct cxl_memdev *cxlmd) +{ + struct cxl_dev_state *cxlds = cxlmd->cxlds; + struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); + struct cxl_port *root_port; + int rc; + + struct cxl_root *cxl_root __free(put_cxl_root) = + find_cxl_root(cxlmd->endpoint); + + if (!cxl_root) + return -ENODEV; + + root_port = &cxl_root->port; + + /* Check that the QTG IDs are all sane between end device and root decoders */ + if (!cxl_qos_match(root_port, &mds->ram_perf)) + reset_dpa_perf(&mds->ram_perf); + if (!cxl_qos_match(root_port, &mds->pmem_perf)) + reset_dpa_perf(&mds->pmem_perf); + + /* Check to make sure that the device's host bridge is under a root decoder */ + rc = device_for_each_child(&root_port->dev, + cxlmd->endpoint->host_bridge, match_cxlrd_hb); + if (!rc) { + reset_dpa_perf(&mds->ram_perf); + reset_dpa_perf(&mds->pmem_perf); + } + + return rc; +} + +static void discard_dsmas(struct xarray *xa) +{ + unsigned long index; + void *ent; + + xa_for_each(xa, index, ent) { + xa_erase(xa, index); + kfree(ent); + } + xa_destroy(xa); +} +DEFINE_FREE(dsmas, struct xarray *, if (_T) discard_dsmas(_T)) + +void cxl_endpoint_parse_cdat(struct cxl_port *port) +{ + struct cxl_memdev *cxlmd = to_cxl_memdev(port->uport_dev); + struct cxl_dev_state *cxlds = cxlmd->cxlds; + struct xarray __dsmas_xa; + struct xarray *dsmas_xa __free(dsmas) = &__dsmas_xa; + int rc; + + xa_init(&__dsmas_xa); + if (!port->cdat.table) + return; + + rc = cxl_cdat_endpoint_process(port, dsmas_xa); + if (rc < 0) { + dev_dbg(&port->dev, "Failed to parse CDAT: %d\n", rc); + return; + } + + rc = cxl_port_perf_data_calculate(port, dsmas_xa); + if (rc) { + dev_dbg(&port->dev, "Failed to do perf coord calculations.\n"); + return; + } + + cxl_memdev_set_qos_class(cxlds, dsmas_xa); + cxl_qos_class_verify(cxlmd); + cxl_memdev_update_perf(cxlmd); +} +EXPORT_SYMBOL_NS_GPL(cxl_endpoint_parse_cdat, CXL); + +static int cdat_sslbis_handler(union acpi_subtable_headers *header, void *arg, + const unsigned long end) +{ + struct acpi_cdat_sslbis_table { + struct acpi_cdat_header header; + struct acpi_cdat_sslbis sslbis_header; + struct acpi_cdat_sslbe entries[]; + } *tbl = (struct acpi_cdat_sslbis_table *)header; + int size = sizeof(header->cdat) + sizeof(tbl->sslbis_header); + struct acpi_cdat_sslbis *sslbis; + struct cxl_port *port = arg; + struct device *dev = &port->dev; + int remain, entries, i; + u16 len; + + len = le16_to_cpu((__force __le16)header->cdat.length); + remain = len - size; + if (!remain || remain % sizeof(tbl->entries[0]) || + (unsigned long)header + len > end) { + dev_warn(dev, "Malformed SSLBIS table length: (%u)\n", len); + return -EINVAL; + } + + sslbis = &tbl->sslbis_header; + /* Unrecognized data type, we can skip */ + if (sslbis->data_type > ACPI_HMAT_WRITE_BANDWIDTH) + return 0; + + entries = remain / sizeof(tbl->entries[0]); + if (struct_size(tbl, entries, entries) != len) + return -EINVAL; + + for (i = 0; i < entries; i++) { + u16 x = le16_to_cpu((__force __le16)tbl->entries[i].portx_id); + u16 y = le16_to_cpu((__force __le16)tbl->entries[i].porty_id); + __le64 le_base; + __le16 le_val; + struct cxl_dport *dport; + unsigned long index; + u16 dsp_id; + u64 val; + + switch (x) { + case ACPI_CDAT_SSLBIS_US_PORT: + dsp_id = y; + break; + case ACPI_CDAT_SSLBIS_ANY_PORT: + switch (y) { + case ACPI_CDAT_SSLBIS_US_PORT: + dsp_id = x; + break; + case ACPI_CDAT_SSLBIS_ANY_PORT: + dsp_id = ACPI_CDAT_SSLBIS_ANY_PORT; + break; + default: + dsp_id = y; + break; + } + break; + default: + dsp_id = x; + break; + } + + le_base = (__force __le64)tbl->sslbis_header.entry_base_unit; + le_val = (__force __le16)tbl->entries[i].latency_or_bandwidth; + val = cdat_normalize(le16_to_cpu(le_val), le64_to_cpu(le_base), + sslbis->data_type); + + xa_for_each(&port->dports, index, dport) { + if (dsp_id == ACPI_CDAT_SSLBIS_ANY_PORT || + dsp_id == dport->port_id) { + cxl_access_coordinate_set(dport->coord, + sslbis->data_type, + val); + } + } + } + + return 0; +} + +void cxl_switch_parse_cdat(struct cxl_port *port) +{ + int rc; + + if (!port->cdat.table) + return; + + rc = cdat_table_parse(ACPI_CDAT_TYPE_SSLBIS, cdat_sslbis_handler, + port, port->cdat.table, port->cdat.length); + rc = cdat_table_parse_output(rc); + if (rc) + dev_dbg(&port->dev, "Failed to parse SSLBIS: %d\n", rc); +} +EXPORT_SYMBOL_NS_GPL(cxl_switch_parse_cdat, CXL); + +static void __cxl_coordinates_combine(struct access_coordinate *out, + struct access_coordinate *c1, + struct access_coordinate *c2) +{ + if (c1->write_bandwidth && c2->write_bandwidth) + out->write_bandwidth = min(c1->write_bandwidth, + c2->write_bandwidth); + out->write_latency = c1->write_latency + c2->write_latency; + + if (c1->read_bandwidth && c2->read_bandwidth) + out->read_bandwidth = min(c1->read_bandwidth, + c2->read_bandwidth); + out->read_latency = c1->read_latency + c2->read_latency; +} + +/** + * cxl_coordinates_combine - Combine the two input coordinates + * + * @out: Output coordinate of c1 and c2 combined + * @c1: input coordinates + * @c2: input coordinates + */ +void cxl_coordinates_combine(struct access_coordinate *out, + struct access_coordinate *c1, + struct access_coordinate *c2) +{ + for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) + __cxl_coordinates_combine(&out[i], &c1[i], &c2[i]); +} + +MODULE_IMPORT_NS(CXL); + +void cxl_region_perf_data_calculate(struct cxl_region *cxlr, + struct cxl_endpoint_decoder *cxled) +{ + struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); + struct cxl_dev_state *cxlds = cxlmd->cxlds; + struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); + struct range dpa = { + .start = cxled->dpa_res->start, + .end = cxled->dpa_res->end, + }; + struct cxl_dpa_perf *perf; + + switch (cxlr->mode) { + case CXL_DECODER_RAM: + perf = &mds->ram_perf; + break; + case CXL_DECODER_PMEM: + perf = &mds->pmem_perf; + break; + default: + return; + } + + lockdep_assert_held(&cxl_dpa_rwsem); + + if (!range_contains(&perf->dpa_range, &dpa)) + return; + + for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) { + /* Get total bandwidth and the worst latency for the cxl region */ + cxlr->coord[i].read_latency = max_t(unsigned int, + cxlr->coord[i].read_latency, + perf->coord[i].read_latency); + cxlr->coord[i].write_latency = max_t(unsigned int, + cxlr->coord[i].write_latency, + perf->coord[i].write_latency); + cxlr->coord[i].read_bandwidth += perf->coord[i].read_bandwidth; + cxlr->coord[i].write_bandwidth += perf->coord[i].write_bandwidth; + } +} + +int cxl_update_hmat_access_coordinates(int nid, struct cxl_region *cxlr, + enum access_coordinate_class access) +{ + return hmat_update_target_coordinates(nid, &cxlr->coord[access], access); +} diff --git a/drivers/cxl/core/core.h b/drivers/cxl/core/core.h index 6444cc827c9ce..01ce740573aab 100644 --- a/drivers/cxl/core/core.h +++ b/drivers/cxl/core/core.h @@ -80,6 +80,7 @@ struct cxl_rcrb_info; resource_size_t __rcrb_to_component(struct device *dev, struct cxl_rcrb_info *ri, enum cxl_rcrb which); +u16 cxl_rcrb_to_aer(struct device *dev, resource_size_t rcrb); extern struct rw_semaphore cxl_dpa_rwsem; extern struct rw_semaphore cxl_region_rwsem; @@ -94,4 +95,9 @@ enum cxl_poison_trace_type { CXL_POISON_TRACE_CLEAR, }; +long cxl_pci_get_latency(struct pci_dev *pdev); + +int cxl_update_hmat_access_coordinates(int nid, struct cxl_region *cxlr, + enum access_coordinate_class access); + #endif /* __CXL_CORE_H__ */ diff --git a/drivers/cxl/core/hdm.c b/drivers/cxl/core/hdm.c index 36b20b844b109..b653f570d0cce 100644 --- a/drivers/cxl/core/hdm.c +++ b/drivers/cxl/core/hdm.c @@ -94,26 +94,6 @@ static void parse_hdm_decoder_caps(struct cxl_hdm *cxlhdm) cxlhdm->iw_cap_mask |= BIT(16); } -static int map_hdm_decoder_regs(struct cxl_port *port, void __iomem *crb, - struct cxl_component_regs *regs) -{ - struct cxl_register_map map = { - .host = &port->dev, - .resource = port->component_reg_phys, - .base = crb, - .max_size = CXL_COMPONENT_REG_BLOCK_SIZE, - }; - - cxl_probe_component_regs(&port->dev, crb, &map.component_map); - if (!map.component_map.hdm_decoder.valid) { - dev_dbg(&port->dev, "HDM decoder registers not implemented\n"); - /* unique error code to indicate no HDM decoder capability */ - return -ENODEV; - } - - return cxl_map_component_regs(&map, regs, BIT(CXL_CM_CAP_CAP_ID_HDM)); -} - static bool should_emulate_decoders(struct cxl_endpoint_dvsec_info *info) { struct cxl_hdm *cxlhdm; @@ -166,9 +146,9 @@ static bool should_emulate_decoders(struct cxl_endpoint_dvsec_info *info) struct cxl_hdm *devm_cxl_setup_hdm(struct cxl_port *port, struct cxl_endpoint_dvsec_info *info) { + struct cxl_register_map *reg_map = &port->reg_map; struct device *dev = &port->dev; struct cxl_hdm *cxlhdm; - void __iomem *crb; int rc; cxlhdm = devm_kzalloc(dev, sizeof(*cxlhdm), GFP_KERNEL); @@ -177,19 +157,29 @@ struct cxl_hdm *devm_cxl_setup_hdm(struct cxl_port *port, cxlhdm->port = port; dev_set_drvdata(dev, cxlhdm); - crb = ioremap(port->component_reg_phys, CXL_COMPONENT_REG_BLOCK_SIZE); - if (!crb && info && info->mem_enabled) { + /* Memory devices can configure device HDM using DVSEC range regs. */ + if (reg_map->resource == CXL_RESOURCE_NONE) { + if (!info || !info->mem_enabled) { + dev_err(dev, "No component registers mapped\n"); + return ERR_PTR(-ENXIO); + } + cxlhdm->decoder_count = info->ranges; return cxlhdm; - } else if (!crb) { - dev_err(dev, "No component registers mapped\n"); - return ERR_PTR(-ENXIO); } - rc = map_hdm_decoder_regs(port, crb, &cxlhdm->regs); - iounmap(crb); - if (rc) + if (!reg_map->component_map.hdm_decoder.valid) { + dev_dbg(&port->dev, "HDM decoder registers not implemented\n"); + /* unique error code to indicate no HDM decoder capability */ + return ERR_PTR(-ENODEV); + } + + rc = cxl_map_component_regs(reg_map, &cxlhdm->regs, + BIT(CXL_CM_CAP_CAP_ID_HDM)); + if (rc) { + dev_err(dev, "Failed to map HDM capability.\n"); return ERR_PTR(rc); + } parse_hdm_decoder_caps(cxlhdm); if (cxlhdm->decoder_count == 0) { diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c index fecaa18f4dd20..65185c9fa0013 100644 --- a/drivers/cxl/core/mbox.c +++ b/drivers/cxl/core/mbox.c @@ -63,6 +63,7 @@ static struct cxl_mem_command cxl_mem_commands[CXL_MEM_COMMAND_ID_MAX] = { CXL_CMD(GET_SHUTDOWN_STATE, 0, 0x1, 0), CXL_CMD(SET_SHUTDOWN_STATE, 0x1, 0, 0), CXL_CMD(GET_SCAN_MEDIA_CAPS, 0x10, 0x4, 0), + CXL_CMD(GET_TIMESTAMP, 0, 0x8, 0), }; /* @@ -836,54 +837,37 @@ int cxl_enumerate_cmds(struct cxl_memdev_state *mds) } EXPORT_SYMBOL_NS_GPL(cxl_enumerate_cmds, CXL); -/* - * General Media Event Record - * CXL rev 3.0 Section 8.2.9.2.1.1; Table 8-43 - */ -static const uuid_t gen_media_event_uuid = - UUID_INIT(0xfbcd0a77, 0xc260, 0x417f, - 0x85, 0xa9, 0x08, 0x8b, 0x16, 0x21, 0xeb, 0xa6); - -/* - * DRAM Event Record - * CXL rev 3.0 section 8.2.9.2.1.2; Table 8-44 - */ -static const uuid_t dram_event_uuid = - UUID_INIT(0x601dcbb3, 0x9c06, 0x4eab, - 0xb8, 0xaf, 0x4e, 0x9b, 0xfb, 0x5c, 0x96, 0x24); - -/* - * Memory Module Event Record - * CXL rev 3.0 section 8.2.9.2.1.3; Table 8-45 - */ -static const uuid_t mem_mod_event_uuid = - UUID_INIT(0xfe927475, 0xdd59, 0x4339, - 0xa5, 0x86, 0x79, 0xba, 0xb1, 0x13, 0xb7, 0x74); - -static void cxl_event_trace_record(const struct cxl_memdev *cxlmd, - enum cxl_event_log_type type, - struct cxl_event_record_raw *record) +void cxl_event_trace_record(const struct cxl_memdev *cxlmd, + enum cxl_event_log_type type, + enum cxl_event_type event_type, + const uuid_t *uuid, union cxl_event *evt) { - uuid_t *id = &record->hdr.id; - - if (uuid_equal(id, &gen_media_event_uuid)) { - struct cxl_event_gen_media *rec = - (struct cxl_event_gen_media *)record; + if (event_type == CXL_CPER_EVENT_GEN_MEDIA) + trace_cxl_general_media(cxlmd, type, &evt->gen_media); + else if (event_type == CXL_CPER_EVENT_DRAM) + trace_cxl_dram(cxlmd, type, &evt->dram); + else if (event_type == CXL_CPER_EVENT_MEM_MODULE) + trace_cxl_memory_module(cxlmd, type, &evt->mem_module); + else + trace_cxl_generic_event(cxlmd, type, uuid, &evt->generic); +} +EXPORT_SYMBOL_NS_GPL(cxl_event_trace_record, CXL); - trace_cxl_general_media(cxlmd, type, rec); - } else if (uuid_equal(id, &dram_event_uuid)) { - struct cxl_event_dram *rec = (struct cxl_event_dram *)record; +static void __cxl_event_trace_record(const struct cxl_memdev *cxlmd, + enum cxl_event_log_type type, + struct cxl_event_record_raw *record) +{ + enum cxl_event_type ev_type = CXL_CPER_EVENT_GENERIC; + const uuid_t *uuid = &record->id; - trace_cxl_dram(cxlmd, type, rec); - } else if (uuid_equal(id, &mem_mod_event_uuid)) { - struct cxl_event_mem_module *rec = - (struct cxl_event_mem_module *)record; + if (uuid_equal(uuid, &CXL_EVENT_GEN_MEDIA_UUID)) + ev_type = CXL_CPER_EVENT_GEN_MEDIA; + else if (uuid_equal(uuid, &CXL_EVENT_DRAM_UUID)) + ev_type = CXL_CPER_EVENT_DRAM; + else if (uuid_equal(uuid, &CXL_EVENT_MEM_MODULE_UUID)) + ev_type = CXL_CPER_EVENT_MEM_MODULE; - trace_cxl_memory_module(cxlmd, type, rec); - } else { - /* For unknown record types print just the header */ - trace_cxl_generic_event(cxlmd, type, record); - } + cxl_event_trace_record(cxlmd, type, ev_type, uuid, &record->event); } static int cxl_clear_event_record(struct cxl_memdev_state *mds, @@ -926,7 +910,10 @@ static int cxl_clear_event_record(struct cxl_memdev_state *mds, */ i = 0; for (cnt = 0; cnt < total; cnt++) { - payload->handles[i++] = get_pl->records[cnt].hdr.handle; + struct cxl_event_record_raw *raw = &get_pl->records[cnt]; + struct cxl_event_generic *gen = &raw->event.generic; + + payload->handles[i++] = gen->hdr.handle; dev_dbg(mds->cxlds.dev, "Event log '%d': Clearing %u\n", log, le16_to_cpu(payload->handles[i - 1])); @@ -989,8 +976,8 @@ static void cxl_mem_get_records_log(struct cxl_memdev_state *mds, break; for (i = 0; i < nr_rec; i++) - cxl_event_trace_record(cxlmd, type, - &payload->records[i]); + __cxl_event_trace_record(cxlmd, type, + &payload->records[i]); if (payload->flags & CXL_GET_EVENT_FLAG_OVERFLOW) trace_cxl_overflow(cxlmd, type, payload); @@ -1247,8 +1234,7 @@ int cxl_mem_create_range_info(struct cxl_memdev_state *mds) return 0; } - cxlds->dpa_res = - (struct resource)DEFINE_RES_MEM(0, mds->total_bytes); + cxlds->dpa_res = DEFINE_RES_MEM(0, mds->total_bytes); if (mds->partition_align_bytes == 0) { rc = add_dpa_res(dev, &cxlds->dpa_res, &cxlds->ram_res, 0, @@ -1399,7 +1385,11 @@ struct cxl_memdev_state *cxl_memdev_state_create(struct device *dev) mutex_init(&mds->mbox_mutex); mutex_init(&mds->event.log_lock); mds->cxlds.dev = dev; + mds->cxlds.reg_map.host = dev; + mds->cxlds.reg_map.resource = CXL_RESOURCE_NONE; mds->cxlds.type = CXL_DEVTYPE_CLASSMEM; + mds->ram_perf.qos_class = CXL_QOS_CLASS_INVALID; + mds->pmem_perf.qos_class = CXL_QOS_CLASS_INVALID; return mds; } diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c index eb895c70043fd..0277726afd049 100644 --- a/drivers/cxl/core/memdev.c +++ b/drivers/cxl/core/memdev.c @@ -114,7 +114,7 @@ static DEVICE_ATTR_RO(serial); static ssize_t numa_node_show(struct device *dev, struct device_attribute *attr, char *buf) { - return sprintf(buf, "%d\n", dev_to_node(dev)); + return sysfs_emit(buf, "%d\n", dev_to_node(dev)); } static DEVICE_ATTR_RO(numa_node); @@ -403,13 +403,41 @@ static struct attribute *cxl_memdev_attributes[] = { NULL, }; +static ssize_t pmem_qos_class_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct cxl_memdev *cxlmd = to_cxl_memdev(dev); + struct cxl_dev_state *cxlds = cxlmd->cxlds; + struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); + + return sysfs_emit(buf, "%d\n", mds->pmem_perf.qos_class); +} + +static struct device_attribute dev_attr_pmem_qos_class = + __ATTR(qos_class, 0444, pmem_qos_class_show, NULL); + static struct attribute *cxl_memdev_pmem_attributes[] = { &dev_attr_pmem_size.attr, + &dev_attr_pmem_qos_class.attr, NULL, }; +static ssize_t ram_qos_class_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct cxl_memdev *cxlmd = to_cxl_memdev(dev); + struct cxl_dev_state *cxlds = cxlmd->cxlds; + struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); + + return sysfs_emit(buf, "%d\n", mds->ram_perf.qos_class); +} + +static struct device_attribute dev_attr_ram_qos_class = + __ATTR(qos_class, 0444, ram_qos_class_show, NULL); + static struct attribute *cxl_memdev_ram_attributes[] = { &dev_attr_ram_size.attr, + &dev_attr_ram_qos_class.attr, NULL, }; @@ -433,14 +461,42 @@ static struct attribute_group cxl_memdev_attribute_group = { .is_visible = cxl_memdev_visible, }; +static umode_t cxl_ram_visible(struct kobject *kobj, struct attribute *a, int n) +{ + struct device *dev = kobj_to_dev(kobj); + struct cxl_memdev *cxlmd = to_cxl_memdev(dev); + struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); + + if (a == &dev_attr_ram_qos_class.attr) + if (mds->ram_perf.qos_class == CXL_QOS_CLASS_INVALID) + return 0; + + return a->mode; +} + static struct attribute_group cxl_memdev_ram_attribute_group = { .name = "ram", .attrs = cxl_memdev_ram_attributes, + .is_visible = cxl_ram_visible, }; +static umode_t cxl_pmem_visible(struct kobject *kobj, struct attribute *a, int n) +{ + struct device *dev = kobj_to_dev(kobj); + struct cxl_memdev *cxlmd = to_cxl_memdev(dev); + struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); + + if (a == &dev_attr_pmem_qos_class.attr) + if (mds->pmem_perf.qos_class == CXL_QOS_CLASS_INVALID) + return 0; + + return a->mode; +} + static struct attribute_group cxl_memdev_pmem_attribute_group = { .name = "pmem", .attrs = cxl_memdev_pmem_attributes, + .is_visible = cxl_pmem_visible, }; static umode_t cxl_memdev_security_visible(struct kobject *kobj, @@ -475,6 +531,13 @@ static const struct attribute_group *cxl_memdev_attribute_groups[] = { NULL, }; +void cxl_memdev_update_perf(struct cxl_memdev *cxlmd) +{ + sysfs_update_group(&cxlmd->dev.kobj, &cxl_memdev_ram_attribute_group); + sysfs_update_group(&cxlmd->dev.kobj, &cxl_memdev_pmem_attribute_group); +} +EXPORT_SYMBOL_NS_GPL(cxl_memdev_update_perf, CXL); + static const struct device_type cxl_memdev_type = { .name = "cxl_memdev", .release = cxl_memdev_release, diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c index 6edfd05466737..b76ad047c2a3d 100644 --- a/drivers/cxl/core/pci.c +++ b/drivers/cxl/core/pci.c @@ -1,10 +1,12 @@ // SPDX-License-Identifier: GPL-2.0-only /* Copyright(c) 2021 Intel Corporation. All rights reserved. */ +#include #include #include #include #include #include +#include #include #include #include @@ -510,14 +512,14 @@ EXPORT_SYMBOL_NS_GPL(cxl_hdm_decode_init, CXL); FIELD_PREP(CXL_DOE_TABLE_ACCESS_ENTRY_HANDLE, (entry_handle))) static int cxl_cdat_get_length(struct device *dev, - struct pci_doe_mb *cdat_doe, + struct pci_doe_mb *doe_mb, size_t *length) { __le32 request = CDAT_DOE_REQ(0); __le32 response[2]; int rc; - rc = pci_doe(cdat_doe, PCI_DVSEC_VENDOR_ID_CXL, + rc = pci_doe(doe_mb, PCI_DVSEC_VENDOR_ID_CXL, CXL_DOE_PROTOCOL_TABLE_ACCESS, &request, sizeof(request), &response, sizeof(response)); @@ -535,60 +537,72 @@ static int cxl_cdat_get_length(struct device *dev, } static int cxl_cdat_read_table(struct device *dev, - struct pci_doe_mb *cdat_doe, - void *cdat_table, size_t *cdat_length) + struct pci_doe_mb *doe_mb, + struct cdat_doe_rsp *rsp, size_t *length) { - size_t length = *cdat_length + sizeof(__le32); - __le32 *data = cdat_table; - int entry_handle = 0; + size_t received, remaining = *length; + unsigned int entry_handle = 0; + union cdat_data *data; __le32 saved_dw = 0; do { __le32 request = CDAT_DOE_REQ(entry_handle); - struct cdat_entry_header *entry; - size_t entry_dw; int rc; - rc = pci_doe(cdat_doe, PCI_DVSEC_VENDOR_ID_CXL, + rc = pci_doe(doe_mb, PCI_DVSEC_VENDOR_ID_CXL, CXL_DOE_PROTOCOL_TABLE_ACCESS, &request, sizeof(request), - data, length); + rsp, sizeof(*rsp) + remaining); if (rc < 0) { dev_err(dev, "DOE failed: %d", rc); return rc; } - /* 1 DW Table Access Response Header + CDAT entry */ - entry = (struct cdat_entry_header *)(data + 1); - if ((entry_handle == 0 && - rc != sizeof(__le32) + sizeof(struct cdat_header)) || - (entry_handle > 0 && - (rc < sizeof(__le32) + sizeof(*entry) || - rc != sizeof(__le32) + le16_to_cpu(entry->length)))) + if (rc < sizeof(*rsp)) return -EIO; + data = (union cdat_data *)rsp->data; + received = rc - sizeof(*rsp); + + if (entry_handle == 0) { + if (received != sizeof(data->header)) + return -EIO; + } else { + if (received < sizeof(data->entry) || + received != le16_to_cpu(data->entry.length)) + return -EIO; + } + /* Get the CXL table access header entry handle */ entry_handle = FIELD_GET(CXL_DOE_TABLE_ACCESS_ENTRY_HANDLE, - le32_to_cpu(data[0])); - entry_dw = rc / sizeof(__le32); - /* Skip Header */ - entry_dw -= 1; + le32_to_cpu(rsp->doe_header)); + /* * Table Access Response Header overwrote the last DW of * previous entry, so restore that DW */ - *data = saved_dw; - length -= entry_dw * sizeof(__le32); - data += entry_dw; - saved_dw = *data; + rsp->doe_header = saved_dw; + remaining -= received; + rsp = (void *)rsp + received; + saved_dw = rsp->doe_header; } while (entry_handle != CXL_DOE_TABLE_ACCESS_LAST_ENTRY); /* Length in CDAT header may exceed concatenation of CDAT entries */ - *cdat_length -= length - sizeof(__le32); + *length -= remaining; return 0; } +static unsigned char cdat_checksum(void *buf, size_t size) +{ + unsigned char sum, *data = buf; + size_t i; + + for (sum = 0, i = 0; i < size; i++) + sum += data[i]; + return sum; +} + /** * read_cdat_data - Read the CDAT data on this port * @port: Port to read data from @@ -597,75 +611,105 @@ static int cxl_cdat_read_table(struct device *dev, */ void read_cdat_data(struct cxl_port *port) { - struct cxl_memdev *cxlmd = to_cxl_memdev(port->uport_dev); - struct device *host = cxlmd->dev.parent; + struct device *uport = port->uport_dev; struct device *dev = &port->dev; - struct pci_doe_mb *cdat_doe; - size_t cdat_length; - void *cdat_table; + struct pci_doe_mb *doe_mb; + struct pci_dev *pdev = NULL; + struct cxl_memdev *cxlmd; + struct cdat_doe_rsp *buf; + size_t table_length, length; int rc; - if (!dev_is_pci(host)) + if (is_cxl_memdev(uport)) { + struct device *host; + + cxlmd = to_cxl_memdev(uport); + host = cxlmd->dev.parent; + if (dev_is_pci(host)) + pdev = to_pci_dev(host); + } else if (dev_is_pci(uport)) { + pdev = to_pci_dev(uport); + } + + if (!pdev) return; - cdat_doe = pci_find_doe_mailbox(to_pci_dev(host), - PCI_DVSEC_VENDOR_ID_CXL, - CXL_DOE_PROTOCOL_TABLE_ACCESS); - if (!cdat_doe) { + + doe_mb = pci_find_doe_mailbox(pdev, PCI_DVSEC_VENDOR_ID_CXL, + CXL_DOE_PROTOCOL_TABLE_ACCESS); + if (!doe_mb) { dev_dbg(dev, "No CDAT mailbox\n"); return; } port->cdat_available = true; - if (cxl_cdat_get_length(dev, cdat_doe, &cdat_length)) { + if (cxl_cdat_get_length(dev, doe_mb, &length)) { dev_dbg(dev, "No CDAT length\n"); return; } - cdat_table = devm_kzalloc(dev, cdat_length + sizeof(__le32), - GFP_KERNEL); - if (!cdat_table) - return; + /* + * The begin of the CDAT buffer needs space for additional 4 + * bytes for the DOE header. Table data starts afterwards. + */ + buf = devm_kzalloc(dev, sizeof(*buf) + length, GFP_KERNEL); + if (!buf) + goto err; - rc = cxl_cdat_read_table(dev, cdat_doe, cdat_table, &cdat_length); - if (rc) { - /* Don't leave table data allocated on error */ - devm_kfree(dev, cdat_table); - dev_err(dev, "CDAT data read error\n"); - return; - } + table_length = length; + + rc = cxl_cdat_read_table(dev, doe_mb, buf, &length); + if (rc) + goto err; + + if (table_length != length) + dev_warn(dev, "Malformed CDAT table length (%zu:%zu), discarding trailing data\n", + table_length, length); + + if (cdat_checksum(buf->data, length)) + goto err; - port->cdat.table = cdat_table + sizeof(__le32); - port->cdat.length = cdat_length; + port->cdat.table = buf->data; + port->cdat.length = length; + + return; +err: + /* Don't leave table data allocated on error */ + devm_kfree(dev, buf); + dev_err(dev, "Failed to read/validate CDAT.\n"); } EXPORT_SYMBOL_NS_GPL(read_cdat_data, CXL); -void cxl_cor_error_detected(struct pci_dev *pdev) +static void __cxl_handle_cor_ras(struct cxl_dev_state *cxlds, + void __iomem *ras_base) { - struct cxl_dev_state *cxlds = pci_get_drvdata(pdev); void __iomem *addr; u32 status; - if (!cxlds->regs.ras) + if (!ras_base) return; - addr = cxlds->regs.ras + CXL_RAS_CORRECTABLE_STATUS_OFFSET; + addr = ras_base + CXL_RAS_CORRECTABLE_STATUS_OFFSET; status = readl(addr); if (status & CXL_RAS_CORRECTABLE_STATUS_MASK) { writel(status & CXL_RAS_CORRECTABLE_STATUS_MASK, addr); trace_cxl_aer_correctable_error(cxlds->cxlmd, status); } } -EXPORT_SYMBOL_NS_GPL(cxl_cor_error_detected, CXL); + +static void cxl_handle_endpoint_cor_ras(struct cxl_dev_state *cxlds) +{ + return __cxl_handle_cor_ras(cxlds, cxlds->regs.ras); +} /* CXL spec rev3.0 8.2.4.16.1 */ -static void header_log_copy(struct cxl_dev_state *cxlds, u32 *log) +static void header_log_copy(void __iomem *ras_base, u32 *log) { void __iomem *addr; u32 *log_addr; int i, log_u32_size = CXL_HEADERLOG_SIZE / sizeof(u32); - addr = cxlds->regs.ras + CXL_RAS_HEADER_LOG_OFFSET; + addr = ras_base + CXL_RAS_HEADER_LOG_OFFSET; log_addr = log; for (i = 0; i < log_u32_size; i++) { @@ -679,17 +723,18 @@ static void header_log_copy(struct cxl_dev_state *cxlds, u32 *log) * Log the state of the RAS status registers and prepare them to log the * next error status. Return 1 if reset needed. */ -static bool cxl_report_and_clear(struct cxl_dev_state *cxlds) +static bool __cxl_handle_ras(struct cxl_dev_state *cxlds, + void __iomem *ras_base) { u32 hl[CXL_HEADERLOG_SIZE_U32]; void __iomem *addr; u32 status; u32 fe; - if (!cxlds->regs.ras) + if (!ras_base) return false; - addr = cxlds->regs.ras + CXL_RAS_UNCORRECTABLE_STATUS_OFFSET; + addr = ras_base + CXL_RAS_UNCORRECTABLE_STATUS_OFFSET; status = readl(addr); if (!(status & CXL_RAS_UNCORRECTABLE_STATUS_MASK)) return false; @@ -697,7 +742,7 @@ static bool cxl_report_and_clear(struct cxl_dev_state *cxlds) /* If multiple errors, log header points to first error from ctrl reg */ if (hweight32(status) > 1) { void __iomem *rcc_addr = - cxlds->regs.ras + CXL_RAS_CAP_CONTROL_OFFSET; + ras_base + CXL_RAS_CAP_CONTROL_OFFSET; fe = BIT(FIELD_GET(CXL_RAS_CAP_CONTROL_FE_MASK, readl(rcc_addr))); @@ -705,13 +750,213 @@ static bool cxl_report_and_clear(struct cxl_dev_state *cxlds) fe = status; } - header_log_copy(cxlds, hl); + header_log_copy(ras_base, hl); trace_cxl_aer_uncorrectable_error(cxlds->cxlmd, status, fe, hl); writel(status & CXL_RAS_UNCORRECTABLE_STATUS_MASK, addr); return true; } +static bool cxl_handle_endpoint_ras(struct cxl_dev_state *cxlds) +{ + return __cxl_handle_ras(cxlds, cxlds->regs.ras); +} + +#ifdef CONFIG_PCIEAER_CXL + +static void cxl_dport_map_rch_aer(struct cxl_dport *dport) +{ + struct cxl_rcrb_info *ri = &dport->rcrb; + void __iomem *dport_aer = NULL; + resource_size_t aer_phys; + struct device *host; + + if (dport->rch && ri->aer_cap) { + host = dport->reg_map.host; + aer_phys = ri->aer_cap + ri->base; + dport_aer = devm_cxl_iomap_block(host, aer_phys, + sizeof(struct aer_capability_regs)); + } + + dport->regs.dport_aer = dport_aer; +} + +static void cxl_dport_map_regs(struct cxl_dport *dport) +{ + struct cxl_register_map *map = &dport->reg_map; + struct device *dev = dport->dport_dev; + + if (!map->component_map.ras.valid) + dev_dbg(dev, "RAS registers not found\n"); + else if (cxl_map_component_regs(map, &dport->regs.component, + BIT(CXL_CM_CAP_CAP_ID_RAS))) + dev_dbg(dev, "Failed to map RAS capability.\n"); + + if (dport->rch) + cxl_dport_map_rch_aer(dport); +} + +static void cxl_disable_rch_root_ints(struct cxl_dport *dport) +{ + void __iomem *aer_base = dport->regs.dport_aer; + struct pci_host_bridge *bridge; + u32 aer_cmd_mask, aer_cmd; + + if (!aer_base) + return; + + bridge = to_pci_host_bridge(dport->dport_dev); + + /* + * Disable RCH root port command interrupts. + * CXL 3.0 12.2.1.1 - RCH Downstream Port-detected Errors + * + * This sequence may not be necessary. CXL spec states disabling + * the root cmd register's interrupts is required. But, PCI spec + * shows these are disabled by default on reset. + */ + if (bridge->native_aer) { + aer_cmd_mask = (PCI_ERR_ROOT_CMD_COR_EN | + PCI_ERR_ROOT_CMD_NONFATAL_EN | + PCI_ERR_ROOT_CMD_FATAL_EN); + aer_cmd = readl(aer_base + PCI_ERR_ROOT_COMMAND); + aer_cmd &= ~aer_cmd_mask; + writel(aer_cmd, aer_base + PCI_ERR_ROOT_COMMAND); + } +} + +void cxl_setup_parent_dport(struct device *host, struct cxl_dport *dport) +{ + struct device *dport_dev = dport->dport_dev; + + if (dport->rch) { + struct pci_host_bridge *host_bridge = to_pci_host_bridge(dport_dev); + + if (host_bridge->native_aer) + dport->rcrb.aer_cap = cxl_rcrb_to_aer(dport_dev, dport->rcrb.base); + } + + dport->reg_map.host = host; + cxl_dport_map_regs(dport); + + if (dport->rch) + cxl_disable_rch_root_ints(dport); +} +EXPORT_SYMBOL_NS_GPL(cxl_setup_parent_dport, CXL); + +static void cxl_handle_rdport_cor_ras(struct cxl_dev_state *cxlds, + struct cxl_dport *dport) +{ + return __cxl_handle_cor_ras(cxlds, dport->regs.ras); +} + +static bool cxl_handle_rdport_ras(struct cxl_dev_state *cxlds, + struct cxl_dport *dport) +{ + return __cxl_handle_ras(cxlds, dport->regs.ras); +} + +/* + * Copy the AER capability registers using 32 bit read accesses. + * This is necessary because RCRB AER capability is MMIO mapped. Clear the + * status after copying. + * + * @aer_base: base address of AER capability block in RCRB + * @aer_regs: destination for copying AER capability + */ +static bool cxl_rch_get_aer_info(void __iomem *aer_base, + struct aer_capability_regs *aer_regs) +{ + int read_cnt = sizeof(struct aer_capability_regs) / sizeof(u32); + u32 *aer_regs_buf = (u32 *)aer_regs; + int n; + + if (!aer_base) + return false; + + /* Use readl() to guarantee 32-bit accesses */ + for (n = 0; n < read_cnt; n++) + aer_regs_buf[n] = readl(aer_base + n * sizeof(u32)); + + writel(aer_regs->uncor_status, aer_base + PCI_ERR_UNCOR_STATUS); + writel(aer_regs->cor_status, aer_base + PCI_ERR_COR_STATUS); + + return true; +} + +/* Get AER severity. Return false if there is no error. */ +static bool cxl_rch_get_aer_severity(struct aer_capability_regs *aer_regs, + int *severity) +{ + if (aer_regs->uncor_status & ~aer_regs->uncor_mask) { + if (aer_regs->uncor_status & PCI_ERR_ROOT_FATAL_RCV) + *severity = AER_FATAL; + else + *severity = AER_NONFATAL; + return true; + } + + if (aer_regs->cor_status & ~aer_regs->cor_mask) { + *severity = AER_CORRECTABLE; + return true; + } + + return false; +} + +static void cxl_handle_rdport_errors(struct cxl_dev_state *cxlds) +{ + struct pci_dev *pdev = to_pci_dev(cxlds->dev); + struct aer_capability_regs aer_regs; + struct cxl_dport *dport; + struct cxl_port *port; + int severity; + + port = cxl_pci_find_port(pdev, &dport); + if (!port) + return; + + put_device(&port->dev); + + if (!cxl_rch_get_aer_info(dport->regs.dport_aer, &aer_regs)) + return; + + if (!cxl_rch_get_aer_severity(&aer_regs, &severity)) + return; + + pci_print_aer(pdev, severity, &aer_regs); + + if (severity == AER_CORRECTABLE) + cxl_handle_rdport_cor_ras(cxlds, dport); + else + cxl_handle_rdport_ras(cxlds, dport); +} + +#else +static void cxl_handle_rdport_errors(struct cxl_dev_state *cxlds) { } +#endif + +void cxl_cor_error_detected(struct pci_dev *pdev) +{ + struct cxl_dev_state *cxlds = pci_get_drvdata(pdev); + struct device *dev = &cxlds->cxlmd->dev; + + scoped_guard(device, dev) { + if (!dev->driver) { + dev_warn(&pdev->dev, + "%s: memdev disabled, abort error handling\n", + dev_name(dev)); + return; + } + + if (cxlds->rcd) + cxl_handle_rdport_errors(cxlds); + + cxl_handle_endpoint_cor_ras(cxlds); + } +} +EXPORT_SYMBOL_NS_GPL(cxl_cor_error_detected, CXL); + pci_ers_result_t cxl_error_detected(struct pci_dev *pdev, pci_channel_state_t state) { @@ -720,13 +965,25 @@ pci_ers_result_t cxl_error_detected(struct pci_dev *pdev, struct device *dev = &cxlmd->dev; bool ue; - /* - * A frozen channel indicates an impending reset which is fatal to - * CXL.mem operation, and will likely crash the system. On the off - * chance the situation is recoverable dump the status of the RAS - * capability registers and bounce the active state of the memdev. - */ - ue = cxl_report_and_clear(cxlds); + scoped_guard(device, dev) { + if (!dev->driver) { + dev_warn(&pdev->dev, + "%s: memdev disabled, abort error handling\n", + dev_name(dev)); + return PCI_ERS_RESULT_DISCONNECT; + } + + if (cxlds->rcd) + cxl_handle_rdport_errors(cxlds); + /* + * A frozen channel indicates an impending reset which is fatal to + * CXL.mem operation, and will likely crash the system. On the off + * chance the situation is recoverable dump the status of the RAS + * capability registers and bounce the active state of the memdev. + */ + ue = cxl_handle_endpoint_ras(cxlds); + } + switch (state) { case pci_channel_io_normal: @@ -749,3 +1006,38 @@ pci_ers_result_t cxl_error_detected(struct pci_dev *pdev, return PCI_ERS_RESULT_NEED_RESET; } EXPORT_SYMBOL_NS_GPL(cxl_error_detected, CXL); + +static int cxl_flit_size(struct pci_dev *pdev) +{ + if (cxl_pci_flit_256(pdev)) + return 256; + + return 68; +} + +/** + * cxl_pci_get_latency - calculate the link latency for the PCIe link + * @pdev: PCI device + * + * return: calculated latency or 0 for no latency + * + * CXL Memory Device SW Guide v1.0 2.11.4 Link latency calculation + * Link latency = LinkPropagationLatency + FlitLatency + RetimerLatency + * LinkProgationLatency is negligible, so 0 will be used + * RetimerLatency is assumed to be negligible and 0 will be used + * FlitLatency = FlitSize / LinkBandwidth + * FlitSize is defined by spec. CXL rev3.0 4.2.1. + * 68B flit is used up to 32GT/s. >32GT/s, 256B flit size is used. + * The FlitLatency is converted to picoseconds. + */ +long cxl_pci_get_latency(struct pci_dev *pdev) +{ + long bw; + + bw = pcie_link_speed_mbps(pdev); + if (bw < 0) + return 0; + bw /= BITS_PER_BYTE; + + return cxl_flit_size(pdev) * MEGA / bw; +} diff --git a/drivers/cxl/core/pmem.c b/drivers/cxl/core/pmem.c index fc94f52403271..e69625a8d6a1d 100644 --- a/drivers/cxl/core/pmem.c +++ b/drivers/cxl/core/pmem.c @@ -64,14 +64,14 @@ static int match_nvdimm_bridge(struct device *dev, void *data) struct cxl_nvdimm_bridge *cxl_find_nvdimm_bridge(struct cxl_memdev *cxlmd) { - struct cxl_port *port = find_cxl_root(cxlmd->endpoint); + struct cxl_root *cxl_root __free(put_cxl_root) = + find_cxl_root(cxlmd->endpoint); struct device *dev; - if (!port) + if (!cxl_root) return NULL; - dev = device_find_child(&port->dev, NULL, match_nvdimm_bridge); - put_device(&port->dev); + dev = device_find_child(&cxl_root->port.dev, NULL, match_nvdimm_bridge); if (!dev) return NULL; diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index 7f28d1021fa99..a90dff590154f 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -3,12 +3,14 @@ #include #include #include +#include #include #include #include #include #include #include +#include #include #include #include @@ -287,6 +289,15 @@ static ssize_t interleave_ways_show(struct device *dev, static DEVICE_ATTR_RO(interleave_ways); +static ssize_t qos_class_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev); + + return sysfs_emit(buf, "%d\n", cxlrd->qos_class); +} +static DEVICE_ATTR_RO(qos_class); + static struct attribute *cxl_decoder_base_attrs[] = { &dev_attr_start.attr, &dev_attr_size.attr, @@ -306,6 +317,7 @@ static struct attribute *cxl_decoder_root_attrs[] = { &dev_attr_cap_type2.attr, &dev_attr_cap_type3.attr, &dev_attr_target_list.attr, + &dev_attr_qos_class.attr, SET_CXL_REGION_ATTR(create_pmem_region) SET_CXL_REGION_ATTR(create_ram_region) SET_CXL_REGION_ATTR(delete_region) @@ -527,11 +539,39 @@ static void cxl_port_release(struct device *dev) xa_destroy(&port->dports); xa_destroy(&port->regions); ida_free(&cxl_port_ida, port->id); - kfree(port); + if (is_cxl_root(port)) + kfree(to_cxl_root(port)); + else + kfree(port); } +static ssize_t decoders_committed_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct cxl_port *port = to_cxl_port(dev); + int rc; + + down_read(&cxl_region_rwsem); + rc = sysfs_emit(buf, "%d\n", cxl_num_decoders_committed(port)); + up_read(&cxl_region_rwsem); + + return rc; +} + +static DEVICE_ATTR_RO(decoders_committed); + +static struct attribute *cxl_port_attrs[] = { + &dev_attr_decoders_committed.attr, + NULL, +}; + +static struct attribute_group cxl_port_attribute_group = { + .attrs = cxl_port_attrs, +}; + static const struct attribute_group *cxl_port_attribute_groups[] = { &cxl_base_attribute_group, + &cxl_port_attribute_group, NULL, }; @@ -628,20 +668,33 @@ static int devm_cxl_link_parent_dport(struct device *host, static struct lock_class_key cxl_port_key; static struct cxl_port *cxl_port_alloc(struct device *uport_dev, - resource_size_t component_reg_phys, struct cxl_dport *parent_dport) { - struct cxl_port *port; + struct cxl_root *cxl_root __free(kfree) = NULL; + struct cxl_port *port, *_port __free(kfree) = NULL; struct device *dev; int rc; - port = kzalloc(sizeof(*port), GFP_KERNEL); - if (!port) - return ERR_PTR(-ENOMEM); + /* No parent_dport, root cxl_port */ + if (!parent_dport) { + cxl_root = kzalloc(sizeof(*cxl_root), GFP_KERNEL); + if (!cxl_root) + return ERR_PTR(-ENOMEM); + } else { + _port = kzalloc(sizeof(*port), GFP_KERNEL); + if (!_port) + return ERR_PTR(-ENOMEM); + } rc = ida_alloc(&cxl_port_ida, GFP_KERNEL); if (rc < 0) - goto err; + return ERR_PTR(rc); + + if (cxl_root) + port = &no_free_ptr(cxl_root)->port; + else + port = no_free_ptr(_port); + port->id = rc; port->uport_dev = uport_dev; @@ -679,7 +732,6 @@ static struct cxl_port *cxl_port_alloc(struct device *uport_dev, } else dev->parent = uport_dev; - port->component_reg_phys = component_reg_phys; ida_init(&port->decoder_ida); port->hdm_end = -1; port->commit_end = -1; @@ -694,25 +746,23 @@ static struct cxl_port *cxl_port_alloc(struct device *uport_dev, dev->type = &cxl_port_type; return port; - -err: - kfree(port); - return ERR_PTR(rc); } static int cxl_setup_comp_regs(struct device *host, struct cxl_register_map *map, resource_size_t component_reg_phys) { - if (component_reg_phys == CXL_RESOURCE_NONE) - return 0; - *map = (struct cxl_register_map) { .host = host, - .reg_type = CXL_REGLOC_RBI_COMPONENT, + .reg_type = CXL_REGLOC_RBI_EMPTY, .resource = component_reg_phys, - .max_size = CXL_COMPONENT_REG_BLOCK_SIZE, }; + if (component_reg_phys == CXL_RESOURCE_NONE) + return 0; + + map->reg_type = CXL_REGLOC_RBI_COMPONENT; + map->max_size = CXL_COMPONENT_REG_BLOCK_SIZE; + return cxl_setup_regs(map); } @@ -721,7 +771,7 @@ static int cxl_port_setup_regs(struct cxl_port *port, { if (dev_is_platform(port->uport_dev)) return 0; - return cxl_setup_comp_regs(&port->dev, &port->comp_map, + return cxl_setup_comp_regs(&port->dev, &port->reg_map, component_reg_phys); } @@ -738,12 +788,50 @@ static int cxl_dport_setup_regs(struct device *host, struct cxl_dport *dport, * register probing, and fixup @host after the fact, since @host may be * NULL. */ - rc = cxl_setup_comp_regs(dport->dport_dev, &dport->comp_map, + rc = cxl_setup_comp_regs(dport->dport_dev, &dport->reg_map, component_reg_phys); - dport->comp_map.host = host; + dport->reg_map.host = host; return rc; } +DEFINE_SHOW_ATTRIBUTE(einj_cxl_available_error_type); + +static int cxl_einj_inject(void *data, u64 type) +{ + struct cxl_dport *dport = data; + + if (dport->rch) + return einj_cxl_inject_rch_error(dport->rcrb.base, type); + + return einj_cxl_inject_error(to_pci_dev(dport->dport_dev), type); +} +DEFINE_DEBUGFS_ATTRIBUTE(cxl_einj_inject_fops, NULL, cxl_einj_inject, + "0x%llx\n"); + +static void cxl_debugfs_create_dport_dir(struct cxl_dport *dport) +{ + struct cxl_port *port = dport->port; + struct cxl_port *parent = (port && port->parent_dport) ? + port->parent_dport->port : NULL; + struct dentry *dir; + + if (!einj_cxl_is_initialized()) + return; + + /* + * Protocol error injection is only available for CXL 2.0+ root ports + * and CXL 1.1 downstream ports + */ + if (!dport->rch && + !(dev_is_pci(dport->dport_dev) && parent && is_cxl_root(parent))) + return; + + dir = cxl_debugfs_create_dir(dev_name(dport->dport_dev)); + + debugfs_create_file("einj_inject", 0200, dir, dport, + &cxl_einj_inject_fops); +} + static struct cxl_port *__devm_cxl_add_port(struct device *host, struct device *uport_dev, resource_size_t component_reg_phys, @@ -753,21 +841,37 @@ static struct cxl_port *__devm_cxl_add_port(struct device *host, struct device *dev; int rc; - port = cxl_port_alloc(uport_dev, component_reg_phys, parent_dport); + port = cxl_port_alloc(uport_dev, parent_dport); if (IS_ERR(port)) return port; dev = &port->dev; - if (is_cxl_memdev(uport_dev)) + if (is_cxl_memdev(uport_dev)) { + struct cxl_memdev *cxlmd = to_cxl_memdev(uport_dev); + struct cxl_dev_state *cxlds = cxlmd->cxlds; + rc = dev_set_name(dev, "endpoint%d", port->id); - else if (parent_dport) + if (rc) + goto err; + + /* + * The endpoint driver already enumerated the component and RAS + * registers. Reuse that enumeration while prepping them to be + * mapped by the cxl_port driver. + */ + port->reg_map = cxlds->reg_map; + port->reg_map.host = &port->dev; + cxlmd->endpoint = port; + } else if (parent_dport) { rc = dev_set_name(dev, "port%d", port->id); - else - rc = dev_set_name(dev, "root%d", port->id); - if (rc) - goto err; + if (rc) + goto err; - rc = cxl_port_setup_regs(port, component_reg_phys); + rc = cxl_port_setup_regs(port, component_reg_phys); + if (rc) + goto err; + } else + rc = dev_set_name(dev, "root%d", port->id); if (rc) goto err; @@ -787,6 +891,9 @@ static struct cxl_port *__devm_cxl_add_port(struct device *host, if (rc) return ERR_PTR(rc); + if (parent_dport && dev_is_pci(uport_dev)) + port->pci_latency = cxl_pci_get_latency(to_pci_dev(uport_dev)); + return port; err: @@ -830,6 +937,22 @@ struct cxl_port *devm_cxl_add_port(struct device *host, } EXPORT_SYMBOL_NS_GPL(devm_cxl_add_port, CXL); +struct cxl_root *devm_cxl_add_root(struct device *host, + const struct cxl_root_ops *ops) +{ + struct cxl_root *cxl_root; + struct cxl_port *port; + + port = devm_cxl_add_port(host, host, CXL_RESOURCE_NONE, NULL); + if (IS_ERR(port)) + return (struct cxl_root *)port; + + cxl_root = to_cxl_root(port); + cxl_root->ops = ops; + return cxl_root; +} +EXPORT_SYMBOL_NS_GPL(devm_cxl_add_root, CXL); + struct pci_bus *cxl_port_to_pci_bus(struct cxl_port *port) { /* There is no pci_bus associated with a CXL platform-root port */ @@ -885,7 +1008,7 @@ static bool dev_is_cxl_root_child(struct device *dev) return false; } -struct cxl_port *find_cxl_root(struct cxl_port *port) +struct cxl_root *find_cxl_root(struct cxl_port *port) { struct cxl_port *iter = port; @@ -895,10 +1018,19 @@ struct cxl_port *find_cxl_root(struct cxl_port *port) if (!iter) return NULL; get_device(&iter->dev); - return iter; + return to_cxl_root(iter); } EXPORT_SYMBOL_NS_GPL(find_cxl_root, CXL); +void put_cxl_root(struct cxl_root *cxl_root) +{ + if (!cxl_root) + return; + + put_device(&cxl_root->port.dev); +} +EXPORT_SYMBOL_NS_GPL(put_cxl_root, CXL); + static struct cxl_dport *find_dport(struct cxl_port *port, int id) { struct cxl_dport *dport; @@ -1054,6 +1186,11 @@ __devm_cxl_add_dport(struct cxl_port *port, struct device *dport_dev, if (rc) return ERR_PTR(rc); + if (dev_is_pci(dport_dev)) + dport->link_latency = cxl_pci_get_latency(to_pci_dev(dport_dev)); + + cxl_debugfs_create_dport_dir(dport); + return dport; } @@ -1279,7 +1416,6 @@ int cxl_endpoint_autoremove(struct cxl_memdev *cxlmd, struct cxl_port *endpoint) get_device(host); get_device(&endpoint->dev); - cxlmd->endpoint = endpoint; cxlmd->depth = endpoint->depth; return devm_add_action_or_reset(dev, delete_endpoint, cxlmd); } @@ -1500,7 +1636,11 @@ int devm_cxl_enumerate_ports(struct cxl_memdev *cxlmd) struct cxl_dport *dport; struct cxl_port *port; - if (!dport_dev) + /* + * The terminal "grandparent" in PCI is NULL and @platform_bus + * for platform devices + */ + if (!dport_dev || dport_dev == &platform_bus) return 0; uport_dev = dport_dev->parent; @@ -1719,6 +1859,7 @@ struct cxl_root_decoder *cxl_root_decoder_alloc(struct cxl_port *port, } atomic_set(&cxlrd->region_id, rc); + cxlrd->qos_class = CXL_QOS_CLASS_INVALID; return cxlrd; } EXPORT_SYMBOL_NS_GPL(cxl_root_decoder_alloc, CXL); @@ -2003,6 +2144,132 @@ bool schedule_cxl_memdev_detach(struct cxl_memdev *cxlmd) } EXPORT_SYMBOL_NS_GPL(schedule_cxl_memdev_detach, CXL); +static void add_latency(struct access_coordinate *c, long latency) +{ + for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) { + c[i].write_latency += latency; + c[i].read_latency += latency; + } +} + +static bool coordinates_valid(struct access_coordinate *c) +{ + for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) { + if (c[i].read_bandwidth && c[i].write_bandwidth && + c[i].read_latency && c[i].write_latency) + continue; + return false; + } + + return true; +} + +static void set_min_bandwidth(struct access_coordinate *c, unsigned int bw) +{ + for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) { + c[i].write_bandwidth = min(c[i].write_bandwidth, bw); + c[i].read_bandwidth = min(c[i].read_bandwidth, bw); + } +} + +static void set_access_coordinates(struct access_coordinate *out, + struct access_coordinate *in) +{ + for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) + out[i] = in[i]; +} + +static bool parent_port_is_cxl_root(struct cxl_port *port) +{ + return is_cxl_root(to_cxl_port(port->dev.parent)); +} + +/** + * cxl_endpoint_get_perf_coordinates - Retrieve performance numbers stored in dports + * of CXL path + * @port: endpoint cxl_port + * @coord: output performance data + * + * Return: errno on failure, 0 on success. + */ +int cxl_endpoint_get_perf_coordinates(struct cxl_port *port, + struct access_coordinate *coord) +{ + struct cxl_memdev *cxlmd = to_cxl_memdev(port->uport_dev); + struct access_coordinate c[] = { + { + .read_bandwidth = UINT_MAX, + .write_bandwidth = UINT_MAX, + }, + { + .read_bandwidth = UINT_MAX, + .write_bandwidth = UINT_MAX, + }, + }; + struct cxl_port *iter = port; + struct cxl_dport *dport; + struct pci_dev *pdev; + struct device *dev; + unsigned int bw; + bool is_cxl_root; + + if (!is_cxl_endpoint(port)) + return -EINVAL; + + /* + * Skip calculation for RCD. Expectation is HMAT already covers RCD case + * since RCH does not support hotplug. + */ + if (cxlmd->cxlds->rcd) + return 0; + + /* + * Exit the loop when the parent port of the current iter port is cxl + * root. The iterative loop starts at the endpoint and gathers the + * latency of the CXL link from the current device/port to the connected + * downstream port each iteration. + */ + do { + dport = iter->parent_dport; + iter = to_cxl_port(iter->dev.parent); + is_cxl_root = parent_port_is_cxl_root(iter); + + /* + * There's no valid access_coordinate for a root port since RPs do not + * have CDAT and therefore needs to be skipped. + */ + if (!is_cxl_root) { + if (!coordinates_valid(dport->coord)) + return -EINVAL; + cxl_coordinates_combine(c, c, dport->coord); + } + add_latency(c, dport->link_latency); + } while (!is_cxl_root); + + dport = iter->parent_dport; + /* Retrieve HB coords */ + if (!coordinates_valid(dport->coord)) + return -EINVAL; + cxl_coordinates_combine(c, c, dport->coord); + + dev = port->uport_dev->parent; + if (!dev_is_pci(dev)) + return -ENODEV; + + /* Get the calculated PCI paths bandwidth */ + pdev = to_pci_dev(dev); + bw = pcie_bandwidth_available(pdev, NULL, NULL, NULL); + if (bw == 0) + return -ENXIO; + bw /= BITS_PER_BYTE; + + set_min_bandwidth(c, bw); + set_access_coordinates(coord, c); + + return 0; +} +EXPORT_SYMBOL_NS_GPL(cxl_endpoint_get_perf_coordinates, CXL); + /* for user tooling to ensure port disable work has completed */ static ssize_t flush_store(const struct bus_type *bus, const char *buf, size_t count) { @@ -2054,6 +2321,10 @@ static __init int cxl_core_init(void) cxl_debugfs = debugfs_create_dir("cxl", NULL); + if (einj_cxl_is_initialized()) + debugfs_create_file("einj_types", 0400, cxl_debugfs, NULL, + &einj_cxl_available_error_type_fops); + cxl_mbox_init(); rc = cxl_memdev_init(); @@ -2097,3 +2368,4 @@ static void cxl_core_exit(void) subsys_initcall(cxl_core_init); module_exit(cxl_core_exit); MODULE_LICENSE("GPL v2"); +MODULE_IMPORT_NS(CXL); diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c index 1728cae1e8409..e821136a3900d 100644 --- a/drivers/cxl/core/region.c +++ b/drivers/cxl/core/region.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -28,8 +29,116 @@ * 3. Decoder targets */ +/* + * nodemask that sets per node when the access_coordinates for the node has + * been updated by the CXL memory hotplug notifier. + */ +static nodemask_t nodemask_region_seen = NODE_MASK_NONE; + static struct cxl_region *to_cxl_region(struct device *dev); +#define __ACCESS_ATTR_RO(_level, _name) { \ + .attr = { .name = __stringify(_name), .mode = 0444 }, \ + .show = _name##_access##_level##_show, \ +} + +#define ACCESS_DEVICE_ATTR_RO(level, name) \ + struct device_attribute dev_attr_access##level##_##name = __ACCESS_ATTR_RO(level, name) + +#define ACCESS_ATTR_RO(level, attrib) \ +static ssize_t attrib##_access##level##_show(struct device *dev, \ + struct device_attribute *attr, \ + char *buf) \ +{ \ + struct cxl_region *cxlr = to_cxl_region(dev); \ + \ + if (cxlr->coord[level].attrib == 0) \ + return -ENOENT; \ + \ + return sysfs_emit(buf, "%u\n", cxlr->coord[level].attrib); \ +} \ +static ACCESS_DEVICE_ATTR_RO(level, attrib) + +ACCESS_ATTR_RO(0, read_bandwidth); +ACCESS_ATTR_RO(0, read_latency); +ACCESS_ATTR_RO(0, write_bandwidth); +ACCESS_ATTR_RO(0, write_latency); + +#define ACCESS_ATTR_DECLARE(level, attrib) \ + (&dev_attr_access##level##_##attrib.attr) + +static struct attribute *access0_coordinate_attrs[] = { + ACCESS_ATTR_DECLARE(0, read_bandwidth), + ACCESS_ATTR_DECLARE(0, write_bandwidth), + ACCESS_ATTR_DECLARE(0, read_latency), + ACCESS_ATTR_DECLARE(0, write_latency), + NULL +}; + +ACCESS_ATTR_RO(1, read_bandwidth); +ACCESS_ATTR_RO(1, read_latency); +ACCESS_ATTR_RO(1, write_bandwidth); +ACCESS_ATTR_RO(1, write_latency); + +static struct attribute *access1_coordinate_attrs[] = { + ACCESS_ATTR_DECLARE(1, read_bandwidth), + ACCESS_ATTR_DECLARE(1, write_bandwidth), + ACCESS_ATTR_DECLARE(1, read_latency), + ACCESS_ATTR_DECLARE(1, write_latency), + NULL +}; + +#define ACCESS_VISIBLE(level) \ +static umode_t cxl_region_access##level##_coordinate_visible( \ + struct kobject *kobj, struct attribute *a, int n) \ +{ \ + struct device *dev = kobj_to_dev(kobj); \ + struct cxl_region *cxlr = to_cxl_region(dev); \ + \ + if (a == &dev_attr_access##level##_read_latency.attr && \ + cxlr->coord[level].read_latency == 0) \ + return 0; \ + \ + if (a == &dev_attr_access##level##_write_latency.attr && \ + cxlr->coord[level].write_latency == 0) \ + return 0; \ + \ + if (a == &dev_attr_access##level##_read_bandwidth.attr && \ + cxlr->coord[level].read_bandwidth == 0) \ + return 0; \ + \ + if (a == &dev_attr_access##level##_write_bandwidth.attr && \ + cxlr->coord[level].write_bandwidth == 0) \ + return 0; \ + \ + return a->mode; \ +} + +ACCESS_VISIBLE(0); +ACCESS_VISIBLE(1); + +static const struct attribute_group cxl_region_access0_coordinate_group = { + .name = "access0", + .attrs = access0_coordinate_attrs, + .is_visible = cxl_region_access0_coordinate_visible, +}; + +static const struct attribute_group *get_cxl_region_access0_group(void) +{ + return &cxl_region_access0_coordinate_group; +} + +static const struct attribute_group cxl_region_access1_coordinate_group = { + .name = "access1", + .attrs = access1_coordinate_attrs, + .is_visible = cxl_region_access1_coordinate_visible, +}; + +static const struct attribute_group *get_cxl_region_access1_group(void) +{ + return &cxl_region_access1_coordinate_group; +} + static ssize_t uuid_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -123,7 +232,7 @@ static int cxl_region_invalidate_memregion(struct cxl_region *cxlr) { if (!cpu_cache_has_invalidate_memregion()) { if (IS_ENABLED(CONFIG_CXL_REGION_INVALIDATION_TEST)) { - dev_warn_once( + dev_info_once( &cxlr->dev, "Bypassing cpu_cache_invalidate_memregion() for testing!\n"); return 0; @@ -536,8 +645,9 @@ static int alloc_hpa(struct cxl_region *cxlr, resource_size_t size) res = alloc_free_mem_region(cxlrd->res, size, SZ_256M, dev_name(&cxlr->dev)); if (IS_ERR(res)) { - dev_dbg(&cxlr->dev, "failed to allocate HPA: %ld\n", - PTR_ERR(res)); + dev_dbg(&cxlr->dev, + "HPA allocation error (%ld) for size:%pap in %s %pr\n", + PTR_ERR(res), &size, cxlrd->res->name, cxlrd->res); return PTR_ERR(res); } @@ -1838,6 +1948,8 @@ static int cxl_region_attach(struct cxl_region *cxlr, return -EINVAL; } + cxl_region_perf_data_calculate(cxlr, cxled); + if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) { int i; @@ -2147,6 +2259,8 @@ static const struct attribute_group *region_groups[] = { &cxl_base_attribute_group, &cxl_region_group, &cxl_region_target_group, + &cxl_region_access0_coordinate_group, + &cxl_region_access1_coordinate_group, NULL, }; @@ -2194,13 +2308,14 @@ static struct cxl_region *to_cxl_region(struct device *dev) return container_of(dev, struct cxl_region, dev); } -static void unregister_region(void *dev) +static void unregister_region(void *_cxlr) { - struct cxl_region *cxlr = to_cxl_region(dev); + struct cxl_region *cxlr = _cxlr; struct cxl_region_params *p = &cxlr->params; int i; - device_del(dev); + unregister_memory_notifier(&cxlr->memory_notifier); + device_del(&cxlr->dev); /* * Now that region sysfs is shutdown, the parameter block is now @@ -2211,7 +2326,7 @@ static void unregister_region(void *dev) detach_target(cxlr, i); cxl_region_iomem_release(cxlr); - put_device(dev); + put_device(&cxlr->dev); } static struct lock_class_key cxl_region_key; @@ -2244,6 +2359,72 @@ static struct cxl_region *cxl_region_alloc(struct cxl_root_decoder *cxlrd, int i return cxlr; } +static bool cxl_region_update_coordinates(struct cxl_region *cxlr, int nid) +{ + int cset = 0; + int rc; + + for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) { + if (cxlr->coord[i].read_bandwidth) { + node_update_perf_attrs(nid, &cxlr->coord[i], i); + cset++; + } + } + + if (!cset) + return false; + + rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_access0_group()); + if (rc) + dev_dbg(&cxlr->dev, "Failed to update access0 group\n"); + + rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_access1_group()); + if (rc) + dev_dbg(&cxlr->dev, "Failed to update access1 group\n"); + + return true; +} + +static int cxl_region_nid(struct cxl_region *cxlr) +{ + struct cxl_region_params *p = &cxlr->params; + struct cxl_endpoint_decoder *cxled; + struct cxl_decoder *cxld; + + guard(rwsem_read)(&cxl_region_rwsem); + cxled = p->targets[0]; + if (!cxled) + return NUMA_NO_NODE; + cxld = &cxled->cxld; + return phys_to_target_node(cxld->hpa_range.start); +} + +static int cxl_region_perf_attrs_callback(struct notifier_block *nb, + unsigned long action, void *arg) +{ + struct cxl_region *cxlr = container_of(nb, struct cxl_region, + memory_notifier); + struct memory_notify *mnb = arg; + int nid = mnb->status_change_nid; + int region_nid; + + if (nid == NUMA_NO_NODE || action != MEM_ONLINE) + return NOTIFY_DONE; + + region_nid = cxl_region_nid(cxlr); + if (nid != region_nid) + return NOTIFY_DONE; + + /* No action needed if node bit already set */ + if (node_test_and_set(nid, nodemask_region_seen)) + return NOTIFY_DONE; + + if (!cxl_region_update_coordinates(cxlr, nid)) + return NOTIFY_DONE; + + return NOTIFY_OK; +} + /** * devm_cxl_add_region - Adds a region to a decoder * @cxlrd: root decoder @@ -2282,6 +2463,10 @@ static struct cxl_region *devm_cxl_add_region(struct cxl_root_decoder *cxlrd, if (rc) goto err; + cxlr->memory_notifier.notifier_call = cxl_region_perf_attrs_callback; + cxlr->memory_notifier.priority = CXL_CALLBACK_PRI; + register_memory_notifier(&cxlr->memory_notifier); + rc = devm_add_action_or_reset(port->uport_dev, unregister_region, cxlr); if (rc) return ERR_PTR(rc); diff --git a/drivers/cxl/core/regs.c b/drivers/cxl/core/regs.c index 92ef68849fbea..7eebbcbe2e098 100644 --- a/drivers/cxl/core/regs.c +++ b/drivers/cxl/core/regs.c @@ -216,16 +216,16 @@ int cxl_map_component_regs(const struct cxl_register_map *map, for (i = 0; i < ARRAY_SIZE(mapinfo); i++) { struct mapinfo *mi = &mapinfo[i]; - resource_size_t phys_addr; + resource_size_t addr; resource_size_t length; if (!mi->rmap->valid) continue; if (!test_bit(mi->rmap->id, &map_mask)) continue; - phys_addr = map->resource + mi->rmap->offset; + addr = map->resource + mi->rmap->offset; length = mi->rmap->size; - *(mi->addr) = devm_cxl_iomap_block(host, phys_addr, length); + *(mi->addr) = devm_cxl_iomap_block(host, addr, length); if (!*(mi->addr)) return -ENOMEM; } @@ -387,10 +387,9 @@ int cxl_count_regblock(struct pci_dev *pdev, enum cxl_regloc_type type) } EXPORT_SYMBOL_NS_GPL(cxl_count_regblock, CXL); -int cxl_map_pmu_regs(struct pci_dev *pdev, struct cxl_pmu_regs *regs, - struct cxl_register_map *map) +int cxl_map_pmu_regs(struct cxl_register_map *map, struct cxl_pmu_regs *regs) { - struct device *dev = &pdev->dev; + struct device *dev = map->host; resource_size_t phys_addr; phys_addr = map->resource; @@ -471,6 +470,42 @@ int cxl_setup_regs(struct cxl_register_map *map) } EXPORT_SYMBOL_NS_GPL(cxl_setup_regs, CXL); +u16 cxl_rcrb_to_aer(struct device *dev, resource_size_t rcrb) +{ + void __iomem *addr; + u16 offset = 0; + u32 cap_hdr; + + if (WARN_ON_ONCE(rcrb == CXL_RESOURCE_NONE)) + return 0; + + if (!request_mem_region(rcrb, SZ_4K, dev_name(dev))) + return 0; + + addr = ioremap(rcrb, SZ_4K); + if (!addr) + goto out; + + cap_hdr = readl(addr + offset); + while (PCI_EXT_CAP_ID(cap_hdr) != PCI_EXT_CAP_ID_ERR) { + offset = PCI_EXT_CAP_NEXT(cap_hdr); + + /* Offset 0 terminates capability list. */ + if (!offset) + break; + cap_hdr = readl(addr + offset); + } + + if (offset) + dev_dbg(dev, "found AER extended capability (0x%x)\n", offset); + + iounmap(addr); +out: + release_mem_region(rcrb, SZ_4K); + + return offset; +} + resource_size_t __rcrb_to_component(struct device *dev, struct cxl_rcrb_info *ri, enum cxl_rcrb which) { diff --git a/drivers/cxl/core/trace.h b/drivers/cxl/core/trace.h index 1e1e50d39bf63..e110125bc75c2 100644 --- a/drivers/cxl/core/trace.h +++ b/drivers/cxl/core/trace.h @@ -181,6 +181,7 @@ TRACE_EVENT(cxl_overflow, * 1) Add CXL_EVT_TP_entry to TP_STRUCT__entry * 2) Use CXL_EVT_TP_fast_assign within TP_fast_assign; * pass the dev, log, and CXL event header + * NOTE: The uuid must be assigned by the specific trace event * 3) Use CXL_EVT_TP_printk() instead of TP_printk() * * See the generic_event tracepoint as an example. @@ -203,7 +204,6 @@ TRACE_EVENT(cxl_overflow, __assign_str(host, dev_name((cxlmd)->dev.parent)); \ __entry->log = (l); \ __entry->serial = (cxlmd)->cxlds->serial; \ - memcpy(&__entry->hdr_uuid, &(hdr).id, sizeof(uuid_t)); \ __entry->hdr_length = (hdr).length; \ __entry->hdr_flags = get_unaligned_le24((hdr).flags); \ __entry->hdr_handle = le16_to_cpu((hdr).handle); \ @@ -225,9 +225,9 @@ TRACE_EVENT(cxl_overflow, TRACE_EVENT(cxl_generic_event, TP_PROTO(const struct cxl_memdev *cxlmd, enum cxl_event_log_type log, - struct cxl_event_record_raw *rec), + const uuid_t *uuid, struct cxl_event_generic *gen_rec), - TP_ARGS(cxlmd, log, rec), + TP_ARGS(cxlmd, log, uuid, gen_rec), TP_STRUCT__entry( CXL_EVT_TP_entry @@ -235,8 +235,9 @@ TRACE_EVENT(cxl_generic_event, ), TP_fast_assign( - CXL_EVT_TP_fast_assign(cxlmd, log, rec->hdr); - memcpy(__entry->data, &rec->data, CXL_EVENT_RECORD_DATA_LENGTH); + CXL_EVT_TP_fast_assign(cxlmd, log, gen_rec->hdr); + memcpy(&__entry->hdr_uuid, uuid, sizeof(uuid_t)); + memcpy(__entry->data, gen_rec->data, CXL_EVENT_RECORD_DATA_LENGTH); ), CXL_EVT_TP_printk("%s", @@ -337,6 +338,7 @@ TRACE_EVENT(cxl_general_media, TP_fast_assign( CXL_EVT_TP_fast_assign(cxlmd, log, rec->hdr); + memcpy(&__entry->hdr_uuid, &CXL_EVENT_GEN_MEDIA_UUID, sizeof(uuid_t)); /* General Media */ __entry->dpa = le64_to_cpu(rec->phys_addr); @@ -434,6 +436,7 @@ TRACE_EVENT(cxl_dram, TP_fast_assign( CXL_EVT_TP_fast_assign(cxlmd, log, rec->hdr); + memcpy(&__entry->hdr_uuid, &CXL_EVENT_DRAM_UUID, sizeof(uuid_t)); /* DRAM */ __entry->dpa = le64_to_cpu(rec->phys_addr); @@ -581,6 +584,7 @@ TRACE_EVENT(cxl_memory_module, TP_fast_assign( CXL_EVT_TP_fast_assign(cxlmd, log, rec->hdr); + memcpy(&__entry->hdr_uuid, &CXL_EVENT_MEM_MODULE_UUID, sizeof(uuid_t)); /* Memory Module Event */ __entry->event_type = rec->event_type; diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index bcfcbf65d18a0..371945d95b9ab 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -6,8 +6,10 @@ #include #include +#include #include #include +#include #include /** @@ -223,6 +225,14 @@ struct cxl_regs { struct_group_tagged(cxl_pmu_regs, pmu_regs, void __iomem *pmu; ); + + /* + * RCH downstream port specific RAS register + * @aer: CXL 3.0 8.2.1.1 RCH Downstream Port RCRB + */ + struct_group_tagged(cxl_rch_regs, rch_regs, + void __iomem *dport_aer; + ); }; struct cxl_reg_map { @@ -280,8 +290,7 @@ int cxl_map_component_regs(const struct cxl_register_map *map, unsigned long map_mask); int cxl_map_device_regs(const struct cxl_register_map *map, struct cxl_device_regs *regs); -int cxl_map_pmu_regs(struct pci_dev *pdev, struct cxl_pmu_regs *regs, - struct cxl_register_map *map); +int cxl_map_pmu_regs(struct cxl_register_map *map, struct cxl_pmu_regs *regs); enum cxl_regloc_type; int cxl_count_regblock(struct pci_dev *pdev, enum cxl_regloc_type type); @@ -323,6 +332,7 @@ enum cxl_decoder_type { */ #define CXL_DECODER_MAX_INTERLEAVE 16 +#define CXL_QOS_CLASS_INVALID -1 /** * struct cxl_decoder - Common CXL HDM Decoder Attributes @@ -432,6 +442,7 @@ typedef struct cxl_dport *(*cxl_calc_hb_fn)(struct cxl_root_decoder *cxlrd, * @calc_hb: which host bridge covers the n'th position by granularity * @platform_data: platform specific configuration data * @range_lock: sync region autodiscovery by address range + * @qos_class: QoS performance class cookie * @cxlsd: base cxl switch decoder */ struct cxl_root_decoder { @@ -440,6 +451,7 @@ struct cxl_root_decoder { cxl_calc_hb_fn calc_hb; void *platform_data; struct mutex range_lock; + int qos_class; struct cxl_switch_decoder cxlsd; }; @@ -508,6 +520,8 @@ struct cxl_region_params { * @cxlr_pmem: (for pmem regions) cached copy of the nvdimm bridge * @flags: Region state flags * @params: active + config params for the region + * @coord: QoS access coordinates for the region + * @memory_notifier: notifier for setting the access coordinates to node */ struct cxl_region { struct device dev; @@ -518,6 +532,8 @@ struct cxl_region { struct cxl_pmem_region *cxlr_pmem; unsigned long flags; struct cxl_region_params params; + struct access_coordinate coord[ACCESS_COORDINATE_MAX]; + struct notifier_block memory_notifier; }; struct cxl_nvdimm_bridge { @@ -572,15 +588,15 @@ struct cxl_dax_region { * @regions: cxl_region_ref instances, regions mapped by this port * @parent_dport: dport that points to this port in the parent * @decoder_ida: allocator for decoder ids - * @comp_map: component register capability mappings + * @reg_map: component and ras register mapping parameters * @nr_dports: number of entries in @dports * @hdm_end: track last allocated HDM decoder instance for allocation ordering * @commit_end: cursor to track highest committed decoder for commit ordering - * @component_reg_phys: component register capability base address (optional) * @dead: last ep has been removed, force port re-creation * @depth: How deep this port is relative to the root. depth 0 is the root. * @cdat: Cached CDAT data * @cdat_available: Should a CDAT attribute be available in sysfs + * @pci_latency: Upstream latency in picoseconds */ struct cxl_port { struct device dev; @@ -592,11 +608,10 @@ struct cxl_port { struct xarray regions; struct cxl_dport *parent_dport; struct ida decoder_ida; - struct cxl_register_map comp_map; + struct cxl_register_map reg_map; int nr_dports; int hdm_end; int commit_end; - resource_size_t component_reg_phys; bool dead; unsigned int depth; struct cxl_cdat { @@ -604,6 +619,30 @@ struct cxl_port { size_t length; } cdat; bool cdat_available; + long pci_latency; +}; + +/** + * struct cxl_root - logical collection of root cxl_port items + * + * @port: cxl_port member + * @ops: cxl root operations + */ +struct cxl_root { + struct cxl_port port; + const struct cxl_root_ops *ops; +}; + +static inline struct cxl_root * +to_cxl_root(const struct cxl_port *port) +{ + return container_of(port, struct cxl_root, port); +} + +struct cxl_root_ops { + int (*qos_class)(struct cxl_root *cxl_root, + struct access_coordinate *coord, int entries, + int *qos_class); }; static inline struct cxl_dport * @@ -620,19 +659,25 @@ struct cxl_rcrb_info { /** * struct cxl_dport - CXL downstream port * @dport_dev: PCI bridge or firmware device representing the downstream link - * @comp_map: component register capability mappings + * @reg_map: component and ras register mapping parameters * @port_id: unique hardware identifier for dport in decoder target list * @rcrb: Data about the Root Complex Register Block layout * @rch: Indicate whether this dport was enumerated in RCH or VH mode * @port: reference to cxl_port that contains this downstream port + * @regs: Dport parsed register blocks + * @coord: access coordinates (bandwidth and latency performance attributes) + * @link_latency: calculated PCIe downstream latency */ struct cxl_dport { struct device *dport_dev; - struct cxl_register_map comp_map; + struct cxl_register_map reg_map; int port_id; struct cxl_rcrb_info rcrb; bool rch; struct cxl_port *port; + struct cxl_regs regs; + struct access_coordinate coord[ACCESS_COORDINATE_MAX]; + long link_latency; }; /** @@ -691,7 +736,12 @@ struct cxl_port *devm_cxl_add_port(struct device *host, struct device *uport_dev, resource_size_t component_reg_phys, struct cxl_dport *parent_dport); -struct cxl_port *find_cxl_root(struct cxl_port *port); +struct cxl_root *devm_cxl_add_root(struct device *host, + const struct cxl_root_ops *ops); +struct cxl_root *find_cxl_root(struct cxl_port *port); +void put_cxl_root(struct cxl_root *cxl_root); +DEFINE_FREE(put_cxl_root, struct cxl_root *, if (_T) put_cxl_root(_T)) + int devm_cxl_enumerate_ports(struct cxl_memdev *cxlmd); void cxl_bus_rescan(void); void cxl_bus_drain(void); @@ -708,6 +758,13 @@ struct cxl_dport *devm_cxl_add_rch_dport(struct cxl_port *port, struct device *dport_dev, int port_id, resource_size_t rcrb); +#ifdef CONFIG_PCIEAER_CXL +void cxl_setup_parent_dport(struct device *host, struct cxl_dport *dport); +#else +static inline void cxl_setup_parent_dport(struct device *host, + struct cxl_dport *dport) { } +#endif + struct cxl_decoder *to_cxl_decoder(struct device *dev); struct cxl_root_decoder *to_cxl_root_decoder(struct device *dev); struct cxl_switch_decoder *to_cxl_switch_decoder(struct device *dev); @@ -823,6 +880,20 @@ static inline struct cxl_dax_region *to_cxl_dax_region(struct device *dev) } #endif +void cxl_endpoint_parse_cdat(struct cxl_port *port); +void cxl_switch_parse_cdat(struct cxl_port *port); + +int cxl_endpoint_get_perf_coordinates(struct cxl_port *port, + struct access_coordinate *coord); +void cxl_region_perf_data_calculate(struct cxl_region *cxlr, + struct cxl_endpoint_decoder *cxled); + +void cxl_memdev_update_perf(struct cxl_memdev *cxlmd); + +void cxl_coordinates_combine(struct access_coordinate *out, + struct access_coordinate *c1, + struct access_coordinate *c2); + /* * Unit test builds overrides this to __weak, find the 'strong' version * of these symbols in tools/testing/cxl/. diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h index edb46123e3eb0..b901b6ad45de5 100644 --- a/drivers/cxl/cxlmem.h +++ b/drivers/cxl/cxlmem.h @@ -6,6 +6,8 @@ #include #include #include +#include +#include #include "cxl.h" /* CXL 2.0 8.2.8.5.1.1 Memory Device Status Register */ @@ -391,6 +393,18 @@ enum cxl_devtype { CXL_DEVTYPE_CLASSMEM, }; +/** + * struct cxl_dpa_perf - DPA performance property entry + * @dpa_range - range for DPA address + * @coord - QoS performance data (i.e. latency, bandwidth) + * @qos_class - QoS Class cookies + */ +struct cxl_dpa_perf { + struct range dpa_range; + struct access_coordinate coord[ACCESS_COORDINATE_MAX]; + int qos_class; +}; + /** * struct cxl_dev_state - The driver device state * @@ -400,6 +414,7 @@ enum cxl_devtype { * * @dev: The device associated with this CXL state * @cxlmd: The device representing the CXL.mem capabilities of @dev + * @reg_map: component and ras register mapping parameters * @regs: Parsed register blocks * @cxl_dvsec: Offset to the PCIe device DVSEC * @rcd: operating in RCD mode (CXL 3.0 9.11.8 CXL Devices Attached to an RCH) @@ -407,13 +422,13 @@ enum cxl_devtype { * @dpa_res: Overall DPA resource tree for the device * @pmem_res: Active Persistent memory capacity configuration * @ram_res: Active Volatile memory capacity configuration - * @component_reg_phys: register base of component registers * @serial: PCIe Device Serial Number * @type: Generic Memory Class device or Vendor Specific Memory device */ struct cxl_dev_state { struct device *dev; struct cxl_memdev *cxlmd; + struct cxl_register_map reg_map; struct cxl_regs regs; int cxl_dvsec; bool rcd; @@ -421,7 +436,6 @@ struct cxl_dev_state { struct resource dpa_res; struct resource pmem_res; struct resource ram_res; - resource_size_t component_reg_phys; u64 serial; enum cxl_devtype type; }; @@ -455,6 +469,8 @@ struct cxl_dev_state { * @security: security driver state info * @fw: firmware upload / activation state * @mbox_send: @dev specific transport for transmitting mailbox commands + * @ram_perf: performance data entry matched to RAM partition + * @pmem_perf: performance data entry matched to PMEM partition * * See CXL 3.0 8.2.9.8.2 Capacity Configuration and Label Storage for * details on capacity parameters. @@ -475,6 +491,10 @@ struct cxl_memdev_state { u64 active_persistent_bytes; u64 next_volatile_bytes; u64 next_persistent_bytes; + + struct cxl_dpa_perf ram_perf; + struct cxl_dpa_perf pmem_perf; + struct cxl_event_state event; struct cxl_poison_state poison; struct cxl_security_state security; @@ -503,6 +523,7 @@ enum cxl_opcode { CXL_MBOX_OP_GET_FW_INFO = 0x0200, CXL_MBOX_OP_TRANSFER_FW = 0x0201, CXL_MBOX_OP_ACTIVATE_FW = 0x0202, + CXL_MBOX_OP_GET_TIMESTAMP = 0x0300, CXL_MBOX_OP_SET_TIMESTAMP = 0x0301, CXL_MBOX_OP_GET_SUPPORTED_LOGS = 0x0400, CXL_MBOX_OP_GET_LOG = 0x0401, @@ -580,25 +601,28 @@ struct cxl_mbox_identify { } __packed; /* - * Common Event Record Format - * CXL rev 3.0 section 8.2.9.2.1; Table 8-42 + * General Media Event Record UUID + * CXL rev 3.0 Section 8.2.9.2.1.1; Table 8-43 */ -struct cxl_event_record_hdr { - uuid_t id; - u8 length; - u8 flags[3]; - __le16 handle; - __le16 related_handle; - __le64 timestamp; - u8 maint_op_class; - u8 reserved[15]; -} __packed; +#define CXL_EVENT_GEN_MEDIA_UUID \ + UUID_INIT(0xfbcd0a77, 0xc260, 0x417f, 0x85, 0xa9, 0x08, 0x8b, 0x16, \ + 0x21, 0xeb, 0xa6) -#define CXL_EVENT_RECORD_DATA_LENGTH 0x50 -struct cxl_event_record_raw { - struct cxl_event_record_hdr hdr; - u8 data[CXL_EVENT_RECORD_DATA_LENGTH]; -} __packed; +/* + * DRAM Event Record UUID + * CXL rev 3.0 section 8.2.9.2.1.2; Table 8-44 + */ +#define CXL_EVENT_DRAM_UUID \ + UUID_INIT(0x601dcbb3, 0x9c06, 0x4eab, 0xb8, 0xaf, 0x4e, 0x9b, 0xfb, \ + 0x5c, 0x96, 0x24) + +/* + * Memory Module Event Record UUID + * CXL rev 3.0 section 8.2.9.2.1.3; Table 8-45 + */ +#define CXL_EVENT_MEM_MODULE_UUID \ + UUID_INIT(0xfe927475, 0xdd59, 0x4339, 0xa5, 0x86, 0x79, 0xba, 0xb1, \ + 0x13, 0xb7, 0x74) /* * Get Event Records output payload @@ -641,74 +665,6 @@ struct cxl_mbox_clear_event_payload { } __packed; #define CXL_CLEAR_EVENT_MAX_HANDLES U8_MAX -/* - * General Media Event Record - * CXL rev 3.0 Section 8.2.9.2.1.1; Table 8-43 - */ -#define CXL_EVENT_GEN_MED_COMP_ID_SIZE 0x10 -struct cxl_event_gen_media { - struct cxl_event_record_hdr hdr; - __le64 phys_addr; - u8 descriptor; - u8 type; - u8 transaction_type; - u8 validity_flags[2]; - u8 channel; - u8 rank; - u8 device[3]; - u8 component_id[CXL_EVENT_GEN_MED_COMP_ID_SIZE]; - u8 reserved[46]; -} __packed; - -/* - * DRAM Event Record - DER - * CXL rev 3.0 section 8.2.9.2.1.2; Table 3-44 - */ -#define CXL_EVENT_DER_CORRECTION_MASK_SIZE 0x20 -struct cxl_event_dram { - struct cxl_event_record_hdr hdr; - __le64 phys_addr; - u8 descriptor; - u8 type; - u8 transaction_type; - u8 validity_flags[2]; - u8 channel; - u8 rank; - u8 nibble_mask[3]; - u8 bank_group; - u8 bank; - u8 row[3]; - u8 column[2]; - u8 correction_mask[CXL_EVENT_DER_CORRECTION_MASK_SIZE]; - u8 reserved[0x17]; -} __packed; - -/* - * Get Health Info Record - * CXL rev 3.0 section 8.2.9.8.3.1; Table 8-100 - */ -struct cxl_get_health_info { - u8 health_status; - u8 media_status; - u8 add_status; - u8 life_used; - u8 device_temp[2]; - u8 dirty_shutdown_cnt[4]; - u8 cor_vol_err_cnt[4]; - u8 cor_per_err_cnt[4]; -} __packed; - -/* - * Memory Module Event Record - * CXL rev 3.0 section 8.2.9.2.1.3; Table 8-45 - */ -struct cxl_event_mem_module { - struct cxl_event_record_hdr hdr; - u8 event_type; - struct cxl_get_health_info info; - u8 reserved[0x3d]; -} __packed; - struct cxl_mbox_get_partition_info { __le64 active_volatile_cap; __le64 active_persistent_cap; @@ -866,6 +822,10 @@ void set_exclusive_cxl_commands(struct cxl_memdev_state *mds, void clear_exclusive_cxl_commands(struct cxl_memdev_state *mds, unsigned long *cmds); void cxl_mem_get_event_records(struct cxl_memdev_state *mds, u32 status); +void cxl_event_trace_record(const struct cxl_memdev *cxlmd, + enum cxl_event_log_type type, + enum cxl_event_type event_type, + const uuid_t *uuid, union cxl_event *evt); int cxl_set_timestamp(struct cxl_memdev_state *mds); int cxl_poison_state_init(struct cxl_memdev_state *mds); int cxl_mem_get_poison(struct cxl_memdev *cxlmd, u64 offset, u64 len, diff --git a/drivers/cxl/cxlpci.h b/drivers/cxl/cxlpci.h index 0fa4799ea316c..93992a1c8eecf 100644 --- a/drivers/cxl/cxlpci.h +++ b/drivers/cxl/cxlpci.h @@ -71,6 +71,15 @@ enum cxl_regloc_type { CXL_REGLOC_RBI_TYPES }; +/* + * Table Access DOE, CDAT Read Entry Response + * + * Spec refs: + * + * CXL 3.1 8.1.11, Table 8-14: Read Entry Response + * CDAT Specification 1.03: 2 CDAT Data Structures + */ + struct cdat_header { __le32 length; u8 revision; @@ -85,6 +94,34 @@ struct cdat_entry_header { __le16 length; } __packed; +/* + * The DOE CDAT read response contains a CDAT read entry (either the + * CDAT header or a structure). + */ +union cdat_data { + struct cdat_header header; + struct cdat_entry_header entry; +} __packed; + +/* There is an additional CDAT response header of 4 bytes. */ +struct cdat_doe_rsp { + __le32 doe_header; + u8 data[]; +} __packed; + +/* + * CXL v3.0 6.2.3 Table 6-4 + * The table indicates that if PCIe Flit Mode is set, then CXL is in 256B flits + * mode, otherwise it's 68B flits mode. + */ +static inline bool cxl_pci_flit_256(struct pci_dev *pdev) +{ + u16 lnksta2; + + pcie_capability_read_word(pdev, PCI_EXP_LNKSTA2, &lnksta2); + return lnksta2 & PCI_EXP_LNKSTA2_FLIT; +} + int devm_cxl_port_enumerate_dports(struct cxl_port *port); struct cxl_dev_state; int cxl_hdm_decode_init(struct cxl_dev_state *cxlds, struct cxl_hdm *cxlhdm, diff --git a/drivers/cxl/mem.c b/drivers/cxl/mem.c index 317c7548e4e9b..0c79d9ce877cc 100644 --- a/drivers/cxl/mem.c +++ b/drivers/cxl/mem.c @@ -49,7 +49,6 @@ static int devm_cxl_add_endpoint(struct device *host, struct cxl_memdev *cxlmd, struct cxl_dport *parent_dport) { struct cxl_port *parent_port = parent_dport->port; - struct cxl_dev_state *cxlds = cxlmd->cxlds; struct cxl_port *endpoint, *iter, *down; int rc; @@ -65,8 +64,8 @@ static int devm_cxl_add_endpoint(struct device *host, struct cxl_memdev *cxlmd, ep->next = down; } - endpoint = devm_cxl_add_port(host, &cxlmd->dev, - cxlds->component_reg_phys, + /* Note: endpoint port component registers are derived from @cxlds */ + endpoint = devm_cxl_add_port(host, &cxlmd->dev, CXL_RESOURCE_NONE, parent_dport); if (IS_ERR(endpoint)) return PTR_ERR(endpoint); @@ -158,6 +157,8 @@ static int cxl_mem_probe(struct device *dev) else endpoint_parent = &parent_port->dev; + cxl_setup_parent_dport(dev, dport); + device_lock(endpoint_parent); if (!endpoint_parent->driver) { dev_err(dev, "CXL port topology %s not enabled\n", @@ -216,16 +217,15 @@ static DEVICE_ATTR_WO(trigger_poison_list); static umode_t cxl_mem_visible(struct kobject *kobj, struct attribute *a, int n) { - if (a == &dev_attr_trigger_poison_list.attr) { - struct device *dev = kobj_to_dev(kobj); - struct cxl_memdev *cxlmd = to_cxl_memdev(dev); - struct cxl_memdev_state *mds = - to_cxl_memdev_state(cxlmd->cxlds); + struct device *dev = kobj_to_dev(kobj); + struct cxl_memdev *cxlmd = to_cxl_memdev(dev); + struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); + if (a == &dev_attr_trigger_poison_list.attr) if (!test_bit(CXL_POISON_ENABLED_LIST, mds->poison.enabled_cmds)) return 0; - } + return a->mode; } diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c index aacd93f9067da..bf900eba504c9 100644 --- a/drivers/cxl/pci.c +++ b/drivers/cxl/pci.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only /* Copyright(c) 2020 Intel Corporation. All rights reserved. */ +#include #include #include #include @@ -85,25 +86,28 @@ static int cxl_pci_mbox_wait_for_doorbell(struct cxl_dev_state *cxlds) status & CXLMDEV_DEV_FATAL ? " fatal" : "", \ status & CXLMDEV_FW_HALT ? " firmware-halt" : "") +/* + * Threaded irq dev_id's must be globally unique. cxl_dev_id provides a unique + * wrapper object for each irq within the same cxlds. + */ struct cxl_dev_id { struct cxl_dev_state *cxlds; }; static int cxl_request_irq(struct cxl_dev_state *cxlds, int irq, - irq_handler_t handler, irq_handler_t thread_fn) + irq_handler_t thread_fn) { struct device *dev = cxlds->dev; struct cxl_dev_id *dev_id; - /* dev_id must be globally unique and must contain the cxlds */ dev_id = devm_kzalloc(dev, sizeof(*dev_id), GFP_KERNEL); if (!dev_id) return -ENOMEM; dev_id->cxlds = cxlds; - return devm_request_threaded_irq(dev, irq, handler, thread_fn, - IRQF_SHARED | IRQF_ONESHOT, - NULL, dev_id); + return devm_request_threaded_irq(dev, irq, NULL, thread_fn, + IRQF_SHARED | IRQF_ONESHOT, NULL, + dev_id); } static bool cxl_mbox_background_complete(struct cxl_dev_state *cxlds) @@ -378,7 +382,7 @@ static int cxl_pci_mbox_send(struct cxl_memdev_state *mds, return rc; } -static int cxl_pci_setup_mailbox(struct cxl_memdev_state *mds) +static int cxl_pci_setup_mailbox(struct cxl_memdev_state *mds, bool irq_avail) { struct cxl_dev_state *cxlds = &mds->cxlds; const int cap = readl(cxlds->regs.mbox + CXLDEV_MBOX_CAPS_OFFSET); @@ -437,7 +441,7 @@ static int cxl_pci_setup_mailbox(struct cxl_memdev_state *mds) INIT_DELAYED_WORK(&mds->security.poll_dwork, cxl_mbox_sanitize_work); /* background command interrupts are optional */ - if (!(cap & CXLDEV_MBOX_CAP_BG_CMD_IRQ)) + if (!(cap & CXLDEV_MBOX_CAP_BG_CMD_IRQ) || !irq_avail) return 0; msgnum = FIELD_GET(CXLDEV_MBOX_CAP_IRQ_MSGNUM_MASK, cap); @@ -445,7 +449,7 @@ static int cxl_pci_setup_mailbox(struct cxl_memdev_state *mds) if (irq < 0) return 0; - if (cxl_request_irq(cxlds, irq, NULL, cxl_pci_mbox_irq)) + if (cxl_request_irq(cxlds, irq, cxl_pci_mbox_irq)) return 0; dev_dbg(cxlds->dev, "Mailbox interrupts enabled\n"); @@ -584,7 +588,7 @@ static int cxl_mem_alloc_event_buf(struct cxl_memdev_state *mds) return devm_add_action_or_reset(mds->cxlds.dev, free_event_buf, buf); } -static int cxl_alloc_irq_vectors(struct pci_dev *pdev) +static bool cxl_alloc_irq_vectors(struct pci_dev *pdev) { int nvecs; @@ -601,9 +605,9 @@ static int cxl_alloc_irq_vectors(struct pci_dev *pdev) PCI_IRQ_MSIX | PCI_IRQ_MSI); if (nvecs < 1) { dev_dbg(&pdev->dev, "Failed to alloc irq vectors: %d\n", nvecs); - return -ENXIO; + return false; } - return 0; + return true; } static irqreturn_t cxl_event_thread(int irq, void *id) @@ -643,7 +647,7 @@ static int cxl_event_req_irq(struct cxl_dev_state *cxlds, u8 setting) if (irq < 0) return irq; - return cxl_request_irq(cxlds, irq, NULL, cxl_event_thread); + return cxl_request_irq(cxlds, irq, cxl_event_thread); } static int cxl_event_get_int_policy(struct cxl_memdev_state *mds, @@ -739,7 +743,7 @@ static bool cxl_event_int_is_fw(u8 setting) } static int cxl_event_config(struct pci_host_bridge *host_bridge, - struct cxl_memdev_state *mds) + struct cxl_memdev_state *mds, bool irq_avail) { struct cxl_event_interrupt_policy policy; int rc; @@ -751,6 +755,11 @@ static int cxl_event_config(struct pci_host_bridge *host_bridge, if (!host_bridge->native_cxl_error) return 0; + if (!irq_avail) { + dev_info(mds->cxlds.dev, "No interrupt support, disable event processing.\n"); + return 0; + } + rc = cxl_mem_alloc_event_buf(mds); if (rc) return rc; @@ -785,6 +794,7 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) struct cxl_register_map map; struct cxl_memdev *cxlmd; int i, rc, pmu_count; + bool irq_avail; /* * Double check the anonymous union trickery in struct cxl_regs @@ -824,16 +834,14 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) * If the component registers can't be found, the cxl_pci driver may * still be useful for management functions so don't return an error. */ - cxlds->component_reg_phys = CXL_RESOURCE_NONE; - rc = cxl_pci_setup_regs(pdev, CXL_REGLOC_RBI_COMPONENT, &map); + rc = cxl_pci_setup_regs(pdev, CXL_REGLOC_RBI_COMPONENT, + &cxlds->reg_map); if (rc) dev_warn(&pdev->dev, "No component registers (%d)\n", rc); - else if (!map.component_map.ras.valid) + else if (!cxlds->reg_map.component_map.ras.valid) dev_dbg(&pdev->dev, "RAS registers not found\n"); - cxlds->component_reg_phys = map.resource; - - rc = cxl_map_component_regs(&map, &cxlds->regs.component, + rc = cxl_map_component_regs(&cxlds->reg_map, &cxlds->regs.component, BIT(CXL_CM_CAP_CAP_ID_RAS)); if (rc) dev_dbg(&pdev->dev, "Failed to map RAS capability.\n"); @@ -844,11 +852,9 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) else dev_warn(&pdev->dev, "Media not active (%d)\n", rc); - rc = cxl_alloc_irq_vectors(pdev); - if (rc) - return rc; + irq_avail = cxl_alloc_irq_vectors(pdev); - rc = cxl_pci_setup_mailbox(mds); + rc = cxl_pci_setup_mailbox(mds, irq_avail); if (rc) return rc; @@ -894,7 +900,7 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) break; } - rc = cxl_map_pmu_regs(pdev, &pmu_regs, &map); + rc = cxl_map_pmu_regs(&map, &pmu_regs); if (rc) { dev_dbg(&pdev->dev, "Could not map PMU regs\n"); break; @@ -907,7 +913,7 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) } } - rc = cxl_event_config(host_bridge, mds); + rc = cxl_event_config(host_bridge, mds, irq_avail); if (rc) return rc; @@ -967,6 +973,75 @@ static struct pci_driver cxl_pci_driver = { }, }; +#define CXL_EVENT_HDR_FLAGS_REC_SEVERITY GENMASK(1, 0) +static void cxl_handle_cper_event(enum cxl_event_type ev_type, + struct cxl_cper_event_rec *rec) +{ + struct cper_cxl_event_devid *device_id = &rec->hdr.device_id; + struct pci_dev *pdev __free(pci_dev_put) = NULL; + enum cxl_event_log_type log_type; + struct cxl_dev_state *cxlds; + unsigned int devfn; + u32 hdr_flags; + + pr_debug("CPER event %d for device %u:%u:%u.%u\n", ev_type, + device_id->segment_num, device_id->bus_num, + device_id->device_num, device_id->func_num); + + devfn = PCI_DEVFN(device_id->device_num, device_id->func_num); + pdev = pci_get_domain_bus_and_slot(device_id->segment_num, + device_id->bus_num, devfn); + if (!pdev) + return; + + guard(device)(&pdev->dev); + if (pdev->driver != &cxl_pci_driver) + return; + + cxlds = pci_get_drvdata(pdev); + if (!cxlds) + return; + + /* Fabricate a log type */ + hdr_flags = get_unaligned_le24(rec->event.generic.hdr.flags); + log_type = FIELD_GET(CXL_EVENT_HDR_FLAGS_REC_SEVERITY, hdr_flags); + + cxl_event_trace_record(cxlds->cxlmd, log_type, ev_type, + &uuid_null, &rec->event); +} + +static void cxl_cper_work_fn(struct work_struct *work) +{ + struct cxl_cper_work_data wd; + + while (cxl_cper_kfifo_get(&wd)) + cxl_handle_cper_event(wd.event_type, &wd.rec); +} +static DECLARE_WORK(cxl_cper_work, cxl_cper_work_fn); + +static int __init cxl_pci_driver_init(void) +{ + int rc; + + rc = pci_register_driver(&cxl_pci_driver); + if (rc) + return rc; + + rc = cxl_cper_register_work(&cxl_cper_work); + if (rc) + pci_unregister_driver(&cxl_pci_driver); + + return rc; +} + +static void __exit cxl_pci_driver_exit(void) +{ + cxl_cper_unregister_work(&cxl_cper_work); + cancel_work_sync(&cxl_cper_work); + pci_unregister_driver(&cxl_pci_driver); +} + +module_init(cxl_pci_driver_init); +module_exit(cxl_pci_driver_exit); MODULE_LICENSE("GPL v2"); -module_pci_driver(cxl_pci_driver); MODULE_IMPORT_NS(CXL); diff --git a/drivers/cxl/port.c b/drivers/cxl/port.c index 6240e05b95424..97c21566677aa 100644 --- a/drivers/cxl/port.c +++ b/drivers/cxl/port.c @@ -62,10 +62,15 @@ static int cxl_switch_port_probe(struct cxl_port *port) struct cxl_hdm *cxlhdm; int rc; + /* Cache the data early to ensure is_visible() works */ + read_cdat_data(port); + rc = devm_cxl_port_enumerate_dports(port); if (rc < 0) return rc; + cxl_switch_parse_cdat(port); + cxlhdm = devm_cxl_setup_hdm(port, NULL); if (!IS_ERR(cxlhdm)) return devm_cxl_enumerate_decoders(cxlhdm, NULL); @@ -106,6 +111,7 @@ static int cxl_endpoint_port_probe(struct cxl_port *port) /* Cache the data early to ensure is_visible() works */ read_cdat_data(port); + cxl_endpoint_parse_cdat(port); get_device(&cxlmd->dev); rc = devm_add_action_or_reset(&port->dev, schedule_detach, cxlmd); @@ -124,14 +130,15 @@ static int cxl_endpoint_port_probe(struct cxl_port *port) * This can't fail in practice as CXL root exit unregisters all * descendant ports and that in turn synchronizes with cxl_port_probe() */ - root = find_cxl_root(port); + struct cxl_root *cxl_root __free(put_cxl_root) = find_cxl_root(port); + + root = &cxl_root->port; /* * Now that all endpoint decoders are successfully enumerated, try to * assemble regions from committed decoders */ device_for_each_child(&port->dev, root, discover_region); - put_device(&root->dev); return 0; } diff --git a/drivers/firmware/efi/cper.c b/drivers/firmware/efi/cper.c index 74ffaf44d74ce..481af0b6f22c4 100644 --- a/drivers/firmware/efi/cper.c +++ b/drivers/firmware/efi/cper.c @@ -583,6 +583,17 @@ static void cper_print_tstamp(const char *pfx, } } +struct ignore_section { + guid_t guid; + const char *name; +}; + +static const struct ignore_section ignore_sections[] = { + { .guid = CPER_SEC_CXL_GEN_MEDIA_GUID, .name = "CXL General Media Event" }, + { .guid = CPER_SEC_CXL_DRAM_GUID, .name = "CXL DRAM Event" }, + { .guid = CPER_SEC_CXL_MEM_MODULE_GUID, .name = "CXL Memory Module Event" }, +}; + static void cper_estatus_print_section(const char *pfx, struct acpi_hest_generic_data *gdata, int sec_no) @@ -603,6 +614,14 @@ cper_estatus_print_section(const char *pfx, struct acpi_hest_generic_data *gdata printk("%s""fru_text: %.20s\n", pfx, gdata->fru_text); snprintf(newpfx, sizeof(newpfx), "%s ", pfx); + + for (int i = 0; i < ARRAY_SIZE(ignore_sections); i++) { + if (guid_equal(sec_type, &ignore_sections[i].guid)) { + printk("%ssection_type: %s\n", newpfx, ignore_sections[i].name); + return; + } + } + if (guid_equal(sec_type, &CPER_SEC_PROC_GENERIC)) { struct cper_sec_proc_generic *proc_err = acpi_hest_get_payload(gdata); diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index 26af9e1470751..c9fab7aa428cd 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -6290,6 +6290,41 @@ int pcie_set_mps(struct pci_dev *dev, int mps) } EXPORT_SYMBOL(pcie_set_mps); +static enum pci_bus_speed to_pcie_link_speed(u16 lnksta) +{ + return pcie_link_speed[FIELD_GET(PCI_EXP_LNKSTA_CLS, lnksta)]; +} + +int pcie_link_speed_mbps(struct pci_dev *pdev) +{ + u16 lnksta; + int err; + + err = pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &lnksta); + if (err) + return err; + + switch (to_pcie_link_speed(lnksta)) { + case PCIE_SPEED_2_5GT: + return 2500; + case PCIE_SPEED_5_0GT: + return 5000; + case PCIE_SPEED_8_0GT: + return 8000; + case PCIE_SPEED_16_0GT: + return 16000; + case PCIE_SPEED_32_0GT: + return 32000; + case PCIE_SPEED_64_0GT: + return 64000; + default: + break; + } + + return -EINVAL; +} +EXPORT_SYMBOL(pcie_link_speed_mbps); + /** * pcie_bandwidth_available - determine minimum link settings of a PCIe * device and its bandwidth limitation @@ -6323,7 +6358,7 @@ u32 pcie_bandwidth_available(struct pci_dev *dev, struct pci_dev **limiting_dev, while (dev) { pcie_capability_read_word(dev, PCI_EXP_LNKSTA, &lnksta); - next_speed = pcie_link_speed[lnksta & PCI_EXP_LNKSTA_CLS]; + next_speed = to_pcie_link_speed(lnksta); next_width = FIELD_GET(PCI_EXP_LNKSTA_NLW, lnksta); next_bw = next_width * PCIE_SPEED2MBS_ENC(next_speed); diff --git a/drivers/pci/pcie/Kconfig b/drivers/pci/pcie/Kconfig index 228652a59f27e..8999fcebde6aa 100644 --- a/drivers/pci/pcie/Kconfig +++ b/drivers/pci/pcie/Kconfig @@ -49,6 +49,15 @@ config PCIEAER_INJECT gotten from: https://git.kernel.org/cgit/linux/kernel/git/gong.chen/aer-inject.git/ +config PCIEAER_CXL + bool "PCI Express CXL RAS support" + default y + depends on PCIEAER && CXL_PCI + help + Enables CXL error handling. + + If unsure, say Y. + # # PCI Express ECRC # diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c index 8e700020ee0bc..76a5f96434f41 100644 --- a/drivers/pci/pcie/aer.c +++ b/drivers/pci/pcie/aer.c @@ -760,9 +760,10 @@ int cper_severity_to_aer(int cper_severity) } } EXPORT_SYMBOL_GPL(cper_severity_to_aer); +#endif -void cper_print_aer(struct pci_dev *dev, int aer_severity, - struct aer_capability_regs *aer) +void pci_print_aer(struct pci_dev *dev, int aer_severity, + struct aer_capability_regs *aer) { int layer, agent, tlp_header_valid = 0; u32 status, mask; @@ -801,7 +802,7 @@ void cper_print_aer(struct pci_dev *dev, int aer_severity, trace_aer_event(dev_name(&dev->dev), (status & ~mask), aer_severity, tlp_header_valid, &aer->header_log); } -#endif +EXPORT_SYMBOL_NS_GPL(pci_print_aer, CXL); /** * add_error_device - list device to be handled @@ -939,14 +940,153 @@ static bool find_source_device(struct pci_dev *parent, return true; } +#ifdef CONFIG_PCIEAER_CXL + /** - * handle_error_source - handle logging error into an event log + * pci_aer_unmask_internal_errors - unmask internal errors + * @dev: pointer to the pcie_dev data structure + * + * Unmasks internal errors in the Uncorrectable and Correctable Error + * Mask registers. + * + * Note: AER must be enabled and supported by the device which must be + * checked in advance, e.g. with pcie_aer_is_native(). + */ +static void pci_aer_unmask_internal_errors(struct pci_dev *dev) +{ + int aer = dev->aer_cap; + u32 mask; + + pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, &mask); + mask &= ~PCI_ERR_UNC_INTN; + pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, mask); + + pci_read_config_dword(dev, aer + PCI_ERR_COR_MASK, &mask); + mask &= ~PCI_ERR_COR_INTERNAL; + pci_write_config_dword(dev, aer + PCI_ERR_COR_MASK, mask); +} + +static bool is_cxl_mem_dev(struct pci_dev *dev) +{ + /* + * The capability, status, and control fields in Device 0, + * Function 0 DVSEC control the CXL functionality of the + * entire device (CXL 3.0, 8.1.3). + */ + if (dev->devfn != PCI_DEVFN(0, 0)) + return false; + + /* + * CXL Memory Devices must have the 502h class code set (CXL + * 3.0, 8.1.12.1). + */ + if ((dev->class >> 8) != PCI_CLASS_MEMORY_CXL) + return false; + + return true; +} + +static bool cxl_error_is_native(struct pci_dev *dev) +{ + struct pci_host_bridge *host = pci_find_host_bridge(dev->bus); + + return (pcie_ports_native || host->native_aer); +} + +static bool is_internal_error(struct aer_err_info *info) +{ + if (info->severity == AER_CORRECTABLE) + return info->status & PCI_ERR_COR_INTERNAL; + + return info->status & PCI_ERR_UNC_INTN; +} + +static int cxl_rch_handle_error_iter(struct pci_dev *dev, void *data) +{ + struct aer_err_info *info = (struct aer_err_info *)data; + const struct pci_error_handlers *err_handler; + + if (!is_cxl_mem_dev(dev) || !cxl_error_is_native(dev)) + return 0; + + /* protect dev->driver */ + device_lock(&dev->dev); + + err_handler = dev->driver ? dev->driver->err_handler : NULL; + if (!err_handler) + goto out; + + if (info->severity == AER_CORRECTABLE) { + if (err_handler->cor_error_detected) + err_handler->cor_error_detected(dev); + } else if (err_handler->error_detected) { + if (info->severity == AER_NONFATAL) + err_handler->error_detected(dev, pci_channel_io_normal); + else if (info->severity == AER_FATAL) + err_handler->error_detected(dev, pci_channel_io_frozen); + } +out: + device_unlock(&dev->dev); + return 0; +} + +static void cxl_rch_handle_error(struct pci_dev *dev, struct aer_err_info *info) +{ + /* + * Internal errors of an RCEC indicate an AER error in an + * RCH's downstream port. Check and handle them in the CXL.mem + * device driver. + */ + if (pci_pcie_type(dev) == PCI_EXP_TYPE_RC_EC && + is_internal_error(info)) + pcie_walk_rcec(dev, cxl_rch_handle_error_iter, info); +} + +static int handles_cxl_error_iter(struct pci_dev *dev, void *data) +{ + bool *handles_cxl = data; + + if (!*handles_cxl) + *handles_cxl = is_cxl_mem_dev(dev) && cxl_error_is_native(dev); + + /* Non-zero terminates iteration */ + return *handles_cxl; +} + +static bool handles_cxl_errors(struct pci_dev *rcec) +{ + bool handles_cxl = false; + + if (pci_pcie_type(rcec) == PCI_EXP_TYPE_RC_EC && + pcie_aer_is_native(rcec)) + pcie_walk_rcec(rcec, handles_cxl_error_iter, &handles_cxl); + + return handles_cxl; +} + +static void cxl_rch_enable_rcec(struct pci_dev *rcec) +{ + if (!handles_cxl_errors(rcec)) + return; + + pci_aer_unmask_internal_errors(rcec); + pci_info(rcec, "CXL: Internal errors unmasked"); +} + +#else +static inline void cxl_rch_enable_rcec(struct pci_dev *dev) { } +static inline void cxl_rch_handle_error(struct pci_dev *dev, + struct aer_err_info *info) { } +#endif + +/** + * pci_aer_handle_error - handle logging error into an event log * @dev: pointer to pci_dev data structure of error source device * @info: comprehensive error information * * Invoked when an error being detected by Root Port. */ -static void handle_error_source(struct pci_dev *dev, struct aer_err_info *info) +static void pci_aer_handle_error(struct pci_dev *dev, struct aer_err_info *info) { int aer = dev->aer_cap; @@ -970,6 +1110,12 @@ static void handle_error_source(struct pci_dev *dev, struct aer_err_info *info) pcie_do_recovery(dev, pci_channel_io_normal, aer_root_reset); else if (info->severity == AER_FATAL) pcie_do_recovery(dev, pci_channel_io_frozen, aer_root_reset); +} + +static void handle_error_source(struct pci_dev *dev, struct aer_err_info *info) +{ + cxl_rch_handle_error(dev, info); + pci_aer_handle_error(dev, info); pci_dev_put(dev); } @@ -1002,7 +1148,7 @@ static void aer_recover_work_func(struct work_struct *work) PCI_SLOT(entry.devfn), PCI_FUNC(entry.devfn)); continue; } - cper_print_aer(pdev, entry.severity, entry.regs); + pci_print_aer(pdev, entry.severity, entry.regs); /* * Memory for aer_capability_regs(entry.regs) is being allocated from the * ghes_estatus_pool to protect it from overwriting when multiple sections @@ -1337,6 +1483,7 @@ static int aer_probe(struct pcie_device *dev) return status; } + cxl_rch_enable_rcec(port); aer_enable_rootport(rpc); pci_info(port, "enabled with IRQ %d\n", dev->irq); return 0; diff --git a/include/acpi/actbl1.h b/include/acpi/actbl1.h index 8d5572ad48cbe..a33375e055ad7 100644 --- a/include/acpi/actbl1.h +++ b/include/acpi/actbl1.h @@ -465,6 +465,9 @@ struct acpi_cdat_sslbe { u16 reserved; }; +#define ACPI_CDAT_SSLBIS_US_PORT 0x0100 +#define ACPI_CDAT_SSLBIS_ANY_PORT 0xffff + /******************************************************************************* * * CEDT - CXL Early Discovery Table diff --git a/include/linux/acpi.h b/include/linux/acpi.h index 7c6f4006389da..a057841c7f676 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -15,6 +15,7 @@ #include #include #include +#include struct irq_domain; struct irq_domain_ops; @@ -30,6 +31,7 @@ struct irq_domain_ops; #include #include #include +#include #include #include @@ -37,6 +39,16 @@ struct irq_domain_ops; #include #include +#ifdef CONFIG_ACPI_TABLE_LIB +#define EXPORT_SYMBOL_ACPI_LIB(x) EXPORT_SYMBOL_NS_GPL(x, ACPI) +#define __init_or_acpilib +#define __initdata_or_acpilib +#else +#define EXPORT_SYMBOL_ACPI_LIB(x) +#define __init_or_acpilib __init +#define __initdata_or_acpilib __initdata +#endif + static inline acpi_handle acpi_device_handle(struct acpi_device *adev) { return adev ? adev->handle : NULL; @@ -119,21 +131,8 @@ enum acpi_address_range_id { /* Table Handlers */ -union acpi_subtable_headers { - struct acpi_subtable_header common; - struct acpi_hmat_structure hmat; - struct acpi_prmt_module_header prmt; - struct acpi_cedt_header cedt; -}; - typedef int (*acpi_tbl_table_handler)(struct acpi_table_header *table); -typedef int (*acpi_tbl_entry_handler)(union acpi_subtable_headers *header, - const unsigned long end); - -typedef int (*acpi_tbl_entry_handler_arg)(union acpi_subtable_headers *header, - void *arg, const unsigned long end); - /* Debugger support */ struct acpi_debugger_ops { @@ -207,14 +206,6 @@ static inline int acpi_debugger_notify_command_complete(void) (!entry) || (unsigned long)entry + sizeof(*entry) > end || \ ((struct acpi_subtable_header *)entry)->length < sizeof(*entry)) -struct acpi_subtable_proc { - int id; - acpi_tbl_entry_handler handler; - acpi_tbl_entry_handler_arg handler_arg; - void *arg; - int count; -}; - void __iomem *__acpi_map_table(unsigned long phys, unsigned long size); void __acpi_unmap_table(void __iomem *map, unsigned long size); int early_acpi_boot_init(void); @@ -229,16 +220,6 @@ void acpi_reserve_initial_tables (void); void acpi_table_init_complete (void); int acpi_table_init (void); -#ifdef CONFIG_ACPI_TABLE_LIB -#define EXPORT_SYMBOL_ACPI_LIB(x) EXPORT_SYMBOL_NS_GPL(x, ACPI) -#define __init_or_acpilib -#define __initdata_or_acpilib -#else -#define EXPORT_SYMBOL_ACPI_LIB(x) -#define __init_or_acpilib __init -#define __initdata_or_acpilib __initdata -#endif - int acpi_table_parse(char *id, acpi_tbl_table_handler handler); int __init_or_acpilib acpi_table_parse_entries(char *id, unsigned long table_size, int entry_id, @@ -439,6 +420,16 @@ extern int acpi_blacklisted(void); extern void acpi_osi_setup(char *str); extern bool acpi_osi_is_win8(void); +#ifdef CONFIG_ACPI_HMAT +int acpi_get_genport_coordinates(u32 uid, struct access_coordinate *coord); +#else +static inline int acpi_get_genport_coordinates(u32 uid, + struct access_coordinate *coord) +{ + return -EOPNOTSUPP; +} +#endif + #ifdef CONFIG_ACPI_NUMA int acpi_map_pxm_to_node(int pxm); int acpi_get_node(acpi_handle handle); @@ -1555,4 +1546,30 @@ static inline void acpi_device_notify(struct device *dev) { } static inline void acpi_device_notify_remove(struct device *dev) { } #endif +static inline void acpi_use_parent_companion(struct device *dev) +{ + ACPI_COMPANION_SET(dev, ACPI_COMPANION(dev->parent)); +} + +#ifdef CONFIG_ACPI_HMAT +int hmat_update_target_coordinates(int nid, struct access_coordinate *coord, + enum access_coordinate_class access); +#else +static inline int hmat_update_target_coordinates(int nid, + struct access_coordinate *coord, + enum access_coordinate_class access) +{ + return -EOPNOTSUPP; +} +#endif + +#ifdef CONFIG_ACPI_NUMA +bool acpi_node_backed_by_real_pxm(int nid); +#else +static inline bool acpi_node_backed_by_real_pxm(int nid) +{ + return false; +} +#endif + #endif /*_LINUX_ACPI_H*/ diff --git a/include/linux/aer.h b/include/linux/aer.h index 29cc10220952d..f6ea2f57d8089 100644 --- a/include/linux/aer.h +++ b/include/linux/aer.h @@ -51,7 +51,7 @@ static inline int pci_aer_clear_nonfatal_status(struct pci_dev *dev) static inline int pcie_aer_is_native(struct pci_dev *dev) { return 0; } #endif -void cper_print_aer(struct pci_dev *dev, int aer_severity, +void pci_print_aer(struct pci_dev *dev, int aer_severity, struct aer_capability_regs *aer); int cper_severity_to_aer(int cper_severity); void aer_recover_queue(int domain, unsigned int bus, unsigned int devfn, diff --git a/include/linux/cper.h b/include/linux/cper.h index ad1ed24730917..3670b866ac119 100644 --- a/include/linux/cper.h +++ b/include/linux/cper.h @@ -90,6 +90,29 @@ enum { GUID_INIT(0x667DD791, 0xC6B3, 0x4c27, 0x8A, 0x6B, 0x0F, 0x8E, \ 0x72, 0x2D, 0xEB, 0x41) +/* CXL Event record UUIDs are formatted as GUIDs and reported in section type */ +/* + * General Media Event Record + * CXL rev 3.0 Section 8.2.9.2.1.1; Table 8-43 + */ +#define CPER_SEC_CXL_GEN_MEDIA_GUID \ + GUID_INIT(0xfbcd0a77, 0xc260, 0x417f, \ + 0x85, 0xa9, 0x08, 0x8b, 0x16, 0x21, 0xeb, 0xa6) +/* + * DRAM Event Record + * CXL rev 3.0 section 8.2.9.2.1.2; Table 8-44 + */ +#define CPER_SEC_CXL_DRAM_GUID \ + GUID_INIT(0x601dcbb3, 0x9c06, 0x4eab, \ + 0xb8, 0xaf, 0x4e, 0x9b, 0xfb, 0x5c, 0x96, 0x24) +/* + * Memory Module Event Record + * CXL rev 3.0 section 8.2.9.2.1.3; Table 8-45 + */ +#define CPER_SEC_CXL_MEM_MODULE_GUID \ + GUID_INIT(0xfe927475, 0xdd59, 0x4339, \ + 0xa5, 0x86, 0x79, 0xba, 0xb1, 0x13, 0xb7, 0x74) + /* * Flags bits definitions for flags in struct cper_record_header * If set, the error has been recovered diff --git a/include/linux/cxl-event.h b/include/linux/cxl-event.h new file mode 100644 index 0000000000000..02cf03cbd3fa8 --- /dev/null +++ b/include/linux/cxl-event.h @@ -0,0 +1,170 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2023 Intel Corporation. */ +#ifndef _LINUX_CXL_EVENT_H +#define _LINUX_CXL_EVENT_H + +#include + +/* + * Common Event Record Format + * CXL rev 3.0 section 8.2.9.2.1; Table 8-42 + */ +struct cxl_event_record_hdr { + u8 length; + u8 flags[3]; + __le16 handle; + __le16 related_handle; + __le64 timestamp; + u8 maint_op_class; + u8 reserved[15]; +} __packed; + +#define CXL_EVENT_RECORD_DATA_LENGTH 0x50 +struct cxl_event_generic { + struct cxl_event_record_hdr hdr; + u8 data[CXL_EVENT_RECORD_DATA_LENGTH]; +} __packed; + +/* + * General Media Event Record + * CXL rev 3.0 Section 8.2.9.2.1.1; Table 8-43 + */ +#define CXL_EVENT_GEN_MED_COMP_ID_SIZE 0x10 +struct cxl_event_gen_media { + struct cxl_event_record_hdr hdr; + __le64 phys_addr; + u8 descriptor; + u8 type; + u8 transaction_type; + u8 validity_flags[2]; + u8 channel; + u8 rank; + u8 device[3]; + u8 component_id[CXL_EVENT_GEN_MED_COMP_ID_SIZE]; + u8 reserved[46]; +} __packed; + +/* + * DRAM Event Record - DER + * CXL rev 3.0 section 8.2.9.2.1.2; Table 3-44 + */ +#define CXL_EVENT_DER_CORRECTION_MASK_SIZE 0x20 +struct cxl_event_dram { + struct cxl_event_record_hdr hdr; + __le64 phys_addr; + u8 descriptor; + u8 type; + u8 transaction_type; + u8 validity_flags[2]; + u8 channel; + u8 rank; + u8 nibble_mask[3]; + u8 bank_group; + u8 bank; + u8 row[3]; + u8 column[2]; + u8 correction_mask[CXL_EVENT_DER_CORRECTION_MASK_SIZE]; + u8 reserved[0x17]; +} __packed; + +/* + * Get Health Info Record + * CXL rev 3.0 section 8.2.9.8.3.1; Table 8-100 + */ +struct cxl_get_health_info { + u8 health_status; + u8 media_status; + u8 add_status; + u8 life_used; + u8 device_temp[2]; + u8 dirty_shutdown_cnt[4]; + u8 cor_vol_err_cnt[4]; + u8 cor_per_err_cnt[4]; +} __packed; + +/* + * Memory Module Event Record + * CXL rev 3.0 section 8.2.9.2.1.3; Table 8-45 + */ +struct cxl_event_mem_module { + struct cxl_event_record_hdr hdr; + u8 event_type; + struct cxl_get_health_info info; + u8 reserved[0x3d]; +} __packed; + +union cxl_event { + struct cxl_event_generic generic; + struct cxl_event_gen_media gen_media; + struct cxl_event_dram dram; + struct cxl_event_mem_module mem_module; +} __packed; + +/* + * Common Event Record Format; in event logs + * CXL rev 3.0 section 8.2.9.2.1; Table 8-42 + */ +struct cxl_event_record_raw { + uuid_t id; + union cxl_event event; +} __packed; + +enum cxl_event_type { + CXL_CPER_EVENT_GENERIC, + CXL_CPER_EVENT_GEN_MEDIA, + CXL_CPER_EVENT_DRAM, + CXL_CPER_EVENT_MEM_MODULE, +}; + +#define CPER_CXL_DEVICE_ID_VALID BIT(0) +#define CPER_CXL_DEVICE_SN_VALID BIT(1) +#define CPER_CXL_COMP_EVENT_LOG_VALID BIT(2) +struct cxl_cper_event_rec { + struct { + u32 length; + u64 validation_bits; + struct cper_cxl_event_devid { + u16 vendor_id; + u16 device_id; + u8 func_num; + u8 device_num; + u8 bus_num; + u16 segment_num; + u16 slot_num; /* bits 2:0 reserved */ + u8 reserved; + } __packed device_id; + struct cper_cxl_event_sn { + u32 lower_dw; + u32 upper_dw; + } __packed dev_serial_num; + } __packed hdr; + + union cxl_event event; +} __packed; + +struct cxl_cper_work_data { + enum cxl_event_type event_type; + struct cxl_cper_event_rec rec; +}; + +#ifdef CONFIG_ACPI_APEI_GHES +int cxl_cper_register_work(struct work_struct *work); +int cxl_cper_unregister_work(struct work_struct *work); +int cxl_cper_kfifo_get(struct cxl_cper_work_data *wd); +#else +static inline int cxl_cper_register_work(struct work_struct *work) +{ + return 0; +} + +static inline int cxl_cper_unregister_work(struct work_struct *work) +{ + return 0; +} +static inline int cxl_cper_kfifo_get(struct cxl_cper_work_data *wd) +{ + return 0; +} +#endif + +#endif /* _LINUX_CXL_EVENT_H */ diff --git a/include/linux/einj-cxl.h b/include/linux/einj-cxl.h new file mode 100644 index 0000000000000..624ff6ff41f96 --- /dev/null +++ b/include/linux/einj-cxl.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * CXL protocol Error INJection support. + * + * Copyright (c) 2023 Advanced Micro Devices, Inc. + * All Rights Reserved. + * + * Author: Ben Cheatham + */ +#ifndef EINJ_CXL_H +#define EINJ_CXL_H + +#include +#include + +struct pci_dev; +struct seq_file; + +#if IS_ENABLED(CONFIG_ACPI_APEI_EINJ_CXL) +int einj_cxl_available_error_type_show(struct seq_file *m, void *v); +int einj_cxl_inject_error(struct pci_dev *dport_dev, u64 type); +int einj_cxl_inject_rch_error(u64 rcrb, u64 type); +bool einj_cxl_is_initialized(void); +#else /* !IS_ENABLED(CONFIG_ACPI_APEI_EINJ_CXL) */ +static inline int einj_cxl_available_error_type_show(struct seq_file *m, + void *v) +{ + return -ENXIO; +} + +static inline int einj_cxl_inject_error(struct pci_dev *dport_dev, u64 type) +{ + return -ENXIO; +} + +static inline int einj_cxl_inject_rch_error(u64 rcrb, u64 type) +{ + return -ENXIO; +} + +static inline bool einj_cxl_is_initialized(void) { return false; } +#endif /* CONFIG_ACPI_APEI_EINJ_CXL */ + +#endif /* EINJ_CXL_H */ diff --git a/include/linux/fw_table.h b/include/linux/fw_table.h new file mode 100644 index 0000000000000..3ff4c277296fb --- /dev/null +++ b/include/linux/fw_table.h @@ -0,0 +1,61 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * fw_tables.h - Parsing support for ACPI and ACPI-like tables provided by + * platform or device firmware + * + * Copyright (C) 2001 Paul Diefenbaugh + * Copyright (C) 2023 Intel Corp. + */ +#ifndef _FW_TABLE_H_ +#define _FW_TABLE_H_ + +union acpi_subtable_headers; + +typedef int (*acpi_tbl_entry_handler)(union acpi_subtable_headers *header, + const unsigned long end); + +typedef int (*acpi_tbl_entry_handler_arg)(union acpi_subtable_headers *header, + void *arg, const unsigned long end); + +struct acpi_subtable_proc { + int id; + acpi_tbl_entry_handler handler; + acpi_tbl_entry_handler_arg handler_arg; + void *arg; + int count; +}; + +union fw_table_header { + struct acpi_table_header acpi; + struct acpi_table_cdat cdat; +}; + +union acpi_subtable_headers { + struct acpi_subtable_header common; + struct acpi_hmat_structure hmat; + struct acpi_prmt_module_header prmt; + struct acpi_cedt_header cedt; + struct acpi_cdat_header cdat; +}; + +int acpi_parse_entries_array(char *id, unsigned long table_size, + union fw_table_header *table_header, + unsigned long max_length, + struct acpi_subtable_proc *proc, + int proc_num, unsigned int max_entries); + +int cdat_table_parse(enum acpi_cdat_type type, + acpi_tbl_entry_handler_arg handler_arg, void *arg, + struct acpi_table_cdat *table_header, + unsigned long length); + +/* CXL is the only non-ACPI consumer of the FIRMWARE_TABLE library */ +#if IS_ENABLED(CONFIG_ACPI) && !IS_ENABLED(CONFIG_CXL_BUS) +#define EXPORT_SYMBOL_FWTBL_LIB(x) EXPORT_SYMBOL_ACPI_LIB(x) +#define __init_or_fwtbl_lib __init_or_acpilib +#else +#define EXPORT_SYMBOL_FWTBL_LIB(x) EXPORT_SYMBOL_NS_GPL(x, CXL) +#define __init_or_fwtbl_lib +#endif + +#endif diff --git a/include/linux/memory.h b/include/linux/memory.h index f53cfdaaaa416..d5385592a03b8 100644 --- a/include/linux/memory.h +++ b/include/linux/memory.h @@ -113,7 +113,8 @@ struct mem_section; */ #define DEFAULT_CALLBACK_PRI 0 #define SLAB_CALLBACK_PRI 1 -#define HMAT_CALLBACK_PRI 2 +#define CXL_CALLBACK_PRI 5 +#define HMAT_CALLBACK_PRI 6 #define MM_COMPUTE_BATCH_PRI 10 #define CPUSET_CALLBACK_PRI 10 #define MEMTIER_HOTPLUG_PRI 100 diff --git a/include/linux/node.h b/include/linux/node.h index dfc004e4bee74..c1d60295cb61c 100644 --- a/include/linux/node.h +++ b/include/linux/node.h @@ -79,6 +79,8 @@ struct node_cache_attrs { void node_add_cache(unsigned int nid, struct node_cache_attrs *cache_attrs); void node_set_perf_attrs(unsigned int nid, struct access_coordinate *coord, enum access_coordinate_class access); +void node_update_perf_attrs(unsigned int nid, struct access_coordinate *coord, + enum access_coordinate_class access); #else static inline void node_add_cache(unsigned int nid, struct node_cache_attrs *cache_attrs) @@ -90,6 +92,12 @@ static inline void node_set_perf_attrs(unsigned int nid, enum access_coordinate_class access) { } + +static inline void node_update_perf_attrs(unsigned int nid, + struct access_coordinate *coord, + enum access_coordinate_class access) +{ +} #endif struct node { diff --git a/include/linux/pci.h b/include/linux/pci.h index 2f48988278077..d23851dedd138 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -1367,6 +1367,7 @@ int pcie_set_mps(struct pci_dev *dev, int mps); u32 pcie_bandwidth_available(struct pci_dev *dev, struct pci_dev **limiting_dev, enum pci_bus_speed *speed, enum pcie_link_width *width); +int pcie_link_speed_mbps(struct pci_dev *pdev); void pcie_print_link_status(struct pci_dev *dev); int pcie_reset_flr(struct pci_dev *dev, bool probe); int pcie_flr(struct pci_dev *dev); diff --git a/include/uapi/linux/cxl_mem.h b/include/uapi/linux/cxl_mem.h index 14bc6e7421483..42066f4eb8903 100644 --- a/include/uapi/linux/cxl_mem.h +++ b/include/uapi/linux/cxl_mem.h @@ -46,6 +46,7 @@ ___C(GET_SCAN_MEDIA_CAPS, "Get Scan Media Capabilities"), \ ___DEPRECATED(SCAN_MEDIA, "Scan Media"), \ ___DEPRECATED(GET_SCAN_MEDIA, "Get Scan Media Results"), \ + ___C(GET_TIMESTAMP, "Get Timestamp"), \ ___C(MAX, "invalid / last command") #define ___C(a, b) CXL_MEM_COMMAND_ID_##a diff --git a/lib/Kconfig b/lib/Kconfig index 43d69669465af..b27a54257d1bc 100644 --- a/lib/Kconfig +++ b/lib/Kconfig @@ -759,3 +759,6 @@ config ASN1_ENCODER config POLYNOMIAL tristate + +config FIRMWARE_TABLE + bool diff --git a/lib/Makefile b/lib/Makefile index b9d2577fbbe19..9b2d631aaf6fb 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -407,6 +407,8 @@ CFLAGS_longest_symbol_kunit.o += $(call cc-disable-warning, missing-prototypes) obj-$(CONFIG_GENERIC_LIB_DEVMEM_IS_ALLOWED) += devmem_is_allowed.o +obj-$(CONFIG_FIRMWARE_TABLE) += fw_table.o + # FORTIFY_SOURCE compile-time behavior tests TEST_FORTIFY_SRCS = $(wildcard $(srctree)/$(src)/test_fortify/*-*.c) TEST_FORTIFY_LOGS = $(patsubst $(srctree)/$(src)/%.c, %.log, $(TEST_FORTIFY_SRCS)) diff --git a/lib/fw_table.c b/lib/fw_table.c new file mode 100644 index 0000000000000..ddb67853b7ac1 --- /dev/null +++ b/lib/fw_table.c @@ -0,0 +1,244 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * fw_tables.c - Parsing support for ACPI and ACPI-like tables provided by + * platform or device firmware + * + * Copyright (C) 2001 Paul Diefenbaugh + * Copyright (C) 2023 Intel Corp. + */ +#include +#include +#include +#include +#include +#include +#include + +enum acpi_subtable_type { + ACPI_SUBTABLE_COMMON, + ACPI_SUBTABLE_HMAT, + ACPI_SUBTABLE_PRMT, + ACPI_SUBTABLE_CEDT, + CDAT_SUBTABLE, +}; + +struct acpi_subtable_entry { + union acpi_subtable_headers *hdr; + enum acpi_subtable_type type; +}; + +static unsigned long __init_or_fwtbl_lib +acpi_get_entry_type(struct acpi_subtable_entry *entry) +{ + switch (entry->type) { + case ACPI_SUBTABLE_COMMON: + return entry->hdr->common.type; + case ACPI_SUBTABLE_HMAT: + return entry->hdr->hmat.type; + case ACPI_SUBTABLE_PRMT: + return 0; + case ACPI_SUBTABLE_CEDT: + return entry->hdr->cedt.type; + case CDAT_SUBTABLE: + return entry->hdr->cdat.type; + } + return 0; +} + +static unsigned long __init_or_fwtbl_lib +acpi_get_entry_length(struct acpi_subtable_entry *entry) +{ + switch (entry->type) { + case ACPI_SUBTABLE_COMMON: + return entry->hdr->common.length; + case ACPI_SUBTABLE_HMAT: + return entry->hdr->hmat.length; + case ACPI_SUBTABLE_PRMT: + return entry->hdr->prmt.length; + case ACPI_SUBTABLE_CEDT: + return entry->hdr->cedt.length; + case CDAT_SUBTABLE: { + __le16 length = (__force __le16)entry->hdr->cdat.length; + + return le16_to_cpu(length); + } + } + return 0; +} + +static unsigned long __init_or_fwtbl_lib +acpi_get_subtable_header_length(struct acpi_subtable_entry *entry) +{ + switch (entry->type) { + case ACPI_SUBTABLE_COMMON: + return sizeof(entry->hdr->common); + case ACPI_SUBTABLE_HMAT: + return sizeof(entry->hdr->hmat); + case ACPI_SUBTABLE_PRMT: + return sizeof(entry->hdr->prmt); + case ACPI_SUBTABLE_CEDT: + return sizeof(entry->hdr->cedt); + case CDAT_SUBTABLE: + return sizeof(entry->hdr->cdat); + } + return 0; +} + +static enum acpi_subtable_type __init_or_fwtbl_lib +acpi_get_subtable_type(char *id) +{ + if (strncmp(id, ACPI_SIG_HMAT, 4) == 0) + return ACPI_SUBTABLE_HMAT; + if (strncmp(id, ACPI_SIG_PRMT, 4) == 0) + return ACPI_SUBTABLE_PRMT; + if (strncmp(id, ACPI_SIG_CEDT, 4) == 0) + return ACPI_SUBTABLE_CEDT; + if (strncmp(id, ACPI_SIG_CDAT, 4) == 0) + return CDAT_SUBTABLE; + return ACPI_SUBTABLE_COMMON; +} + +static unsigned long __init_or_fwtbl_lib +acpi_table_get_length(enum acpi_subtable_type type, + union fw_table_header *header) +{ + if (type == CDAT_SUBTABLE) { + __le32 length = (__force __le32)header->cdat.length; + + return le32_to_cpu(length); + } + + return header->acpi.length; +} + +static __init_or_fwtbl_lib bool has_handler(struct acpi_subtable_proc *proc) +{ + return proc->handler || proc->handler_arg; +} + +static __init_or_fwtbl_lib int call_handler(struct acpi_subtable_proc *proc, + union acpi_subtable_headers *hdr, + unsigned long end) +{ + if (proc->handler) + return proc->handler(hdr, end); + if (proc->handler_arg) + return proc->handler_arg(hdr, proc->arg, end); + return -EINVAL; +} + +/** + * acpi_parse_entries_array - for each proc_num find a suitable subtable + * + * @id: table id (for debugging purposes) + * @table_size: size of the root table + * @max_length: maximum size of the table (ignore if 0) + * @table_header: where does the table start? + * @proc: array of acpi_subtable_proc struct containing entry id + * and associated handler with it + * @proc_num: how big proc is? + * @max_entries: how many entries can we process? + * + * For each proc_num find a subtable with proc->id and run proc->handler + * on it. Assumption is that there's only single handler for particular + * entry id. + * + * The table_size is not the size of the complete ACPI table (the length + * field in the header struct), but only the size of the root table; i.e., + * the offset from the very first byte of the complete ACPI table, to the + * first byte of the very first subtable. + * + * On success returns sum of all matching entries for all proc handlers. + * Otherwise, -ENODEV or -EINVAL is returned. + */ +int __init_or_fwtbl_lib +acpi_parse_entries_array(char *id, unsigned long table_size, + union fw_table_header *table_header, + unsigned long max_length, + struct acpi_subtable_proc *proc, + int proc_num, unsigned int max_entries) +{ + unsigned long table_len, table_end, subtable_len, entry_len; + struct acpi_subtable_entry entry; + enum acpi_subtable_type type; + int count = 0; + int errs = 0; + int i; + + type = acpi_get_subtable_type(id); + table_len = acpi_table_get_length(type, table_header); + if (max_length && max_length < table_len) + table_len = max_length; + table_end = (unsigned long)table_header + table_len; + + /* Parse all entries looking for a match. */ + + entry.type = type; + entry.hdr = (union acpi_subtable_headers *) + ((unsigned long)table_header + table_size); + subtable_len = acpi_get_subtable_header_length(&entry); + + while (((unsigned long)entry.hdr) + subtable_len < table_end) { + if (max_entries && count >= max_entries) + break; + + for (i = 0; i < proc_num; i++) { + if (acpi_get_entry_type(&entry) != proc[i].id) + continue; + if (!has_handler(&proc[i]) || + (!errs && + call_handler(&proc[i], entry.hdr, table_end))) { + errs++; + continue; + } + + proc[i].count++; + break; + } + if (i != proc_num) + count++; + + /* + * If entry->length is 0, break from this loop to avoid + * infinite loop. + */ + entry_len = acpi_get_entry_length(&entry); + if (entry_len == 0) { + pr_err("[%4.4s:0x%02x] Invalid zero length\n", id, proc->id); + return -EINVAL; + } + + entry.hdr = (union acpi_subtable_headers *) + ((unsigned long)entry.hdr + entry_len); + } + + if (max_entries && count > max_entries) { + pr_warn("[%4.4s:0x%02x] found the maximum %i entries\n", + id, proc->id, count); + } + + return errs ? -EINVAL : count; +} + +int __init_or_fwtbl_lib +cdat_table_parse(enum acpi_cdat_type type, + acpi_tbl_entry_handler_arg handler_arg, + void *arg, + struct acpi_table_cdat *table_header, + unsigned long length) +{ + struct acpi_subtable_proc proc = { + .id = type, + .handler_arg = handler_arg, + .arg = arg, + }; + + if (!table_header) + return -EINVAL; + + return acpi_parse_entries_array(ACPI_SIG_CDAT, + sizeof(struct acpi_table_cdat), + (union fw_table_header *)table_header, + length, &proc, 1, 0); +} +EXPORT_SYMBOL_FWTBL_LIB(cdat_table_parse); diff --git a/tools/testing/cxl/Kbuild b/tools/testing/cxl/Kbuild index 95dc58b94178b..3d1ca9e38b1fa 100644 --- a/tools/testing/cxl/Kbuild +++ b/tools/testing/cxl/Kbuild @@ -13,6 +13,8 @@ ldflags-y += --wrap=cxl_hdm_decode_init ldflags-y += --wrap=cxl_dvsec_rr_decode ldflags-y += --wrap=devm_cxl_add_rch_dport ldflags-y += --wrap=cxl_rcd_component_reg_phys +ldflags-y += --wrap=cxl_endpoint_parse_cdat +ldflags-y += --wrap=cxl_setup_parent_dport DRIVERS := ../../../drivers CXL_SRC := $(DRIVERS)/cxl @@ -58,10 +60,13 @@ cxl_core-y += $(CXL_CORE_SRC)/mbox.o cxl_core-y += $(CXL_CORE_SRC)/pci.o cxl_core-y += $(CXL_CORE_SRC)/hdm.o cxl_core-y += $(CXL_CORE_SRC)/pmu.o +cxl_core-y += $(CXL_CORE_SRC)/cdat.o cxl_core-$(CONFIG_TRACING) += $(CXL_CORE_SRC)/trace.o cxl_core-$(CONFIG_CXL_REGION) += $(CXL_CORE_SRC)/region.o cxl_core-y += config_check.o cxl_core-y += cxl_core_test.o cxl_core-y += cxl_core_exports.o +KBUILD_CFLAGS := $(filter-out -Wmissing-prototypes -Wmissing-declarations, $(KBUILD_CFLAGS)) + obj-m += test/ diff --git a/tools/testing/cxl/test/Kbuild b/tools/testing/cxl/test/Kbuild index 61d5f7bcddf9a..6b19278978561 100644 --- a/tools/testing/cxl/test/Kbuild +++ b/tools/testing/cxl/test/Kbuild @@ -8,3 +8,5 @@ obj-m += cxl_mock_mem.o cxl_test-y := cxl.o cxl_mock-y := mock.o cxl_mock_mem-y := mem.o + +KBUILD_CFLAGS := $(filter-out -Wmissing-prototypes -Wmissing-declarations, $(KBUILD_CFLAGS)) diff --git a/tools/testing/cxl/test/cxl.c b/tools/testing/cxl/test/cxl.c index ed1361c8b96c4..c5bbd89b31920 100644 --- a/tools/testing/cxl/test/cxl.c +++ b/tools/testing/cxl/test/cxl.c @@ -15,6 +15,8 @@ static int interleave_arithmetic; +#define FAKE_QTG_ID 42 + #define NR_CXL_HOST_BRIDGES 2 #define NR_CXL_SINGLE_HOST 1 #define NR_CXL_RCH 1 @@ -68,15 +70,19 @@ static struct acpi_device acpi0017_mock; static struct acpi_device host_bridge[NR_BRIDGES] = { [0] = { .handle = &host_bridge[0], + .pnp.unique_id = "0", }, [1] = { .handle = &host_bridge[1], + .pnp.unique_id = "1", }, [2] = { .handle = &host_bridge[2], + .pnp.unique_id = "2", }, [3] = { .handle = &host_bridge[3], + .pnp.unique_id = "3", }, }; @@ -205,7 +211,7 @@ static struct { .granularity = 4, .restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 | ACPI_CEDT_CFMWS_RESTRICT_VOLATILE, - .qtg_id = 0, + .qtg_id = FAKE_QTG_ID, .window_size = SZ_256M * 4UL, }, .target = { 0 }, @@ -220,7 +226,7 @@ static struct { .granularity = 4, .restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 | ACPI_CEDT_CFMWS_RESTRICT_VOLATILE, - .qtg_id = 1, + .qtg_id = FAKE_QTG_ID, .window_size = SZ_256M * 8UL, }, .target = { 0, 1, }, @@ -235,7 +241,7 @@ static struct { .granularity = 4, .restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 | ACPI_CEDT_CFMWS_RESTRICT_PMEM, - .qtg_id = 2, + .qtg_id = FAKE_QTG_ID, .window_size = SZ_256M * 4UL, }, .target = { 0 }, @@ -250,7 +256,7 @@ static struct { .granularity = 4, .restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 | ACPI_CEDT_CFMWS_RESTRICT_PMEM, - .qtg_id = 3, + .qtg_id = FAKE_QTG_ID, .window_size = SZ_256M * 8UL, }, .target = { 0, 1, }, @@ -265,7 +271,7 @@ static struct { .granularity = 4, .restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 | ACPI_CEDT_CFMWS_RESTRICT_PMEM, - .qtg_id = 4, + .qtg_id = FAKE_QTG_ID, .window_size = SZ_256M * 4UL, }, .target = { 2 }, @@ -280,7 +286,7 @@ static struct { .granularity = 4, .restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 | ACPI_CEDT_CFMWS_RESTRICT_VOLATILE, - .qtg_id = 5, + .qtg_id = FAKE_QTG_ID, .window_size = SZ_256M, }, .target = { 3 }, @@ -297,7 +303,7 @@ static struct { .granularity = 4, .restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 | ACPI_CEDT_CFMWS_RESTRICT_PMEM, - .qtg_id = 0, + .qtg_id = FAKE_QTG_ID, .window_size = SZ_256M * 8UL, }, .target = { 0, }, @@ -313,7 +319,7 @@ static struct { .granularity = 0, .restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 | ACPI_CEDT_CFMWS_RESTRICT_PMEM, - .qtg_id = 1, + .qtg_id = FAKE_QTG_ID, .window_size = SZ_256M * 8UL, }, .target = { 0, 1, }, @@ -329,7 +335,7 @@ static struct { .granularity = 0, .restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 | ACPI_CEDT_CFMWS_RESTRICT_PMEM, - .qtg_id = 0, + .qtg_id = FAKE_QTG_ID, .window_size = SZ_256M * 16UL, }, .target = { 0, 1, 0, 1, }, @@ -972,6 +978,57 @@ static int mock_cxl_port_enumerate_dports(struct cxl_port *port) return 0; } +/* + * Faking the cxl_dpa_perf for the memdev when appropriate. + */ +static void dpa_perf_setup(struct cxl_port *endpoint, struct range *range, + struct cxl_dpa_perf *dpa_perf) +{ + dpa_perf->qos_class = FAKE_QTG_ID; + dpa_perf->dpa_range = *range; + for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) { + dpa_perf->coord[i].read_latency = 500; + dpa_perf->coord[i].write_latency = 500; + dpa_perf->coord[i].read_bandwidth = 1000; + dpa_perf->coord[i].write_bandwidth = 1000; + } +} + +static void mock_cxl_endpoint_parse_cdat(struct cxl_port *port) +{ + struct cxl_root *cxl_root __free(put_cxl_root) = + find_cxl_root(port); + struct cxl_memdev *cxlmd = to_cxl_memdev(port->uport_dev); + struct cxl_dev_state *cxlds = cxlmd->cxlds; + struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); + struct access_coordinate ep_c[ACCESS_COORDINATE_MAX]; + struct range pmem_range = { + .start = cxlds->pmem_res.start, + .end = cxlds->pmem_res.end, + }; + struct range ram_range = { + .start = cxlds->ram_res.start, + .end = cxlds->ram_res.end, + }; + + if (!cxl_root) + return; + + if (range_len(&ram_range)) + dpa_perf_setup(port, &ram_range, &mds->ram_perf); + + if (range_len(&pmem_range)) + dpa_perf_setup(port, &pmem_range, &mds->pmem_perf); + + cxl_memdev_update_perf(cxlmd); + + /* + * This function is here to only test the topology iterator. It serves + * no other purpose. + */ + cxl_endpoint_get_perf_coordinates(port, ep_c); +} + static struct cxl_mock_ops cxl_mock_ops = { .is_mock_adev = is_mock_adev, .is_mock_bridge = is_mock_bridge, @@ -985,6 +1042,7 @@ static struct cxl_mock_ops cxl_mock_ops = { .devm_cxl_setup_hdm = mock_cxl_setup_hdm, .devm_cxl_add_passthrough_decoder = mock_cxl_add_passthrough_decoder, .devm_cxl_enumerate_decoders = mock_cxl_enumerate_decoders, + .cxl_endpoint_parse_cdat = mock_cxl_endpoint_parse_cdat, .list = LIST_HEAD_INIT(cxl_mock_ops.list), }; diff --git a/tools/testing/cxl/test/mem.c b/tools/testing/cxl/test/mem.c index 0ed100617d993..0d0ca63de8629 100644 --- a/tools/testing/cxl/test/mem.c +++ b/tools/testing/cxl/test/mem.c @@ -90,6 +90,12 @@ static struct cxl_cel_entry mock_cel[] = { .effect = cpu_to_le16(EFFECT(CONF_CHANGE_COLD_RESET) | EFFECT(CONF_CHANGE_IMMEDIATE)), }, + { + .opcode = cpu_to_le16(CXL_MBOX_OP_SANITIZE), + .effect = cpu_to_le16(EFFECT(DATA_CHANGE_IMMEDIATE) | + EFFECT(SECURITY_CHANGE_IMMEDIATE) | + EFFECT(BACKGROUND_OP)), + }, }; /* See CXL 2.0 Table 181 Get Health Info Output Payload */ @@ -134,7 +140,6 @@ struct mock_event_log { }; struct mock_event_store { - struct cxl_memdev_state *mds; struct mock_event_log mock_logs[CXL_EVENT_TYPE_MAX]; u32 ev_status; }; @@ -151,8 +156,10 @@ struct cxl_mockmem_data { int user_limit; int master_limit; struct mock_event_store mes; + struct cxl_memdev_state *mds; u8 event_buf[SZ_4K]; u64 timestamp; + unsigned long sanitize_timeout; }; static struct mock_event_log *event_find_log(struct device *dev, int log_type) @@ -245,7 +252,8 @@ static int mock_get_event(struct device *dev, struct cxl_mbox_cmd *cmd) for (i = 0; i < CXL_TEST_EVENT_CNT && !event_log_empty(log); i++) { memcpy(&pl->records[i], event_get_current(log), sizeof(pl->records[i])); - pl->records[i].hdr.handle = event_get_cur_event_handle(log); + pl->records[i].event.generic.hdr.handle = + event_get_cur_event_handle(log); log->cur_idx++; } @@ -327,91 +335,113 @@ static void cxl_mock_event_trigger(struct device *dev) event_reset_log(log); } - cxl_mem_get_event_records(mes->mds, mes->ev_status); + cxl_mem_get_event_records(mdata->mds, mes->ev_status); } struct cxl_event_record_raw maint_needed = { - .hdr = { - .id = UUID_INIT(0xBA5EBA11, 0xABCD, 0xEFEB, - 0xa5, 0x5a, 0xa5, 0x5a, 0xa5, 0xa5, 0x5a, 0xa5), - .length = sizeof(struct cxl_event_record_raw), - .flags[0] = CXL_EVENT_RECORD_FLAG_MAINT_NEEDED, - /* .handle = Set dynamically */ - .related_handle = cpu_to_le16(0xa5b6), + .id = UUID_INIT(0xBA5EBA11, 0xABCD, 0xEFEB, + 0xa5, 0x5a, 0xa5, 0x5a, 0xa5, 0xa5, 0x5a, 0xa5), + .event.generic = { + .hdr = { + .length = sizeof(struct cxl_event_record_raw), + .flags[0] = CXL_EVENT_RECORD_FLAG_MAINT_NEEDED, + /* .handle = Set dynamically */ + .related_handle = cpu_to_le16(0xa5b6), + }, + .data = { 0xDE, 0xAD, 0xBE, 0xEF }, }, - .data = { 0xDE, 0xAD, 0xBE, 0xEF }, }; struct cxl_event_record_raw hardware_replace = { - .hdr = { - .id = UUID_INIT(0xABCDEFEB, 0xBA11, 0xBA5E, - 0xa5, 0x5a, 0xa5, 0x5a, 0xa5, 0xa5, 0x5a, 0xa5), - .length = sizeof(struct cxl_event_record_raw), - .flags[0] = CXL_EVENT_RECORD_FLAG_HW_REPLACE, - /* .handle = Set dynamically */ - .related_handle = cpu_to_le16(0xb6a5), + .id = UUID_INIT(0xABCDEFEB, 0xBA11, 0xBA5E, + 0xa5, 0x5a, 0xa5, 0x5a, 0xa5, 0xa5, 0x5a, 0xa5), + .event.generic = { + .hdr = { + .length = sizeof(struct cxl_event_record_raw), + .flags[0] = CXL_EVENT_RECORD_FLAG_HW_REPLACE, + /* .handle = Set dynamically */ + .related_handle = cpu_to_le16(0xb6a5), + }, + .data = { 0xDE, 0xAD, 0xBE, 0xEF }, }, - .data = { 0xDE, 0xAD, 0xBE, 0xEF }, }; -struct cxl_event_gen_media gen_media = { - .hdr = { - .id = UUID_INIT(0xfbcd0a77, 0xc260, 0x417f, - 0x85, 0xa9, 0x08, 0x8b, 0x16, 0x21, 0xeb, 0xa6), - .length = sizeof(struct cxl_event_gen_media), - .flags[0] = CXL_EVENT_RECORD_FLAG_PERMANENT, - /* .handle = Set dynamically */ - .related_handle = cpu_to_le16(0), +struct cxl_test_gen_media { + uuid_t id; + struct cxl_event_gen_media rec; +} __packed; + +struct cxl_test_gen_media gen_media = { + .id = CXL_EVENT_GEN_MEDIA_UUID, + .rec = { + .hdr = { + .length = sizeof(struct cxl_test_gen_media), + .flags[0] = CXL_EVENT_RECORD_FLAG_PERMANENT, + /* .handle = Set dynamically */ + .related_handle = cpu_to_le16(0), + }, + .phys_addr = cpu_to_le64(0x2000), + .descriptor = CXL_GMER_EVT_DESC_UNCORECTABLE_EVENT, + .type = CXL_GMER_MEM_EVT_TYPE_DATA_PATH_ERROR, + .transaction_type = CXL_GMER_TRANS_HOST_WRITE, + /* .validity_flags = */ + .channel = 1, + .rank = 30 }, - .phys_addr = cpu_to_le64(0x2000), - .descriptor = CXL_GMER_EVT_DESC_UNCORECTABLE_EVENT, - .type = CXL_GMER_MEM_EVT_TYPE_DATA_PATH_ERROR, - .transaction_type = CXL_GMER_TRANS_HOST_WRITE, - /* .validity_flags = */ - .channel = 1, - .rank = 30 }; -struct cxl_event_dram dram = { - .hdr = { - .id = UUID_INIT(0x601dcbb3, 0x9c06, 0x4eab, - 0xb8, 0xaf, 0x4e, 0x9b, 0xfb, 0x5c, 0x96, 0x24), - .length = sizeof(struct cxl_event_dram), - .flags[0] = CXL_EVENT_RECORD_FLAG_PERF_DEGRADED, - /* .handle = Set dynamically */ - .related_handle = cpu_to_le16(0), +struct cxl_test_dram { + uuid_t id; + struct cxl_event_dram rec; +} __packed; + +struct cxl_test_dram dram = { + .id = CXL_EVENT_DRAM_UUID, + .rec = { + .hdr = { + .length = sizeof(struct cxl_test_dram), + .flags[0] = CXL_EVENT_RECORD_FLAG_PERF_DEGRADED, + /* .handle = Set dynamically */ + .related_handle = cpu_to_le16(0), + }, + .phys_addr = cpu_to_le64(0x8000), + .descriptor = CXL_GMER_EVT_DESC_THRESHOLD_EVENT, + .type = CXL_GMER_MEM_EVT_TYPE_INV_ADDR, + .transaction_type = CXL_GMER_TRANS_INTERNAL_MEDIA_SCRUB, + /* .validity_flags = */ + .channel = 1, + .bank_group = 5, + .bank = 2, + .column = {0xDE, 0xAD}, }, - .phys_addr = cpu_to_le64(0x8000), - .descriptor = CXL_GMER_EVT_DESC_THRESHOLD_EVENT, - .type = CXL_GMER_MEM_EVT_TYPE_INV_ADDR, - .transaction_type = CXL_GMER_TRANS_INTERNAL_MEDIA_SCRUB, - /* .validity_flags = */ - .channel = 1, - .bank_group = 5, - .bank = 2, - .column = {0xDE, 0xAD}, }; -struct cxl_event_mem_module mem_module = { - .hdr = { - .id = UUID_INIT(0xfe927475, 0xdd59, 0x4339, - 0xa5, 0x86, 0x79, 0xba, 0xb1, 0x13, 0xb7, 0x74), - .length = sizeof(struct cxl_event_mem_module), - /* .handle = Set dynamically */ - .related_handle = cpu_to_le16(0), +struct cxl_test_mem_module { + uuid_t id; + struct cxl_event_mem_module rec; +} __packed; + +struct cxl_test_mem_module mem_module = { + .id = CXL_EVENT_MEM_MODULE_UUID, + .rec = { + .hdr = { + .length = sizeof(struct cxl_test_mem_module), + /* .handle = Set dynamically */ + .related_handle = cpu_to_le16(0), + }, + .event_type = CXL_MMER_TEMP_CHANGE, + .info = { + .health_status = CXL_DHI_HS_PERFORMANCE_DEGRADED, + .media_status = CXL_DHI_MS_ALL_DATA_LOST, + .add_status = (CXL_DHI_AS_CRITICAL << 2) | + (CXL_DHI_AS_WARNING << 4) | + (CXL_DHI_AS_WARNING << 5), + .device_temp = { 0xDE, 0xAD}, + .dirty_shutdown_cnt = { 0xde, 0xad, 0xbe, 0xef }, + .cor_vol_err_cnt = { 0xde, 0xad, 0xbe, 0xef }, + .cor_per_err_cnt = { 0xde, 0xad, 0xbe, 0xef }, + } }, - .event_type = CXL_MMER_TEMP_CHANGE, - .info = { - .health_status = CXL_DHI_HS_PERFORMANCE_DEGRADED, - .media_status = CXL_DHI_MS_ALL_DATA_LOST, - .add_status = (CXL_DHI_AS_CRITICAL << 2) | - (CXL_DHI_AS_WARNING << 4) | - (CXL_DHI_AS_WARNING << 5), - .device_temp = { 0xDE, 0xAD}, - .dirty_shutdown_cnt = { 0xde, 0xad, 0xbe, 0xef }, - .cor_vol_err_cnt = { 0xde, 0xad, 0xbe, 0xef }, - .cor_per_err_cnt = { 0xde, 0xad, 0xbe, 0xef }, - } }; static int mock_set_timestamp(struct cxl_dev_state *cxlds, @@ -433,11 +463,11 @@ static int mock_set_timestamp(struct cxl_dev_state *cxlds, static void cxl_mock_add_event_logs(struct mock_event_store *mes) { put_unaligned_le16(CXL_GMER_VALID_CHANNEL | CXL_GMER_VALID_RANK, - &gen_media.validity_flags); + &gen_media.rec.validity_flags); put_unaligned_le16(CXL_DER_VALID_CHANNEL | CXL_DER_VALID_BANK_GROUP | CXL_DER_VALID_BANK | CXL_DER_VALID_COLUMN, - &dram.validity_flags); + &dram.rec.validity_flags); mes_add_event(mes, CXL_EVENT_TYPE_INFO, &maint_needed); mes_add_event(mes, CXL_EVENT_TYPE_INFO, @@ -568,9 +598,26 @@ static int mock_partition_info(struct cxl_mbox_cmd *cmd) return 0; } +void cxl_mockmem_sanitize_work(struct work_struct *work) +{ + struct cxl_memdev_state *mds = + container_of(work, typeof(*mds), security.poll_dwork.work); + + mutex_lock(&mds->mbox_mutex); + if (mds->security.sanitize_node) + sysfs_notify_dirent(mds->security.sanitize_node); + mds->security.sanitize_active = false; + mutex_unlock(&mds->mbox_mutex); + + dev_dbg(mds->cxlds.dev, "sanitize complete\n"); +} + static int mock_sanitize(struct cxl_mockmem_data *mdata, struct cxl_mbox_cmd *cmd) { + struct cxl_memdev_state *mds = mdata->mds; + int rc = 0; + if (cmd->size_in != 0) return -EINVAL; @@ -586,7 +633,16 @@ static int mock_sanitize(struct cxl_mockmem_data *mdata, return -ENXIO; } - return 0; /* assume less than 2 secs, no bg */ + mutex_lock(&mds->mbox_mutex); + if (schedule_delayed_work(&mds->security.poll_dwork, + msecs_to_jiffies(mdata->sanitize_timeout))) { + mds->security.sanitize_active = true; + dev_dbg(mds->cxlds.dev, "sanitize issued\n"); + } else + rc = -EBUSY; + mutex_unlock(&mds->mbox_mutex); + + return rc; } static int mock_secure_erase(struct cxl_mockmem_data *mdata, @@ -1238,6 +1294,7 @@ static int mock_transfer_fw(struct cxl_mockmem_data *mdata, } memcpy(fw + offset, transfer->data, length); + usleep_range(1500, 2000); return 0; } @@ -1416,16 +1473,16 @@ static int cxl_mock_mem_probe(struct platform_device *pdev) if (IS_ERR(mds)) return PTR_ERR(mds); + mdata->mds = mds; mds->mbox_send = cxl_mock_mbox_send; mds->payload_size = SZ_4K; mds->event.buf = (struct cxl_get_event_payload *) mdata->event_buf; + INIT_DELAYED_WORK(&mds->security.poll_dwork, cxl_mockmem_sanitize_work); cxlds = &mds->cxlds; cxlds->serial = pdev->id; - if (is_rcd(pdev)) { + if (is_rcd(pdev)) cxlds->rcd = true; - cxlds->component_reg_phys = CXL_RESOURCE_NONE; - } rc = cxl_enumerate_cmds(mds); if (rc) @@ -1448,7 +1505,6 @@ static int cxl_mock_mem_probe(struct platform_device *pdev) if (rc) return rc; - mdata->mes.mds = mds; cxl_mock_add_event_logs(&mdata->mes); cxlmd = devm_cxl_add_memdev(&pdev->dev, cxlds); @@ -1459,6 +1515,10 @@ static int cxl_mock_mem_probe(struct platform_device *pdev) if (rc) return rc; + rc = devm_cxl_sanitize_setup_notifier(&pdev->dev, cxlmd); + if (rc) + return rc; + cxl_mem_get_event_records(mds, CXLDEV_EVENT_STATUS_ALL); return 0; @@ -1527,10 +1587,38 @@ static ssize_t fw_buf_checksum_show(struct device *dev, static DEVICE_ATTR_RO(fw_buf_checksum); +static ssize_t sanitize_timeout_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct cxl_mockmem_data *mdata = dev_get_drvdata(dev); + + return sysfs_emit(buf, "%lu\n", mdata->sanitize_timeout); +} + +static ssize_t sanitize_timeout_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct cxl_mockmem_data *mdata = dev_get_drvdata(dev); + unsigned long val; + int rc; + + rc = kstrtoul(buf, 0, &val); + if (rc) + return rc; + + mdata->sanitize_timeout = val; + + return count; +} + +static DEVICE_ATTR_RW(sanitize_timeout); + static struct attribute *cxl_mock_mem_attrs[] = { &dev_attr_security_lock.attr, &dev_attr_event_trigger.attr, &dev_attr_fw_buf_checksum.attr, + &dev_attr_sanitize_timeout.attr, NULL }; ATTRIBUTE_GROUPS(cxl_mock_mem); diff --git a/tools/testing/cxl/test/mock.c b/tools/testing/cxl/test/mock.c index 1a61e68e30950..d619672faa497 100644 --- a/tools/testing/cxl/test/mock.c +++ b/tools/testing/cxl/test/mock.c @@ -285,6 +285,32 @@ resource_size_t __wrap_cxl_rcd_component_reg_phys(struct device *dev, } EXPORT_SYMBOL_NS_GPL(__wrap_cxl_rcd_component_reg_phys, CXL); +void __wrap_cxl_endpoint_parse_cdat(struct cxl_port *port) +{ + int index; + struct cxl_mock_ops *ops = get_cxl_mock_ops(&index); + struct cxl_memdev *cxlmd = to_cxl_memdev(port->uport_dev); + + if (ops && ops->is_mock_dev(cxlmd->dev.parent)) + ops->cxl_endpoint_parse_cdat(port); + else + cxl_endpoint_parse_cdat(port); + put_cxl_mock_ops(index); +} +EXPORT_SYMBOL_NS_GPL(__wrap_cxl_endpoint_parse_cdat, CXL); + +void __wrap_cxl_setup_parent_dport(struct device *host, struct cxl_dport *dport) +{ + int index; + struct cxl_mock_ops *ops = get_cxl_mock_ops(&index); + + if (!ops || !ops->is_mock_port(dport->dport_dev)) + cxl_setup_parent_dport(host, dport); + + put_cxl_mock_ops(index); +} +EXPORT_SYMBOL_NS_GPL(__wrap_cxl_setup_parent_dport, CXL); + MODULE_LICENSE("GPL v2"); MODULE_IMPORT_NS(ACPI); MODULE_IMPORT_NS(CXL); diff --git a/tools/testing/cxl/test/mock.h b/tools/testing/cxl/test/mock.h index a94223750346c..d1b0271d28220 100644 --- a/tools/testing/cxl/test/mock.h +++ b/tools/testing/cxl/test/mock.h @@ -25,6 +25,7 @@ struct cxl_mock_ops { int (*devm_cxl_add_passthrough_decoder)(struct cxl_port *port); int (*devm_cxl_enumerate_decoders)( struct cxl_hdm *hdm, struct cxl_endpoint_dvsec_info *info); + void (*cxl_endpoint_parse_cdat)(struct cxl_port *port); }; void register_cxl_mock_ops(struct cxl_mock_ops *ops); diff --git a/tools/testing/nvdimm/Kbuild b/tools/testing/nvdimm/Kbuild index 8153251ea389a..91a3627f301a7 100644 --- a/tools/testing/nvdimm/Kbuild +++ b/tools/testing/nvdimm/Kbuild @@ -82,4 +82,6 @@ libnvdimm-$(CONFIG_NVDIMM_KEYS) += $(NVDIMM_SRC)/security.o libnvdimm-y += libnvdimm_test.o libnvdimm-y += config_check.o +KBUILD_CFLAGS := $(filter-out -Wmissing-prototypes -Wmissing-declarations, $(KBUILD_CFLAGS)) + obj-m += test/