From 6130fb7a0ce49a1bca2edd8a85d70f1486ca38ea Mon Sep 17 00:00:00 2001 From: Namhyung Kim Date: Tue, 3 Dec 2024 10:04:40 -0800 Subject: [PATCH 01/34] perf/core: Export perf_exclude_event() commit 6057b90ecc84f232dd32a047a086a4c4c271765f upstream. While at it, rename the same function in s390 cpum_sf PMU. Signed-off-by: Namhyung Kim Signed-off-by: Ingo Molnar Tested-by: Ravi Bangoria Reviewed-by: Ravi Bangoria Acked-by: Thomas Richter Link: https://lore.kernel.org/r/20241203180441.1634709-2-namhyung@kernel.org Signed-off-by: suryasaimadhu Signed-off-by: PvsNarasimha --- arch/s390/kernel/perf_cpum_sf.c | 6 +++--- include/linux/perf_event.h | 6 ++++++ kernel/events/core.c | 3 +-- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/arch/s390/kernel/perf_cpum_sf.c b/arch/s390/kernel/perf_cpum_sf.c index e52c89739bc9a..928333a0a8cf1 100644 --- a/arch/s390/kernel/perf_cpum_sf.c +++ b/arch/s390/kernel/perf_cpum_sf.c @@ -1074,7 +1074,7 @@ static void cpumsf_pmu_disable(struct pmu *pmu) cpuhw->flags &= ~PMU_F_ENABLED; } -/* perf_exclude_event() - Filter event +/* perf_event_exclude() - Filter event * @event: The perf event * @regs: pt_regs structure * @sde_regs: Sample-data-entry (sde) regs structure @@ -1083,7 +1083,7 @@ static void cpumsf_pmu_disable(struct pmu *pmu) * * Return non-zero if the event shall be excluded. */ -static int perf_exclude_event(struct perf_event *event, struct pt_regs *regs, +static int perf_event_exclude(struct perf_event *event, struct pt_regs *regs, struct perf_sf_sde_regs *sde_regs) { if (event->attr.exclude_user && user_mode(regs)) @@ -1166,7 +1166,7 @@ static int perf_push_sample(struct perf_event *event, data.tid_entry.pid = basic->hpp & LPP_PID_MASK; overflow = 0; - if (perf_exclude_event(event, ®s, sde_regs)) + if (perf_event_exclude(event, ®s, sde_regs)) goto out; if (perf_event_overflow(event, &data, ®s)) { overflow = 1; diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index 6a1386279b5c9..cd23c64606f82 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -1668,6 +1668,8 @@ static inline int perf_allow_tracepoint(struct perf_event_attr *attr) return security_perf_event_open(attr, PERF_SECURITY_TRACEPOINT); } +extern int perf_exclude_event(struct perf_event *event, struct pt_regs *regs); + extern void perf_event_init(void); extern void perf_tp_event(u16 event_type, u64 count, void *record, int entry_size, struct pt_regs *regs, @@ -1838,6 +1840,10 @@ static inline u64 perf_event_pause(struct perf_event *event, bool reset) { return 0; } +static inline int perf_exclude_event(struct perf_event *event, struct pt_regs *regs) +{ + return 0; +} #endif #if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_INTEL) diff --git a/kernel/events/core.c b/kernel/events/core.c index 65128a44b78da..df7b7768b1ca8 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -9825,8 +9825,7 @@ static void perf_swevent_event(struct perf_event *event, u64 nr, perf_swevent_overflow(event, 0, data, regs); } -static int perf_exclude_event(struct perf_event *event, - struct pt_regs *regs) +int perf_exclude_event(struct perf_event *event, struct pt_regs *regs) { if (event->hw.state & PERF_HES_STOPPED) return 1; From d7d7d380b93243a6b729991dba8ca927ff4b98b3 Mon Sep 17 00:00:00 2001 From: Namhyung Kim Date: Tue, 3 Dec 2024 10:04:41 -0800 Subject: [PATCH 02/34] perf/x86: Relax privilege filter restriction on AMD IBS commit d29e744c71673a71da8f8522799ee02744cad6c9 upstream. While IBS is available for per-thread profiling, still regular users cannot open an event due to the default paranoid setting (2) which doesn't allow unprivileged users to get kernel samples. That means it needs to set exclude_kernel bit in the attribute but IBS driver would reject it since it has PERF_PMU_CAP_NO_EXCLUDE. This is not what we want and I've been getting requests to fix this issue. This should be done in the hardware, but until we get the HW fix we may allow exclude_{kernel,user,hv} in the attribute and silently drop the samples in the PMU IRQ handler. It won't guarantee the sampling frequency or even it'd miss some with fixed period too. Not ideal, but that'd still be helpful to regular users. To minimize the confusion, let's add 'swfilt' bit to attr.config2 which is exposed in the sysfs format directory so that users can figure out if the kernel support the privilege filters by software. $ perf record -e ibs_op/swfilt=1/u true This uses perf_exclude_event() which checks regs->cs. But it should be fine because set_linear_ip() also updates the CS according to the RIP provided by IBS. Signed-off-by: Namhyung Kim Signed-off-by: Ingo Molnar Tested-by: Ravi Bangoria Reviewed-by: Ravi Bangoria Link: https://lore.kernel.org/r/20241203180441.1634709-3-namhyung@kernel.org Signed-off-by: suryasaimadhu Signed-off-by: PvsNarasimha --- arch/x86/events/amd/ibs.c | 59 +++++++++++++++++++++++++++------------ 1 file changed, 41 insertions(+), 18 deletions(-) diff --git a/arch/x86/events/amd/ibs.c b/arch/x86/events/amd/ibs.c index 1022d1dcbe447..4c6b689082c13 100644 --- a/arch/x86/events/amd/ibs.c +++ b/arch/x86/events/amd/ibs.c @@ -31,6 +31,8 @@ static u32 ibs_caps; #define IBS_FETCH_CONFIG_MASK (IBS_FETCH_RAND_EN | IBS_FETCH_MAX_CNT) #define IBS_OP_CONFIG_MASK IBS_OP_MAX_CNT +/* attr.config2 */ +#define IBS_SW_FILTER_MASK 1 /* * IBS states: @@ -290,6 +292,16 @@ static int perf_ibs_init(struct perf_event *event) if (has_branch_stack(event)) return -EOPNOTSUPP; + /* handle exclude_{user,kernel} in the IRQ handler */ + if (event->attr.exclude_host || event->attr.exclude_guest || + event->attr.exclude_idle) + return -EINVAL; + + if (!(event->attr.config2 & IBS_SW_FILTER_MASK) && + (event->attr.exclude_kernel || event->attr.exclude_user || + event->attr.exclude_hv)) + return -EINVAL; + ret = validate_group(event); if (ret) return ret; @@ -559,24 +571,14 @@ static struct attribute *attrs_empty[] = { NULL, }; -static struct attribute_group empty_format_group = { - .name = "format", - .attrs = attrs_empty, -}; - static struct attribute_group empty_caps_group = { .name = "caps", .attrs = attrs_empty, }; -static const struct attribute_group *empty_attr_groups[] = { - &empty_format_group, - &empty_caps_group, - NULL, -}; - PMU_FORMAT_ATTR(rand_en, "config:57"); PMU_FORMAT_ATTR(cnt_ctl, "config:19"); +PMU_FORMAT_ATTR(swfilt, "config2:0"); PMU_EVENT_ATTR_STRING(l3missonly, fetch_l3missonly, "config:59"); PMU_EVENT_ATTR_STRING(l3missonly, op_l3missonly, "config:16"); PMU_EVENT_ATTR_STRING(zen4_ibs_extensions, zen4_ibs_extensions, "1"); @@ -587,8 +589,9 @@ zen4_ibs_extensions_is_visible(struct kobject *kobj, struct attribute *attr, int return ibs_caps & IBS_CAPS_ZEN4 ? attr->mode : 0; } -static struct attribute *rand_en_attrs[] = { +static struct attribute *fetch_attrs[] = { &format_attr_rand_en.attr, + &format_attr_swfilt.attr, NULL, }; @@ -602,9 +605,9 @@ static struct attribute *zen4_ibs_extensions_attrs[] = { NULL, }; -static struct attribute_group group_rand_en = { +static struct attribute_group group_fetch_formats = { .name = "format", - .attrs = rand_en_attrs, + .attrs = fetch_attrs, }; static struct attribute_group group_fetch_l3missonly = { @@ -620,7 +623,7 @@ static struct attribute_group group_zen4_ibs_extensions = { }; static const struct attribute_group *fetch_attr_groups[] = { - &group_rand_en, + &group_fetch_formats, &empty_caps_group, NULL, }; @@ -637,6 +640,11 @@ cnt_ctl_is_visible(struct kobject *kobj, struct attribute *attr, int i) return ibs_caps & IBS_CAPS_OPCNT ? attr->mode : 0; } +static struct attribute *op_attrs[] = { + &format_attr_swfilt.attr, + NULL, +}; + static struct attribute *cnt_ctl_attrs[] = { &format_attr_cnt_ctl.attr, NULL, @@ -647,6 +655,11 @@ static struct attribute *op_l3missonly_attrs[] = { NULL, }; +static struct attribute_group group_op_formats = { + .name = "format", + .attrs = op_attrs, +}; + static struct attribute_group group_cnt_ctl = { .name = "format", .attrs = cnt_ctl_attrs, @@ -659,6 +672,12 @@ static struct attribute_group group_op_l3missonly = { .is_visible = zen4_ibs_extensions_is_visible, }; +static const struct attribute_group *op_attr_groups[] = { + &group_op_formats, + &empty_caps_group, + NULL, +}; + static const struct attribute_group *op_attr_update[] = { &group_cnt_ctl, &group_op_l3missonly, @@ -676,7 +695,6 @@ static struct perf_ibs perf_ibs_fetch = { .start = perf_ibs_start, .stop = perf_ibs_stop, .read = perf_ibs_read, - .capabilities = PERF_PMU_CAP_NO_EXCLUDE, }, .msr = MSR_AMD64_IBSFETCHCTL, .config_mask = IBS_FETCH_CONFIG_MASK, @@ -700,7 +718,6 @@ static struct perf_ibs perf_ibs_op = { .start = perf_ibs_start, .stop = perf_ibs_stop, .read = perf_ibs_read, - .capabilities = PERF_PMU_CAP_NO_EXCLUDE, }, .msr = MSR_AMD64_IBSOPCTL, .config_mask = IBS_OP_CONFIG_MASK, @@ -1120,6 +1137,12 @@ static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs) regs.flags |= PERF_EFLAGS_EXACT; } + if ((event->attr.config2 & IBS_SW_FILTER_MASK) && + perf_exclude_event(event, ®s)) { + throttle = perf_event_account_interrupt(event); + goto out; + } + if (event->attr.sample_type & PERF_SAMPLE_RAW) { raw = (struct perf_raw_record){ .frag = { @@ -1237,7 +1260,7 @@ static __init int perf_ibs_op_init(void) if (ibs_caps & IBS_CAPS_ZEN4) perf_ibs_op.config_mask |= IBS_OP_L3MISSONLY; - perf_ibs_op.pmu.attr_groups = empty_attr_groups; + perf_ibs_op.pmu.attr_groups = op_attr_groups; perf_ibs_op.pmu.attr_update = op_attr_update; return perf_ibs_pmu_init(&perf_ibs_op, "ibs_op"); From f3688111f6e768727be478c00e08496a74037884 Mon Sep 17 00:00:00 2001 From: Ravi Bangoria Date: Wed, 15 Jan 2025 05:44:30 +0000 Subject: [PATCH 03/34] perf/amd/ibs: Remove IBS_{FETCH|OP}_CONFIG_MASK macros commit 003c0414318a1829a1a5b195ad81e8a7960c3f5d upstream. Definition of these macros are very simple and they are used at only one place. Get rid of unnecessary redirection. Signed-off-by: Ravi Bangoria Signed-off-by: Peter Zijlstra (Intel) Acked-by: Namhyung Kim Link: https://lkml.kernel.org/r/20250115054438.1021-2-ravi.bangoria@amd.com Signed-off-by: suryasaimadhu Signed-off-by: PvsNarasimha --- arch/x86/events/amd/ibs.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/arch/x86/events/amd/ibs.c b/arch/x86/events/amd/ibs.c index 4c6b689082c13..75f7108226c63 100644 --- a/arch/x86/events/amd/ibs.c +++ b/arch/x86/events/amd/ibs.c @@ -28,9 +28,6 @@ static u32 ibs_caps; #include #include -#define IBS_FETCH_CONFIG_MASK (IBS_FETCH_RAND_EN | IBS_FETCH_MAX_CNT) -#define IBS_OP_CONFIG_MASK IBS_OP_MAX_CNT - /* attr.config2 */ #define IBS_SW_FILTER_MASK 1 @@ -697,7 +694,7 @@ static struct perf_ibs perf_ibs_fetch = { .read = perf_ibs_read, }, .msr = MSR_AMD64_IBSFETCHCTL, - .config_mask = IBS_FETCH_CONFIG_MASK, + .config_mask = IBS_FETCH_MAX_CNT | IBS_FETCH_RAND_EN, .cnt_mask = IBS_FETCH_MAX_CNT, .enable_mask = IBS_FETCH_ENABLE, .valid_mask = IBS_FETCH_VAL, @@ -720,7 +717,7 @@ static struct perf_ibs perf_ibs_op = { .read = perf_ibs_read, }, .msr = MSR_AMD64_IBSOPCTL, - .config_mask = IBS_OP_CONFIG_MASK, + .config_mask = IBS_OP_MAX_CNT, .cnt_mask = IBS_OP_MAX_CNT | IBS_OP_CUR_CNT | IBS_OP_CUR_CNT_RAND, .enable_mask = IBS_OP_ENABLE, From 63c3a7a06c2cd3103770b7bfc123aa6624e8a96a Mon Sep 17 00:00:00 2001 From: Ravi Bangoria Date: Wed, 15 Jan 2025 05:44:31 +0000 Subject: [PATCH 04/34] perf/amd/ibs: Remove pointless sample period check commit 88c7bcad71c83f52f24108dedcecae0d18dbc627 upstream. Valid perf event sample period value for IBS PMUs (Fetch and Op both) is limited to multiple of 0x10. perf_ibs_init() has this check: if (!event->attr.sample_freq && hwc->sample_period & 0x0f) return -EINVAL; But it's broken since hwc->sample_period will always be 0 when event->attr.sample_freq is 0 (irrespective of event->attr.freq value.) One option to fix this is to change the condition: - if (!event->attr.sample_freq && hwc->sample_period & 0x0f) + if (!event->attr.freq && hwc->sample_period & 0x0f) However, that will break all userspace tools which have been using IBS event with sample_period not multiple of 0x10. Another option is to remove the condition altogether and mask lower nibble _silently_, same as what current code is inadvertently doing. I'm preferring this approach as it keeps the existing behavior. Signed-off-by: Ravi Bangoria Signed-off-by: Peter Zijlstra (Intel) Acked-by: Namhyung Kim Link: https://lkml.kernel.org/r/20250115054438.1021-3-ravi.bangoria@amd.com Signed-off-by: suryasaimadhu Signed-off-by: PvsNarasimha --- arch/x86/events/amd/ibs.c | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/arch/x86/events/amd/ibs.c b/arch/x86/events/amd/ibs.c index 75f7108226c63..d9c84f1d530f1 100644 --- a/arch/x86/events/amd/ibs.c +++ b/arch/x86/events/amd/ibs.c @@ -307,13 +307,8 @@ static int perf_ibs_init(struct perf_event *event) if (config & perf_ibs->cnt_mask) /* raw max_cnt may not be set */ return -EINVAL; - if (!event->attr.sample_freq && hwc->sample_period & 0x0f) - /* - * lower 4 bits can not be set in ibs max cnt, - * but allowing it in case we adjust the - * sample period to set a frequency. - */ - return -EINVAL; + + /* Silently mask off lower nibble. IBS hw mandates it. */ hwc->sample_period &= ~0x0FULL; if (!hwc->sample_period) hwc->sample_period = 0x10; From 0edf68e607e00aa2cd641f9a4e8e86f6b1d5d196 Mon Sep 17 00:00:00 2001 From: Ravi Bangoria Date: Wed, 15 Jan 2025 05:44:34 +0000 Subject: [PATCH 05/34] perf/amd/ibs: Don't allow freq mode event creation through ->config interface commit e1e7844ced88f9558a48579390a7d4eaac6a28eb upstream. Most perf_event_attr->config bits directly maps to IBS_{FETCH|OP}_CTL MSR. Since the sample period is programmed in these control registers, IBS PMU driver allows opening an IBS event by setting sample period value directly in perf_event_attr->config instead of using explicit perf_event_attr->sample_period interface. However, this logic is not applicable for freq mode events since the semantics of control register fields are applicable only to fixed sample period whereas the freq mode event adjusts sample period after each and every sample. Currently, IBS driver (unintentionally) allows creating freq mode event via ->config interface, which is semantically wrong as well as detrimental because it can be misused to bypass perf_event_max_sample_rate checks. Don't allow freq mode event creation through perf_event_attr->config interface. Signed-off-by: Ravi Bangoria Signed-off-by: Peter Zijlstra (Intel) Acked-by: Namhyung Kim Link: https://lkml.kernel.org/r/20250115054438.1021-6-ravi.bangoria@amd.com Signed-off-by: suryasaimadhu Signed-off-by: PvsNarasimha --- arch/x86/events/amd/ibs.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/x86/events/amd/ibs.c b/arch/x86/events/amd/ibs.c index d9c84f1d530f1..3e7ca1e2f25e3 100644 --- a/arch/x86/events/amd/ibs.c +++ b/arch/x86/events/amd/ibs.c @@ -315,6 +315,9 @@ static int perf_ibs_init(struct perf_event *event) } else { u64 period = 0; + if (event->attr.freq) + return -EINVAL; + if (perf_ibs == &perf_ibs_op) { period = (config & IBS_OP_MAX_CNT) << 4; if (ibs_caps & IBS_CAPS_OPCNTEXT) From 37d6ab5d8648a79e36ad693c3ae64f4a94c648a0 Mon Sep 17 00:00:00 2001 From: Ravi Bangoria Date: Wed, 15 Jan 2025 05:44:35 +0000 Subject: [PATCH 06/34] perf/amd/ibs: Add PMU specific minimum period commit b2fc7b282bf7c1253b01c8da84e894539a3e709d upstream. 0x10 is the minimum sample period for IBS Fetch and 0x90 for IBS Op. Current IBS PMU driver uses 0x10 for both the PMUs, which is incorrect. Fix it by adding PMU specific minimum period values in struct perf_ibs. Also, bail out opening a 'sample period mode' event if the user requested sample period is less than PMU supported minimum value. For a 'freq mode' event, start calibrating sample period from PMU specific minimum period. Signed-off-by: Ravi Bangoria Signed-off-by: Peter Zijlstra (Intel) Acked-by: Namhyung Kim Link: https://lkml.kernel.org/r/20250115054438.1021-7-ravi.bangoria@amd.com Signed-off-by: suryasaimadhu Signed-off-by: PvsNarasimha --- arch/x86/events/amd/ibs.c | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/arch/x86/events/amd/ibs.c b/arch/x86/events/amd/ibs.c index 3e7ca1e2f25e3..7b54b76d39f5d 100644 --- a/arch/x86/events/amd/ibs.c +++ b/arch/x86/events/amd/ibs.c @@ -86,6 +86,7 @@ struct perf_ibs { u64 cnt_mask; u64 enable_mask; u64 valid_mask; + u16 min_period; u64 max_period; unsigned long offset_mask[1]; int offset_max; @@ -308,10 +309,14 @@ static int perf_ibs_init(struct perf_event *event) /* raw max_cnt may not be set */ return -EINVAL; - /* Silently mask off lower nibble. IBS hw mandates it. */ - hwc->sample_period &= ~0x0FULL; - if (!hwc->sample_period) - hwc->sample_period = 0x10; + if (event->attr.freq) { + hwc->sample_period = perf_ibs->min_period; + } else { + /* Silently mask off lower nibble. IBS hw mandates it. */ + hwc->sample_period &= ~0x0FULL; + if (hwc->sample_period < perf_ibs->min_period) + return -EINVAL; + } } else { u64 period = 0; @@ -329,10 +334,10 @@ static int perf_ibs_init(struct perf_event *event) config &= ~perf_ibs->cnt_mask; event->attr.sample_period = period; hwc->sample_period = period; - } - if (!hwc->sample_period) - return -EINVAL; + if (hwc->sample_period < perf_ibs->min_period) + return -EINVAL; + } /* * If we modify hwc->sample_period, we also need to update @@ -353,7 +358,8 @@ static int perf_ibs_set_period(struct perf_ibs *perf_ibs, int overflow; /* ignore lower 4 bits in min count: */ - overflow = perf_event_set_period(hwc, 1<<4, perf_ibs->max_period, period); + overflow = perf_event_set_period(hwc, perf_ibs->min_period, + perf_ibs->max_period, period); local64_set(&hwc->prev_count, 0); return overflow; @@ -696,6 +702,7 @@ static struct perf_ibs perf_ibs_fetch = { .cnt_mask = IBS_FETCH_MAX_CNT, .enable_mask = IBS_FETCH_ENABLE, .valid_mask = IBS_FETCH_VAL, + .min_period = 0x10, .max_period = IBS_FETCH_MAX_CNT << 4, .offset_mask = { MSR_AMD64_IBSFETCH_REG_MASK }, .offset_max = MSR_AMD64_IBSFETCH_REG_COUNT, @@ -720,6 +727,7 @@ static struct perf_ibs perf_ibs_op = { IBS_OP_CUR_CNT_RAND, .enable_mask = IBS_OP_ENABLE, .valid_mask = IBS_OP_VAL, + .min_period = 0x90, .max_period = IBS_OP_MAX_CNT << 4, .offset_mask = { MSR_AMD64_IBSOP_REG_MASK }, .offset_max = MSR_AMD64_IBSOP_REG_COUNT, From da3027322262af0007a53c949551b098cc4f2581 Mon Sep 17 00:00:00 2001 From: Ravi Bangoria Date: Wed, 15 Jan 2025 05:44:36 +0000 Subject: [PATCH 07/34] perf/amd/ibs: Add ->check_period() callback commit 1afbdd970f50f2e0431fae26b25d4e54e561fa7f upstream. IBS Fetch and IBS Op PMUs have constraints on sample period. The sample period is verified at the time of opening an event but not at the ioctl() interface. Hence, a user can open an event with valid period but change it later with ioctl(). Add a ->check_period() callback to verify the period provided at ioctl() is also valid. Signed-off-by: Ravi Bangoria Signed-off-by: Peter Zijlstra (Intel) Acked-by: Namhyung Kim Link: https://lkml.kernel.org/r/20250115054438.1021-8-ravi.bangoria@amd.com Signed-off-by: suryasaimadhu Signed-off-by: PvsNarasimha --- arch/x86/events/amd/ibs.c | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/arch/x86/events/amd/ibs.c b/arch/x86/events/amd/ibs.c index 7b54b76d39f5d..aea893a971b6e 100644 --- a/arch/x86/events/amd/ibs.c +++ b/arch/x86/events/amd/ibs.c @@ -564,6 +564,28 @@ static void perf_ibs_del(struct perf_event *event, int flags) static void perf_ibs_read(struct perf_event *event) { } +static int perf_ibs_check_period(struct perf_event *event, u64 value) +{ + struct perf_ibs *perf_ibs; + u64 low_nibble; + + if (event->attr.freq) + return 0; + + perf_ibs = container_of(event->pmu, struct perf_ibs, pmu); + low_nibble = value & 0xFULL; + + /* + * This contradicts with perf_ibs_init() which allows sample period + * with lower nibble bits set but silently masks them off. Whereas + * this returns error. + */ + if (low_nibble || value < perf_ibs->min_period) + return -EINVAL; + + return 0; +} + /* * We need to initialize with empty group if all attributes in the * group are dynamic. @@ -696,6 +718,7 @@ static struct perf_ibs perf_ibs_fetch = { .start = perf_ibs_start, .stop = perf_ibs_stop, .read = perf_ibs_read, + .check_period = perf_ibs_check_period, }, .msr = MSR_AMD64_IBSFETCHCTL, .config_mask = IBS_FETCH_MAX_CNT | IBS_FETCH_RAND_EN, @@ -720,6 +743,7 @@ static struct perf_ibs perf_ibs_op = { .start = perf_ibs_start, .stop = perf_ibs_stop, .read = perf_ibs_read, + .check_period = perf_ibs_check_period, }, .msr = MSR_AMD64_IBSOPCTL, .config_mask = IBS_OP_MAX_CNT, From 54c65024195ce305dc25b9cca982a5fe16ec1cee Mon Sep 17 00:00:00 2001 From: Ravi Bangoria Date: Wed, 15 Jan 2025 05:44:37 +0000 Subject: [PATCH 08/34] perf/amd/ibs: Ceil sample_period to min_period commit fa5d0a824e3bbd1f793d962f9e012ab0a8ee11c5 upstream. The sample_period needs to be recalibrated after every sample to match the desired sampling freq for a 'freq mode event'. Since the next sample_period is calculated by generic kernel, PMU specific constraints are not (explicitly) reckoned. The sample_period value is programmed in a MaxCnt field of IBS PMUs, and the MaxCnt field has following constraints: 1) MaxCnt must be multiple of 0x10. Kernel keeps track of residual / over-counted period into period_left, which should take care of this constraint by programming MaxCnt with (sample_period & ~0xF) and adding remaining period into the next sample. 2) MaxCnt must be >= 0x10 for IBS Fetch PMU and >= 0x90 for IBS Op PMU. Currently, IBS PMU driver allows sample_period below min_period, which is an undefined HW behavior. Reset sample_period to min_period whenever it's less than that. Signed-off-by: Ravi Bangoria Signed-off-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/20250115054438.1021-9-ravi.bangoria@amd.com Signed-off-by: suryasaimadhu Signed-off-by: PvsNarasimha --- arch/x86/events/amd/ibs.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/arch/x86/events/amd/ibs.c b/arch/x86/events/amd/ibs.c index aea893a971b6e..7978d7910adc3 100644 --- a/arch/x86/events/amd/ibs.c +++ b/arch/x86/events/amd/ibs.c @@ -457,6 +457,9 @@ static void perf_ibs_start(struct perf_event *event, int flags) WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE)); hwc->state = 0; + if (event->attr.freq && hwc->sample_period < perf_ibs->min_period) + hwc->sample_period = perf_ibs->min_period; + perf_ibs_set_period(perf_ibs, hwc, &period); if (perf_ibs == &perf_ibs_op && (ibs_caps & IBS_CAPS_OPCNTEXT)) { config |= period & IBS_OP_MAX_CNT_EXT_MASK; @@ -1191,6 +1194,10 @@ static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs) perf_sample_save_callchain(&data, event, iregs); throttle = perf_event_overflow(event, &data, ®s); + + if (event->attr.freq && hwc->sample_period < perf_ibs->min_period) + hwc->sample_period = perf_ibs->min_period; + out: if (throttle) { perf_ibs_stop(event, 0); From c8927b0e8497a92e7b79125b2f9324d985272b8b Mon Sep 17 00:00:00 2001 From: Ravi Bangoria Date: Wed, 5 Feb 2025 06:05:41 +0000 Subject: [PATCH 09/34] perf/amd/ibs: Add support for OP Load Latency Filtering commit d20610c19b4a22bc69085b7eb7a02741d51de30e upstream. IBS Op PMU on Zen5 uarch added new Load Latency filtering capability. It's advertised by CPUID_Fn8000001B_EAX bit 12. When enabled, IBS HW will raise interrupt only for sample that had an IbsDcMissLat value greater than N cycles, where N is a programmable value defined as multiples of 128 (i.e. 128, 256, 384 etc.) from 128-2048 cycles. Similar to L3MissOnly, IBS HW internally drops the sample and restarts if the sample does not meet the filtering criteria. Add support for LdLat filtering in IBS Op PMU. Since hardware supports threshold in multiple of 128, add a software filter on top to support latency threshold with the granularity of 1 cycle between [128-2048]. Example usage: # perf record -a -e ibs_op/ldlat=128/ -- sleep 5 Signed-off-by: Ravi Bangoria Signed-off-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/20250205060547.1337-2-ravi.bangoria@amd.com Signed-off-by: suryasaimadhu Signed-off-by: PvsNarasimha --- arch/x86/events/amd/ibs.c | 93 ++++++++++++++++++++++++++++--- arch/x86/include/asm/amd-ibs.h | 3 +- arch/x86/include/asm/perf_event.h | 3 + 3 files changed, 90 insertions(+), 9 deletions(-) diff --git a/arch/x86/events/amd/ibs.c b/arch/x86/events/amd/ibs.c index 7978d7910adc3..85b29b3f305b2 100644 --- a/arch/x86/events/amd/ibs.c +++ b/arch/x86/events/amd/ibs.c @@ -268,6 +268,14 @@ static int validate_group(struct perf_event *event) return 0; } +static bool perf_ibs_ldlat_event(struct perf_ibs *perf_ibs, + struct perf_event *event) +{ + return perf_ibs == &perf_ibs_op && + (ibs_caps & IBS_CAPS_OPLDLAT) && + (event->attr.config1 & 0xFFF); +} + static int perf_ibs_init(struct perf_event *event) { struct hw_perf_event *hwc = &event->hw; @@ -339,6 +347,17 @@ static int perf_ibs_init(struct perf_event *event) return -EINVAL; } + if (perf_ibs_ldlat_event(perf_ibs, event)) { + u64 ldlat = event->attr.config1 & 0xFFF; + + if (ldlat < 128 || ldlat > 2048) + return -EINVAL; + ldlat >>= 7; + + config |= (ldlat - 1) << 59; + config |= IBS_OP_L3MISSONLY | IBS_OP_LDLAT_EN; + } + /* * If we modify hwc->sample_period, we also need to update * hwc->last_period and hwc->period_left. @@ -607,7 +626,9 @@ PMU_FORMAT_ATTR(cnt_ctl, "config:19"); PMU_FORMAT_ATTR(swfilt, "config2:0"); PMU_EVENT_ATTR_STRING(l3missonly, fetch_l3missonly, "config:59"); PMU_EVENT_ATTR_STRING(l3missonly, op_l3missonly, "config:16"); +PMU_EVENT_ATTR_STRING(ldlat, ibs_op_ldlat_format, "config1:0-11"); PMU_EVENT_ATTR_STRING(zen4_ibs_extensions, zen4_ibs_extensions, "1"); +PMU_EVENT_ATTR_STRING(ldlat, ibs_op_ldlat_cap, "1"); static umode_t zen4_ibs_extensions_is_visible(struct kobject *kobj, struct attribute *attr, int i) @@ -615,6 +636,12 @@ zen4_ibs_extensions_is_visible(struct kobject *kobj, struct attribute *attr, int return ibs_caps & IBS_CAPS_ZEN4 ? attr->mode : 0; } +static umode_t +ibs_op_ldlat_is_visible(struct kobject *kobj, struct attribute *attr, int i) +{ + return ibs_caps & IBS_CAPS_OPLDLAT ? attr->mode : 0; +} + static struct attribute *fetch_attrs[] = { &format_attr_rand_en.attr, &format_attr_swfilt.attr, @@ -631,6 +658,11 @@ static struct attribute *zen4_ibs_extensions_attrs[] = { NULL, }; +static struct attribute *ibs_op_ldlat_cap_attrs[] = { + &ibs_op_ldlat_cap.attr.attr, + NULL, +}; + static struct attribute_group group_fetch_formats = { .name = "format", .attrs = fetch_attrs, @@ -648,6 +680,12 @@ static struct attribute_group group_zen4_ibs_extensions = { .is_visible = zen4_ibs_extensions_is_visible, }; +static struct attribute_group group_ibs_op_ldlat_cap = { + .name = "caps", + .attrs = ibs_op_ldlat_cap_attrs, + .is_visible = ibs_op_ldlat_is_visible, +}; + static const struct attribute_group *fetch_attr_groups[] = { &group_fetch_formats, &empty_caps_group, @@ -686,6 +724,11 @@ static struct attribute_group group_op_formats = { .attrs = op_attrs, }; +static struct attribute *ibs_op_ldlat_format_attrs[] = { + &ibs_op_ldlat_format.attr.attr, + NULL, +}; + static struct attribute_group group_cnt_ctl = { .name = "format", .attrs = cnt_ctl_attrs, @@ -704,10 +747,18 @@ static const struct attribute_group *op_attr_groups[] = { NULL, }; +static struct attribute_group group_ibs_op_ldlat_format = { + .name = "format", + .attrs = ibs_op_ldlat_format_attrs, + .is_visible = ibs_op_ldlat_is_visible, +}; + static const struct attribute_group *op_attr_update[] = { &group_cnt_ctl, &group_op_l3missonly, &group_zen4_ibs_extensions, + &group_ibs_op_ldlat_cap, + &group_ibs_op_ldlat_format, NULL, }; @@ -1060,15 +1111,25 @@ static void perf_ibs_parse_ld_st_data(__u64 sample_type, } } -static int perf_ibs_get_offset_max(struct perf_ibs *perf_ibs, u64 sample_type, +static bool perf_ibs_is_mem_sample_type(struct perf_ibs *perf_ibs, + struct perf_event *event) +{ + u64 sample_type = event->attr.sample_type; + + return perf_ibs == &perf_ibs_op && + sample_type & (PERF_SAMPLE_DATA_SRC | + PERF_SAMPLE_WEIGHT_TYPE | + PERF_SAMPLE_ADDR | + PERF_SAMPLE_PHYS_ADDR); +} + +static int perf_ibs_get_offset_max(struct perf_ibs *perf_ibs, + struct perf_event *event, int check_rip) { - if (sample_type & PERF_SAMPLE_RAW || - (perf_ibs == &perf_ibs_op && - (sample_type & PERF_SAMPLE_DATA_SRC || - sample_type & PERF_SAMPLE_WEIGHT_TYPE || - sample_type & PERF_SAMPLE_ADDR || - sample_type & PERF_SAMPLE_PHYS_ADDR))) + if (event->attr.sample_type & PERF_SAMPLE_RAW || + perf_ibs_is_mem_sample_type(perf_ibs, event) || + perf_ibs_ldlat_event(perf_ibs, event)) return perf_ibs->offset_max; else if (check_rip) return 3; @@ -1123,7 +1184,7 @@ static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs) offset = 1; check_rip = (perf_ibs == &perf_ibs_op && (ibs_caps & IBS_CAPS_RIPINVALIDCHK)); - offset_max = perf_ibs_get_offset_max(perf_ibs, event->attr.sample_type, check_rip); + offset_max = perf_ibs_get_offset_max(perf_ibs, event, check_rip); do { rdmsrl(msr + offset, *buf++); @@ -1132,6 +1193,22 @@ static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs) perf_ibs->offset_max, offset + 1); } while (offset < offset_max); + + if (perf_ibs_ldlat_event(perf_ibs, event)) { + union ibs_op_data3 op_data3; + + op_data3.val = ibs_data.regs[ibs_op_msr_idx(MSR_AMD64_IBSOPDATA3)]; + /* + * Opening event is errored out if load latency threshold is + * outside of [128, 2048] range. Since the event has reached + * interrupt handler, we can safely assume the threshold is + * within [128, 2048] range. + */ + if (!op_data3.ld_op || !op_data3.dc_miss || + op_data3.dc_miss_lat <= (event->attr.config1 & 0xFFF)) + goto out; + } + /* * Read IbsBrTarget, IbsOpData4, and IbsExtdCtl separately * depending on their availability. diff --git a/arch/x86/include/asm/amd-ibs.h b/arch/x86/include/asm/amd-ibs.h index cb2a5e113daa7..77f3a589a99ac 100644 --- a/arch/x86/include/asm/amd-ibs.h +++ b/arch/x86/include/asm/amd-ibs.h @@ -64,7 +64,8 @@ union ibs_op_ctl { opmaxcnt_ext:7, /* 20-26: upper 7 bits of periodic op maximum count */ reserved0:5, /* 27-31: reserved */ opcurcnt:27, /* 32-58: periodic op counter current count */ - reserved1:5; /* 59-63: reserved */ + ldlat_thrsh:4, /* 59-62: Load Latency threshold */ + ldlat_en:1; /* 63: Load Latency enabled */ }; }; diff --git a/arch/x86/include/asm/perf_event.h b/arch/x86/include/asm/perf_event.h index 6404acdea23af..229e8a0c5e8ea 100644 --- a/arch/x86/include/asm/perf_event.h +++ b/arch/x86/include/asm/perf_event.h @@ -637,6 +637,7 @@ struct arch_pebs_cntr_header { #define IBS_CAPS_FETCHCTLEXTD (1U<<9) #define IBS_CAPS_OPDATA4 (1U<<10) #define IBS_CAPS_ZEN4 (1U<<11) +#define IBS_CAPS_OPLDLAT (1U<<12) #define IBS_CAPS_DEFAULT (IBS_CAPS_AVAIL \ | IBS_CAPS_FETCHSAM \ @@ -662,6 +663,8 @@ struct arch_pebs_cntr_header { * The lower 7 bits of the current count are random bits * preloaded by hardware and ignored in software */ +#define IBS_OP_LDLAT_EN (1ULL<<63) +#define IBS_OP_LDLAT_THRSH (0xFULL<<59) #define IBS_OP_CUR_CNT (0xFFF80ULL<<32) #define IBS_OP_CUR_CNT_RAND (0x0007FULL<<32) #define IBS_OP_CUR_CNT_EXT_MASK (0x7FULL<<52) From 60deadeee2d0e39096420c5a9c7ba7ef16b40336 Mon Sep 17 00:00:00 2001 From: Ravi Bangoria Date: Wed, 5 Feb 2025 06:05:42 +0000 Subject: [PATCH 10/34] perf/amd/ibs: Update DTLB/PageSize decode logic commit 0b347a4218da08b1eb400c259d193bff463dae87 upstream. IBS Op PMU on Zen5 reports DTLB and page size information differently compared to prior generation. The change is enumerated by CPUID_Fn8000001B_EAX[19]. IBS_OP_DATA3 Zen3/4 Zen5 ---------------------------------------------------------------- 19 IbsDcL2TlbHit1G Reserved ---------------------------------------------------------------- 6 IbsDcL2tlbHit2M Reserved ---------------------------------------------------------------- 5 IbsDcL1TlbHit1G PageSize: 4 IbsDcL1TlbHit2M 0 - 4K 1 - 2M 2 - 1G 3 - Reserved Valid only if IbsDcPhyAddrValid = 1 ---------------------------------------------------------------- 3 IbsDcL2TlbMiss IbsDcL2TlbMiss Valid only if IbsDcPhyAddrValid = 1 ---------------------------------------------------------------- 2 IbsDcL1tlbMiss IbsDcL1tlbMiss Valid only if IbsDcPhyAddrValid = 1 ---------------------------------------------------------------- o Currently, only bit 2 and 3 are interpreted by IBS NMI handler for PERF_SAMPLE_DATA_SRC. Add dependency on IbsDcPhyAddrValid for those bits. o Introduce new IBS Op PMU capability and expose it to userspace via PMU's sysfs directory. Signed-off-by: Ravi Bangoria Signed-off-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/20250205060547.1337-3-ravi.bangoria@amd.com Signed-off-by: suryasaimadhu Signed-off-by: PvsNarasimha --- arch/x86/events/amd/ibs.c | 23 +++++++++++++++++++++++ arch/x86/include/asm/perf_event.h | 1 + 2 files changed, 24 insertions(+) diff --git a/arch/x86/events/amd/ibs.c b/arch/x86/events/amd/ibs.c index 85b29b3f305b2..7b52b8e3a1851 100644 --- a/arch/x86/events/amd/ibs.c +++ b/arch/x86/events/amd/ibs.c @@ -629,6 +629,7 @@ PMU_EVENT_ATTR_STRING(l3missonly, op_l3missonly, "config:16"); PMU_EVENT_ATTR_STRING(ldlat, ibs_op_ldlat_format, "config1:0-11"); PMU_EVENT_ATTR_STRING(zen4_ibs_extensions, zen4_ibs_extensions, "1"); PMU_EVENT_ATTR_STRING(ldlat, ibs_op_ldlat_cap, "1"); +PMU_EVENT_ATTR_STRING(dtlb_pgsize, ibs_op_dtlb_pgsize_cap, "1"); static umode_t zen4_ibs_extensions_is_visible(struct kobject *kobj, struct attribute *attr, int i) @@ -642,6 +643,12 @@ ibs_op_ldlat_is_visible(struct kobject *kobj, struct attribute *attr, int i) return ibs_caps & IBS_CAPS_OPLDLAT ? attr->mode : 0; } +static umode_t +ibs_op_dtlb_pgsize_is_visible(struct kobject *kobj, struct attribute *attr, int i) +{ + return ibs_caps & IBS_CAPS_OPDTLBPGSIZE ? attr->mode : 0; +} + static struct attribute *fetch_attrs[] = { &format_attr_rand_en.attr, &format_attr_swfilt.attr, @@ -663,6 +670,11 @@ static struct attribute *ibs_op_ldlat_cap_attrs[] = { NULL, }; +static struct attribute *ibs_op_dtlb_pgsize_cap_attrs[] = { + &ibs_op_dtlb_pgsize_cap.attr.attr, + NULL, +}; + static struct attribute_group group_fetch_formats = { .name = "format", .attrs = fetch_attrs, @@ -686,6 +698,12 @@ static struct attribute_group group_ibs_op_ldlat_cap = { .is_visible = ibs_op_ldlat_is_visible, }; +static struct attribute_group group_ibs_op_dtlb_pgsize_cap = { + .name = "caps", + .attrs = ibs_op_dtlb_pgsize_cap_attrs, + .is_visible = ibs_op_dtlb_pgsize_is_visible, +}; + static const struct attribute_group *fetch_attr_groups[] = { &group_fetch_formats, &empty_caps_group, @@ -759,6 +777,7 @@ static const struct attribute_group *op_attr_update[] = { &group_zen4_ibs_extensions, &group_ibs_op_ldlat_cap, &group_ibs_op_ldlat_format, + &group_ibs_op_dtlb_pgsize_cap, NULL, }; @@ -1007,6 +1026,10 @@ static void perf_ibs_get_tlb_lvl(union ibs_op_data3 *op_data3, if (!op_data3->dc_lin_addr_valid) return; + if ((ibs_caps & IBS_CAPS_OPDTLBPGSIZE) && + !op_data3->dc_phy_addr_valid) + return; + if (!op_data3->dc_l1tlb_miss) { data_src->mem_dtlb = PERF_MEM_TLB_L1 | PERF_MEM_TLB_HIT; return; diff --git a/arch/x86/include/asm/perf_event.h b/arch/x86/include/asm/perf_event.h index 229e8a0c5e8ea..d517c62ff4c6e 100644 --- a/arch/x86/include/asm/perf_event.h +++ b/arch/x86/include/asm/perf_event.h @@ -638,6 +638,7 @@ struct arch_pebs_cntr_header { #define IBS_CAPS_OPDATA4 (1U<<10) #define IBS_CAPS_ZEN4 (1U<<11) #define IBS_CAPS_OPLDLAT (1U<<12) +#define IBS_CAPS_OPDTLBPGSIZE (1U<<19) #define IBS_CAPS_DEFAULT (IBS_CAPS_AVAIL \ | IBS_CAPS_FETCHSAM \ From 9fd4f01216b54cb51ed8656273ca070a18952999 Mon Sep 17 00:00:00 2001 From: Namhyung Kim Date: Mon, 17 Mar 2025 09:37:55 -0700 Subject: [PATCH 11/34] perf/x86: Check data address for IBS software filter commit 65a99264f5e5a2bcc8c905f7b2d633e8991672ac upstream. The IBS software filter is filtering kernel samples for regular users in the PMI handler. It checks the instruction address in the IBS register to determine if it was in kernel mode or not. But it turns out that it's possible to report a kernel data address even if the instruction address belongs to user-space. Matteo Rizzo found that when an instruction raises an exception, IBS can report some kernel data addresses like IDT while holding the faulting instruction's RIP. To prevent an information leak, it should double check if the data address in PERF_SAMPLE_DATA is in the kernel space as well. [ mingo: Clarified the changelog ] Suggested-by: Matteo Rizzo Signed-off-by: Namhyung Kim Signed-off-by: Ingo Molnar Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250317163755.1842589-1-namhyung@kernel.org Signed-off-by: suryasaimadhu Signed-off-by: PvsNarasimha --- arch/x86/events/amd/ibs.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/arch/x86/events/amd/ibs.c b/arch/x86/events/amd/ibs.c index 7b52b8e3a1851..fbe10b469e8b0 100644 --- a/arch/x86/events/amd/ibs.c +++ b/arch/x86/events/amd/ibs.c @@ -1267,8 +1267,13 @@ static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs) regs.flags |= PERF_EFLAGS_EXACT; } + if (perf_ibs == &perf_ibs_op) + perf_ibs_parse_ld_st_data(event->attr.sample_type, &ibs_data, &data); + if ((event->attr.config2 & IBS_SW_FILTER_MASK) && - perf_exclude_event(event, ®s)) { + (perf_exclude_event(event, ®s) || + ((data.sample_flags & PERF_SAMPLE_ADDR) && + event->attr.exclude_kernel && kernel_ip(data.addr)))) { throttle = perf_event_account_interrupt(event); goto out; } @@ -1283,9 +1288,6 @@ static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs) perf_sample_save_raw_data(&data, event, &raw); } - if (perf_ibs == &perf_ibs_op) - perf_ibs_parse_ld_st_data(event->attr.sample_type, &ibs_data, &data); - /* * rip recorded by IbsOpRip will not be consistent with rsp and rbp * recorded as part of interrupt regs. Thus we need to use rip from From b45981dde50ad168bda9a88caa15b1b3e2d14112 Mon Sep 17 00:00:00 2001 From: Namhyung Kim Date: Sat, 22 Mar 2025 08:13:01 +0100 Subject: [PATCH 12/34] perf/amd/ibs: Prevent leaking sensitive data to userspace commit 50a53b60e141d7e31368a87e222e4dd5597bd4ae upstream. Although IBS "swfilt" can prevent leaking samples with kernel RIP to the userspace, there are few subtle cases where a 'data' address and/or a 'branch target' address can fall under kernel address range although RIP is from userspace. Prevent leaking kernel 'data' addresses by discarding such samples when {exclude_kernel=1,swfilt=1}. IBS can now be invoked by unprivileged user with the introduction of "swfilt". However, this creates a loophole in the interface where an unprivileged user can get physical address of the userspace virtual addresses through IBS register raw dump (PERF_SAMPLE_RAW). Prevent this as well. This upstream commit fixed the most obvious leak: 65a99264f5e5 perf/x86: Check data address for IBS software filter Follow that up with a more complete fix. Fixes: d29e744c7167 ("perf/x86: Relax privilege filter restriction on AMD IBS") Suggested-by: Matteo Rizzo Co-developed-by: Ravi Bangoria Signed-off-by: Namhyung Kim Signed-off-by: Ravi Bangoria Signed-off-by: Ingo Molnar Cc: Linus Torvalds Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20250321161251.1033-1-ravi.bangoria@amd.com Signed-off-by: suryasaimadhu Signed-off-by: PvsNarasimha --- arch/x86/events/amd/ibs.c | 84 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 78 insertions(+), 6 deletions(-) diff --git a/arch/x86/events/amd/ibs.c b/arch/x86/events/amd/ibs.c index fbe10b469e8b0..66f9818650911 100644 --- a/arch/x86/events/amd/ibs.c +++ b/arch/x86/events/amd/ibs.c @@ -1054,6 +1054,8 @@ static void perf_ibs_get_mem_lock(union ibs_op_data3 *op_data3, data_src->mem_lock = PERF_MEM_LOCK_LOCKED; } +/* Be careful. Works only for contiguous MSRs. */ +#define ibs_fetch_msr_idx(msr) (msr - MSR_AMD64_IBSFETCHCTL) #define ibs_op_msr_idx(msr) (msr - MSR_AMD64_IBSOPCTL) static void perf_ibs_get_data_src(struct perf_ibs_data *ibs_data, @@ -1159,6 +1161,67 @@ static int perf_ibs_get_offset_max(struct perf_ibs *perf_ibs, return 1; } +static bool perf_ibs_is_kernel_data_addr(struct perf_event *event, + struct perf_ibs_data *ibs_data) +{ + u64 sample_type_mask = PERF_SAMPLE_ADDR | PERF_SAMPLE_RAW; + union ibs_op_data3 op_data3; + u64 dc_lin_addr; + + op_data3.val = ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSOPDATA3)]; + dc_lin_addr = ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSDCLINAD)]; + + return unlikely((event->attr.sample_type & sample_type_mask) && + op_data3.dc_lin_addr_valid && kernel_ip(dc_lin_addr)); +} + +static bool perf_ibs_is_kernel_br_target(struct perf_event *event, + struct perf_ibs_data *ibs_data, + int br_target_idx) +{ + union ibs_op_data op_data; + u64 br_target; + + op_data.val = ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSOPDATA)]; + br_target = ibs_data->regs[br_target_idx]; + + return unlikely((event->attr.sample_type & PERF_SAMPLE_RAW) && + op_data.op_brn_ret && kernel_ip(br_target)); +} + +static bool perf_ibs_swfilt_discard(struct perf_ibs *perf_ibs, struct perf_event *event, + struct pt_regs *regs, struct perf_ibs_data *ibs_data, + int br_target_idx) +{ + if (perf_exclude_event(event, regs)) + return true; + + if (perf_ibs != &perf_ibs_op || !event->attr.exclude_kernel) + return false; + + if (perf_ibs_is_kernel_data_addr(event, ibs_data)) + return true; + + if (br_target_idx != -1 && + perf_ibs_is_kernel_br_target(event, ibs_data, br_target_idx)) + return true; + + return false; +} + +static void perf_ibs_phyaddr_clear(struct perf_ibs *perf_ibs, + struct perf_ibs_data *ibs_data) +{ + if (perf_ibs == &perf_ibs_op) { + ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSOPDATA3)] &= ~(1ULL << 18); + ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSDCPHYSAD)] = 0; + return; + } + + ibs_data->regs[ibs_fetch_msr_idx(MSR_AMD64_IBSFETCHCTL)] &= ~(1ULL << 52); + ibs_data->regs[ibs_fetch_msr_idx(MSR_AMD64_IBSFETCHPHYSAD)] = 0; +} + static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs) { struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu); @@ -1171,6 +1234,7 @@ static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs) int offset, size, check_rip, offset_max, throttle = 0; unsigned int msr; u64 *buf, *config, period, new_config = 0; + int br_target_idx = -1; if (!test_bit(IBS_STARTED, pcpu->state)) { fail: @@ -1241,6 +1305,7 @@ static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs) if (perf_ibs == &perf_ibs_op) { if (ibs_caps & IBS_CAPS_BRNTRGT) { rdmsrl(MSR_AMD64_IBSBRTARGET, *buf++); + br_target_idx = size; size++; } if (ibs_caps & IBS_CAPS_OPDATA4) { @@ -1267,16 +1332,20 @@ static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs) regs.flags |= PERF_EFLAGS_EXACT; } - if (perf_ibs == &perf_ibs_op) - perf_ibs_parse_ld_st_data(event->attr.sample_type, &ibs_data, &data); - if ((event->attr.config2 & IBS_SW_FILTER_MASK) && - (perf_exclude_event(event, ®s) || - ((data.sample_flags & PERF_SAMPLE_ADDR) && - event->attr.exclude_kernel && kernel_ip(data.addr)))) { + perf_ibs_swfilt_discard(perf_ibs, event, ®s, &ibs_data, br_target_idx)) { throttle = perf_event_account_interrupt(event); goto out; } + /* + * Prevent leaking physical addresses to unprivileged users. Skip + * PERF_SAMPLE_PHYS_ADDR check since generic code prevents it for + * unprivileged users. + */ + if ((event->attr.sample_type & PERF_SAMPLE_RAW) && + perf_allow_kernel(&event->attr)) { + perf_ibs_phyaddr_clear(perf_ibs, &ibs_data); + } if (event->attr.sample_type & PERF_SAMPLE_RAW) { raw = (struct perf_raw_record){ @@ -1288,6 +1357,9 @@ static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs) perf_sample_save_raw_data(&data, event, &raw); } + if (perf_ibs == &perf_ibs_op) + perf_ibs_parse_ld_st_data(event->attr.sample_type, &ibs_data, &data); + /* * rip recorded by IbsOpRip will not be consistent with rsp and rbp * recorded as part of interrupt regs. Thus we need to use rip from From 145749273456890cfa619c67baed7e381f09d4b4 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Sun, 13 Apr 2025 10:41:39 +0200 Subject: [PATCH 13/34] x86/platform/amd: Move the header to commit 3846389c03a8518884f09056611619bd1461ffc7 upstream. Collect AMD specific platform header files in . Signed-off-by: Ingo Molnar Acked-by: Borislav Petkov (AMD) Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Mario Limonciello Link: https://lore.kernel.org/r/20250413084144.3746608-2-mingo@kernel.org Signed-off-by: suryasaimadhu Signed-off-by: PvsNarasimha --- arch/x86/events/amd/ibs.c | 2 +- arch/x86/include/asm/{amd-ibs.h => amd/ibs.h} | 0 tools/perf/check-headers.sh | 2 +- tools/perf/util/amd-sample-raw.c | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename arch/x86/include/asm/{amd-ibs.h => amd/ibs.h} (100%) diff --git a/arch/x86/events/amd/ibs.c b/arch/x86/events/amd/ibs.c index 66f9818650911..e074a1a0daa31 100644 --- a/arch/x86/events/amd/ibs.c +++ b/arch/x86/events/amd/ibs.c @@ -26,7 +26,7 @@ static u32 ibs_caps; #include #include -#include +#include /* attr.config2 */ #define IBS_SW_FILTER_MASK 1 diff --git a/arch/x86/include/asm/amd-ibs.h b/arch/x86/include/asm/amd/ibs.h similarity index 100% rename from arch/x86/include/asm/amd-ibs.h rename to arch/x86/include/asm/amd/ibs.h diff --git a/tools/perf/check-headers.sh b/tools/perf/check-headers.sh index e21caadda7c11..d62732aa15777 100755 --- a/tools/perf/check-headers.sh +++ b/tools/perf/check-headers.sh @@ -160,7 +160,7 @@ done # diff with extra ignore lines check arch/x86/lib/memcpy_64.S '-I "^EXPORT_SYMBOL" -I "^#include " -I"^SYM_FUNC_START\(_LOCAL\)*(memcpy_\(erms\|orig\))" -I"^#include "' check arch/x86/lib/memset_64.S '-I "^EXPORT_SYMBOL" -I "^#include " -I"^SYM_FUNC_START\(_LOCAL\)*(memset_\(erms\|orig\))"' -check arch/x86/include/asm/amd-ibs.h '-I "^#include [<\"]\(asm/\)*msr-index.h"' +check arch/x86/include/asm/amd/ibs.h '-I "^#include [<\"]\(asm/\)*msr-index.h"' check arch/arm64/include/asm/cputype.h '-I "^#include [<\"]\(asm/\)*sysreg.h"' check include/uapi/asm-generic/mman.h '-I "^#include <\(uapi/\)*asm-generic/mman-common\(-tools\)*.h>"' check include/uapi/linux/mman.h '-I "^#include <\(uapi/\)*asm/mman.h>"' diff --git a/tools/perf/util/amd-sample-raw.c b/tools/perf/util/amd-sample-raw.c index 9d0ce88e90e45..456ce64ad822c 100644 --- a/tools/perf/util/amd-sample-raw.c +++ b/tools/perf/util/amd-sample-raw.c @@ -9,7 +9,7 @@ #include #include -#include "../../arch/x86/include/asm/amd-ibs.h" +#include "../../arch/x86/include/asm/amd/ibs.h" #include "debug.h" #include "session.h" From 592d8bd19e95b73631bbf3b6ee83461894259e0f Mon Sep 17 00:00:00 2001 From: Ravi Bangoria Date: Mon, 16 Feb 2026 04:22:12 +0000 Subject: [PATCH 14/34] perf/amd/ibs: Account interrupt for discarded samples commit 01336b5559785a136de1cac49705f63a70a755bc upstream. Add interrupt throttling accounting for below cases: o IBS Op PMU: A software filter (in addition to the hardware filter) drops samples whose load latency is below the user-specified threshold. o IBS Fetch PMU: Samples discarded due to the zero-RIP erratum (#1197). Although these samples are discarded, the NMI cost is still incurred, so they should be counted for interrupt throttling. Fixes: 26db2e0c51fe83e1dd852c1321407835b481806e ("perf/x86/amd/ibs: Work around erratum #1197") Fixes: d20610c19b4a22bc69085b7eb7a02741d51de30e ("perf/amd/ibs: Add support for OP Load Latency Filtering") Signed-off-by: Ravi Bangoria Signed-off-by: Peter Zijlstra (Intel) Acked-by: Namhyung Kim Link: https://patch.msgid.link/20260216042216.1440-2-ravi.bangoria@amd.com Signed-off-by: suryasaimadhu Signed-off-by: PvsNarasimha --- arch/x86/events/amd/ibs.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/arch/x86/events/amd/ibs.c b/arch/x86/events/amd/ibs.c index e074a1a0daa31..2f8c4094acbe3 100644 --- a/arch/x86/events/amd/ibs.c +++ b/arch/x86/events/amd/ibs.c @@ -1292,8 +1292,10 @@ static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs) * within [128, 2048] range. */ if (!op_data3.ld_op || !op_data3.dc_miss || - op_data3.dc_miss_lat <= (event->attr.config1 & 0xFFF)) + op_data3.dc_miss_lat <= (event->attr.config1 & 0xFFF)) { + throttle = perf_event_account_interrupt(event); goto out; + } } /* @@ -1325,8 +1327,10 @@ static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs) regs.flags &= ~PERF_EFLAGS_EXACT; } else { /* Workaround for erratum #1197 */ - if (perf_ibs->fetch_ignore_if_zero_rip && !(ibs_data.regs[1])) + if (perf_ibs->fetch_ignore_if_zero_rip && !(ibs_data.regs[1])) { + throttle = perf_event_account_interrupt(event); goto out; + } set_linear_ip(®s, ibs_data.regs[1]); regs.flags |= PERF_EFLAGS_EXACT; From 62622b9ca45562938f4593e2b506299c56bc6ff8 Mon Sep 17 00:00:00 2001 From: Ravi Bangoria Date: Mon, 16 Feb 2026 04:22:14 +0000 Subject: [PATCH 15/34] perf/amd/ibs: Preserve PhyAddrVal bit when clearing PhyAddr MSR commit 723a290326e015b07931eabc603d3735999377be upstream. Commit 50a53b60e141 ("perf/amd/ibs: Prevent leaking sensitive data to userspace") zeroed the physical address and also cleared the PhyAddrVal flag before copying the value into a perf sample to avoid exposing physical addresses to unprivileged users. Clearing PhyAddrVal, however, has an unintended side-effect: several other IBS fields are considered valid only when this bit is set. As a result, those otherwise correct fields are discarded, reducing IBS functionality. Continue to zero the physical address, but keep the PhyAddrVal bit intact so the related fields remain usable while still preventing any address leak. Fixes: 50a53b60e141 ("perf/amd/ibs: Prevent leaking sensitive data to userspace") Signed-off-by: Ravi Bangoria Signed-off-by: Peter Zijlstra (Intel) Acked-by: Namhyung Kim Link: https://patch.msgid.link/20260216042216.1440-4-ravi.bangoria@amd.com Signed-off-by: suryasaimadhu Signed-off-by: PvsNarasimha --- arch/x86/events/amd/ibs.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/arch/x86/events/amd/ibs.c b/arch/x86/events/amd/ibs.c index 2f8c4094acbe3..f71da587c69a4 100644 --- a/arch/x86/events/amd/ibs.c +++ b/arch/x86/events/amd/ibs.c @@ -1213,12 +1213,10 @@ static void perf_ibs_phyaddr_clear(struct perf_ibs *perf_ibs, struct perf_ibs_data *ibs_data) { if (perf_ibs == &perf_ibs_op) { - ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSOPDATA3)] &= ~(1ULL << 18); ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSDCPHYSAD)] = 0; return; } - ibs_data->regs[ibs_fetch_msr_idx(MSR_AMD64_IBSFETCHCTL)] &= ~(1ULL << 52); ibs_data->regs[ibs_fetch_msr_idx(MSR_AMD64_IBSFETCHPHYSAD)] = 0; } From 0948b3ec5a1bbec49d61c6dd69f8dc34617c5710 Mon Sep 17 00:00:00 2001 From: Ravi Bangoria Date: Mon, 16 Feb 2026 04:22:15 +0000 Subject: [PATCH 16/34] perf/amd/ibs: Avoid calling perf_allow_kernel() from the IBS NMI handler commit b0a09142622a994c4f4088c3f61db5da87cfc711 upstream. Calling perf_allow_kernel() from the NMI context is unsafe and could be fatal. Capture the permission at event-initialization time by storing it in event->hw.flags, and have the NMI handler rely on that cached flag instead of making the call directly. [Backport Changes] In file arch/x86/events/amd/ibs.c, within the function perf_ibs_init(), the upstream code adds conditional block with perf_allow_kernel(). However, in our codebase perf_allow_kernel() is implemented as perf_allow_kernel(&event->attr). Therefore the call was adjusted accordingly during the backport. Fixes: 50a53b60e141d ("perf/amd/ibs: Prevent leaking sensitive data to userspace") Reported-by: Sadasivan Shaiju Signed-off-by: Ravi Bangoria Signed-off-by: Peter Zijlstra (Intel) Acked-by: Namhyung Kim Link: https://patch.msgid.link/20260216042216.1440-5-ravi.bangoria@amd.com Signed-off-by: suryasaimadhu Signed-off-by: PvsNarasimha --- arch/x86/events/amd/ibs.c | 5 ++++- arch/x86/events/perf_event_flags.h | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/arch/x86/events/amd/ibs.c b/arch/x86/events/amd/ibs.c index f71da587c69a4..3ab137a2577b9 100644 --- a/arch/x86/events/amd/ibs.c +++ b/arch/x86/events/amd/ibs.c @@ -312,6 +312,9 @@ static int perf_ibs_init(struct perf_event *event) if (ret) return ret; + if (perf_allow_kernel(&event->attr)) + hwc->flags |= PERF_X86_EVENT_UNPRIVILEGED; + if (hwc->sample_period) { if (config & perf_ibs->cnt_mask) /* raw max_cnt may not be set */ @@ -1345,7 +1348,7 @@ static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs) * unprivileged users. */ if ((event->attr.sample_type & PERF_SAMPLE_RAW) && - perf_allow_kernel(&event->attr)) { + (hwc->flags & PERF_X86_EVENT_UNPRIVILEGED)) { perf_ibs_phyaddr_clear(perf_ibs, &ibs_data); } diff --git a/arch/x86/events/perf_event_flags.h b/arch/x86/events/perf_event_flags.h index 70078334e4a33..47f84ee8f5409 100644 --- a/arch/x86/events/perf_event_flags.h +++ b/arch/x86/events/perf_event_flags.h @@ -23,3 +23,4 @@ PERF_ARCH(PEBS_LAT_HYBRID, 0x0020000) /* ld and st lat for hybrid */ PERF_ARCH(NEEDS_BRANCH_STACK, 0x0040000) /* require branch stack setup */ PERF_ARCH(BRANCH_COUNTERS, 0x0080000) /* logs the counters in the extra space of each branch */ PERF_ARCH(ACR, 0x0100000) /* Auto counter reload */ +PERF_ARCH(UNPRIVILEGED, 0x0200000) /* Unprivileged event (wrt perf_allow_kernel()) */ From 0108cfc3ec44744183223f7d24c36e4d762d181b Mon Sep 17 00:00:00 2001 From: Ravi Bangoria Date: Mon, 16 Feb 2026 04:22:16 +0000 Subject: [PATCH 17/34] perf/amd/ibs: Avoid race between event add and NMI commit 1b044ff3c17e9d7fd93ffc0ba541ccdeb992d7f5 upstream. Consider the following race: -------- o OP_CTL contains stale value: OP_CTL[Val]=1, OP_CTL[En]=0 o A new IBS OP event is being added o [P]: Process context, [N]: NMI context [P] perf_ibs_add(event) { [P] if (test_and_set_bit(IBS_ENABLED, pcpu->state)) [P] return; [P] /* pcpu->state = IBS_ENABLED */ [P] [P] pcpu->event = event; [P] [P] perf_ibs_start(event) { [P] set_bit(IBS_STARTED, pcpu->state); [P] /* pcpu->state = IBS_ENABLED | IBS_STARTED */ [P] clear_bit(IBS_STOPPING, pcpu->state); [P] /* pcpu->state = IBS_ENABLED | IBS_STARTED */ [N] --> NMI due to genuine FETCH event. perf_ibs_handle_irq() [N] called for OP PMU as well. [N] [N] perf_ibs_handle_irq(perf_ibs) { [N] event = pcpu->event; /* See line 6 */ [N] [N] if (!test_bit(IBS_STARTED, pcpu->state)) /* false */ [N] return 0; [N] [N] if (WARN_ON_ONCE(!event)) /* false */ [N] goto fail; [N] [N] if (!(*buf++ & perf_ibs->valid_mask)) /* false due to stale [N] * IBS_OP_CTL value */ [N] goto fail; [N] [N] ... [N] [N] perf_ibs_enable_event() // *Accidentally* enable the event. [N] } [N] [N] /* [N] * Repeated NMIs may follow due to accidentally enabled IBS OP [N] * event if the sample period is very low. It could also lead [N] * to pcpu->state corruption if the event gets throttled due [N] * to too frequent NMIs. [N] */ [P] perf_ibs_enable_event(); [P] } [P] } -------- We cannot safely clear IBS_{FETCH|OP}_CTL while disabling the event, because the register might be read again later. So, clear the register in the enable path - before we update pcpu->state and enable the event. This guarantees that any NMI that lands in the gap finds Val=0 and bails out cleanly. Signed-off-by: Ravi Bangoria Signed-off-by: Peter Zijlstra (Intel) Acked-by: Namhyung Kim Link: https://patch.msgid.link/20260216042216.1440-6-ravi.bangoria@amd.com Signed-off-by: suryasaimadhu Signed-off-by: PvsNarasimha --- arch/x86/events/amd/ibs.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/arch/x86/events/amd/ibs.c b/arch/x86/events/amd/ibs.c index 3ab137a2577b9..42ddca247e0b8 100644 --- a/arch/x86/events/amd/ibs.c +++ b/arch/x86/events/amd/ibs.c @@ -489,6 +489,14 @@ static void perf_ibs_start(struct perf_event *event, int flags) } config |= period >> 4; + /* + * Reset the IBS_{FETCH|OP}_CTL MSR before updating pcpu->state. + * Doing so prevents a race condition in which an NMI due to other + * source might accidentally activate the event before we enable + * it ourselves. + */ + perf_ibs_disable_event(perf_ibs, hwc, 0); + /* * Set STARTED before enabling the hardware, such that a subsequent NMI * must observe it. From 7544604e0af128cc1bc687046c9060aa2cbeaf13 Mon Sep 17 00:00:00 2001 From: Ravi Bangoria Date: Mon, 16 Feb 2026 04:25:24 +0000 Subject: [PATCH 18/34] perf/amd/ibs: Define macro for ldlat mask and shift commit f9d55ccf0199d1a80c2519084578f0c345dedd2f upstream. Load latency filter threshold is encoded in config1[11:0]. Define a mask for it instead of hardcoded 0xFFF. Unlike "config" fields whose layout maps to PERF_{FETCH|OP}_CTL MSR, layout of "config1" is custom defined so a new set of macros are needed for "config1" fields. Signed-off-by: Ravi Bangoria Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Dapeng Mi Link: https://patch.msgid.link/20260216042530.1546-2-ravi.bangoria@amd.com Signed-off-by: suryasaimadhu Signed-off-by: PvsNarasimha --- arch/x86/events/amd/ibs.c | 11 +++++++---- arch/x86/include/asm/perf_event.h | 1 + 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/arch/x86/events/amd/ibs.c b/arch/x86/events/amd/ibs.c index 42ddca247e0b8..6ec32395d6662 100644 --- a/arch/x86/events/amd/ibs.c +++ b/arch/x86/events/amd/ibs.c @@ -31,6 +31,9 @@ static u32 ibs_caps; /* attr.config2 */ #define IBS_SW_FILTER_MASK 1 +/* attr.config1 */ +#define IBS_OP_CONFIG1_LDLAT_MASK (0xFFFULL << 0) + /* * IBS states: * @@ -273,7 +276,7 @@ static bool perf_ibs_ldlat_event(struct perf_ibs *perf_ibs, { return perf_ibs == &perf_ibs_op && (ibs_caps & IBS_CAPS_OPLDLAT) && - (event->attr.config1 & 0xFFF); + (event->attr.config1 & IBS_OP_CONFIG1_LDLAT_MASK); } static int perf_ibs_init(struct perf_event *event) @@ -351,13 +354,13 @@ static int perf_ibs_init(struct perf_event *event) } if (perf_ibs_ldlat_event(perf_ibs, event)) { - u64 ldlat = event->attr.config1 & 0xFFF; + u64 ldlat = event->attr.config1 & IBS_OP_CONFIG1_LDLAT_MASK; if (ldlat < 128 || ldlat > 2048) return -EINVAL; ldlat >>= 7; - config |= (ldlat - 1) << 59; + config |= (ldlat - 1) << IBS_OP_LDLAT_THRSH_SHIFT; config |= IBS_OP_L3MISSONLY | IBS_OP_LDLAT_EN; } @@ -1301,7 +1304,7 @@ static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs) * within [128, 2048] range. */ if (!op_data3.ld_op || !op_data3.dc_miss || - op_data3.dc_miss_lat <= (event->attr.config1 & 0xFFF)) { + op_data3.dc_miss_lat <= (event->attr.config1 & IBS_OP_CONFIG1_LDLAT_MASK)) { throttle = perf_event_account_interrupt(event); goto out; } diff --git a/arch/x86/include/asm/perf_event.h b/arch/x86/include/asm/perf_event.h index d517c62ff4c6e..cb5d3e8be9f57 100644 --- a/arch/x86/include/asm/perf_event.h +++ b/arch/x86/include/asm/perf_event.h @@ -666,6 +666,7 @@ struct arch_pebs_cntr_header { */ #define IBS_OP_LDLAT_EN (1ULL<<63) #define IBS_OP_LDLAT_THRSH (0xFULL<<59) +#define IBS_OP_LDLAT_THRSH_SHIFT (59) #define IBS_OP_CUR_CNT (0xFFF80ULL<<32) #define IBS_OP_CUR_CNT_RAND (0x0007FULL<<32) #define IBS_OP_CUR_CNT_EXT_MASK (0x7FULL<<52) From 2dc4dea11b542b972186a036f561256036ead4e9 Mon Sep 17 00:00:00 2001 From: Ravi Bangoria Date: Mon, 16 Feb 2026 04:25:25 +0000 Subject: [PATCH 19/34] perf/amd/ibs: Add new MSRs and CPUID bits definitions commit e267b4178134e36e83ddfe4f7f5b4b162a286148 upstream. IBS on upcoming microarch introduced two new control MSRs and couple of new features. Define macros for them. New capabilities: o IBS_CAPS_DIS: Alternate Fetch and Op IBS disable bits o IBS_CAPS_FETCHLAT: Fetch Latency filter o IBS_CAPS_BIT63_FILTER: Virtual address bit 63 based filters for Fetch and Op o IBS_CAPS_STRMST_RMTSOCKET: Streaming store filter and indicator, remote socket indicator New control MSRs for above features: o MSR_AMD64_IBSFETCHCTL2 o MSR_AMD64_IBSOPCTL2 Also do cosmetic alignment changes. Signed-off-by: Ravi Bangoria Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Dapeng Mi Link: https://patch.msgid.link/20260216042530.1546-3-ravi.bangoria@amd.com Signed-off-by: suryasaimadhu Signed-off-by: PvsNarasimha --- arch/x86/include/asm/msr-index.h | 2 ++ arch/x86/include/asm/perf_event.h | 56 ++++++++++++++++++++----------- 2 files changed, 38 insertions(+), 20 deletions(-) diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h index 78bf5d4bd9f5b..41e606243a795 100644 --- a/arch/x86/include/asm/msr-index.h +++ b/arch/x86/include/asm/msr-index.h @@ -652,6 +652,8 @@ #define MSR_AMD64_IBSBRTARGET 0xc001103b #define MSR_AMD64_ICIBSEXTDCTL 0xc001103c #define MSR_AMD64_IBSOPDATA4 0xc001103d +#define MSR_AMD64_IBSOPCTL2 0xc001103e +#define MSR_AMD64_IBSFETCHCTL2 0xc001103f #define MSR_AMD64_IBS_REG_COUNT_MAX 8 /* includes MSR_AMD64_IBSBRTARGET */ #define MSR_AMD64_SVM_AVIC_DOORBELL 0xc001011b #define MSR_AMD64_VM_PAGE_FLUSH 0xc001011e diff --git a/arch/x86/include/asm/perf_event.h b/arch/x86/include/asm/perf_event.h index cb5d3e8be9f57..900bae70d5182 100644 --- a/arch/x86/include/asm/perf_event.h +++ b/arch/x86/include/asm/perf_event.h @@ -638,6 +638,10 @@ struct arch_pebs_cntr_header { #define IBS_CAPS_OPDATA4 (1U<<10) #define IBS_CAPS_ZEN4 (1U<<11) #define IBS_CAPS_OPLDLAT (1U<<12) +#define IBS_CAPS_DIS (1U<<13) +#define IBS_CAPS_FETCHLAT (1U<<14) +#define IBS_CAPS_BIT63_FILTER (1U<<15) +#define IBS_CAPS_STRMST_RMTSOCKET (1U<<16) #define IBS_CAPS_OPDTLBPGSIZE (1U<<19) #define IBS_CAPS_DEFAULT (IBS_CAPS_AVAIL \ @@ -652,32 +656,44 @@ struct arch_pebs_cntr_header { #define IBSCTL_LVT_OFFSET_MASK 0x0F /* IBS fetch bits/masks */ -#define IBS_FETCH_L3MISSONLY (1ULL<<59) -#define IBS_FETCH_RAND_EN (1ULL<<57) -#define IBS_FETCH_VAL (1ULL<<49) -#define IBS_FETCH_ENABLE (1ULL<<48) -#define IBS_FETCH_CNT 0xFFFF0000ULL -#define IBS_FETCH_MAX_CNT 0x0000FFFFULL +#define IBS_FETCH_L3MISSONLY (1ULL << 59) +#define IBS_FETCH_RAND_EN (1ULL << 57) +#define IBS_FETCH_VAL (1ULL << 49) +#define IBS_FETCH_ENABLE (1ULL << 48) +#define IBS_FETCH_CNT 0xFFFF0000ULL +#define IBS_FETCH_MAX_CNT 0x0000FFFFULL + +#define IBS_FETCH_2_DIS (1ULL << 0) +#define IBS_FETCH_2_FETCHLAT_FILTER (0xFULL << 1) +#define IBS_FETCH_2_FETCHLAT_FILTER_SHIFT (1) +#define IBS_FETCH_2_EXCL_RIP_63_EQ_1 (1ULL << 5) +#define IBS_FETCH_2_EXCL_RIP_63_EQ_0 (1ULL << 6) /* * IBS op bits/masks * The lower 7 bits of the current count are random bits * preloaded by hardware and ignored in software */ -#define IBS_OP_LDLAT_EN (1ULL<<63) -#define IBS_OP_LDLAT_THRSH (0xFULL<<59) -#define IBS_OP_LDLAT_THRSH_SHIFT (59) -#define IBS_OP_CUR_CNT (0xFFF80ULL<<32) -#define IBS_OP_CUR_CNT_RAND (0x0007FULL<<32) -#define IBS_OP_CUR_CNT_EXT_MASK (0x7FULL<<52) -#define IBS_OP_CNT_CTL (1ULL<<19) -#define IBS_OP_VAL (1ULL<<18) -#define IBS_OP_ENABLE (1ULL<<17) -#define IBS_OP_L3MISSONLY (1ULL<<16) -#define IBS_OP_MAX_CNT 0x0000FFFFULL -#define IBS_OP_MAX_CNT_EXT 0x007FFFFFULL /* not a register bit mask */ -#define IBS_OP_MAX_CNT_EXT_MASK (0x7FULL<<20) /* separate upper 7 bits */ -#define IBS_RIP_INVALID (1ULL<<38) +#define IBS_OP_LDLAT_EN (1ULL << 63) +#define IBS_OP_LDLAT_THRSH (0xFULL << 59) +#define IBS_OP_LDLAT_THRSH_SHIFT (59) +#define IBS_OP_CUR_CNT (0xFFF80ULL << 32) +#define IBS_OP_CUR_CNT_RAND (0x0007FULL << 32) +#define IBS_OP_CUR_CNT_EXT_MASK (0x7FULL << 52) +#define IBS_OP_CNT_CTL (1ULL << 19) +#define IBS_OP_VAL (1ULL << 18) +#define IBS_OP_ENABLE (1ULL << 17) +#define IBS_OP_L3MISSONLY (1ULL << 16) +#define IBS_OP_MAX_CNT 0x0000FFFFULL +#define IBS_OP_MAX_CNT_EXT 0x007FFFFFULL /* not a register bit mask */ +#define IBS_OP_MAX_CNT_EXT_MASK (0x7FULL << 20) /* separate upper 7 bits */ +#define IBS_RIP_INVALID (1ULL << 38) + +#define IBS_OP_2_DIS (1ULL << 0) +#define IBS_OP_2_EXCL_RIP_63_EQ_0 (1ULL << 1) +#define IBS_OP_2_EXCL_RIP_63_EQ_1 (1ULL << 2) +#define IBS_OP_2_STRM_ST_FILTER (1ULL << 3) +#define IBS_OP_2_STRM_ST_FILTER_SHIFT (3) #ifdef CONFIG_X86_LOCAL_APIC extern u32 get_ibs_caps(void); From da6aac9bf7a5f0afa9756d11517e7f71183f7e11 Mon Sep 17 00:00:00 2001 From: Ravi Bangoria Date: Mon, 16 Feb 2026 04:25:26 +0000 Subject: [PATCH 20/34] perf/amd/ibs: Support IBS_{FETCH|OP}_CTL2[Dis] to eliminate RMW race commit efa5700ec0da66662dc8375fe4e4b888487a6b84 upstream. The existing IBS_{FETCH|OP}_CTL MSRs combine control and status bits which leads to RMW race between HW and SW: HW SW ------------------------ ------------------------------ config = rdmsr(IBS_OP_CTL); config &= ~EN; Set IBS_OP_CTL[Val] to 1 trigger NMI wrmsr(IBS_OP_CTL, config); // Val is accidentally cleared Future hardware adds a control-only MSR, IBS_{FETCH|OP}_CTL2, which provides a second-level "disable" bit (Dis). IBS is now: Enabled: IBS_{FETCH|OP}_CTL[En] = 1 && IBS_{FETCH|OP}_CTL2[Dis] = 0 Disabled: IBS_{FETCH|OP}_CTL[En] = 0 || IBS_{FETCH|OP}_CTL2[Dis] = 1 The separate "Dis" bit lets software disable IBS without touching any status fields, eliminating the hardware/software race. [Backport Changes] 1. Since the commit 78255eb239733 ("Rename 'wrmsrl()' to 'wrmsrq()'"), which renamed wrmsrl() to wrmsrq() globally, is not available in the current source tree and backporting it would introduce large, unrelated changes. Therefore the calls to wrmsrq() were added with equivalent wrmsrl() during the backport. Signed-off-by: Ravi Bangoria Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260216042530.1546-4-ravi.bangoria@amd.com Signed-off-by: suryasaimadhu Signed-off-by: PvsNarasimha --- arch/x86/events/amd/ibs.c | 44 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/arch/x86/events/amd/ibs.c b/arch/x86/events/amd/ibs.c index 6ec32395d6662..c2aa90f59498c 100644 --- a/arch/x86/events/amd/ibs.c +++ b/arch/x86/events/amd/ibs.c @@ -85,9 +85,11 @@ struct cpu_perf_ibs { struct perf_ibs { struct pmu pmu; unsigned int msr; + unsigned int msr2; u64 config_mask; u64 cnt_mask; u64 enable_mask; + u64 disable_mask; u64 valid_mask; u16 min_period; u64 max_period; @@ -291,6 +293,8 @@ static int perf_ibs_init(struct perf_event *event) return -ENOENT; config = event->attr.config; + hwc->extra_reg.config = 0; + hwc->extra_reg.reg = 0; if (event->pmu != &perf_ibs->pmu) return -ENOENT; @@ -318,6 +322,11 @@ static int perf_ibs_init(struct perf_event *event) if (perf_allow_kernel(&event->attr)) hwc->flags |= PERF_X86_EVENT_UNPRIVILEGED; + if (ibs_caps & IBS_CAPS_DIS) { + hwc->extra_reg.config &= ~perf_ibs->disable_mask; + hwc->extra_reg.reg = perf_ibs->msr2; + } + if (hwc->sample_period) { if (config & perf_ibs->cnt_mask) /* raw max_cnt may not be set */ @@ -444,6 +453,9 @@ static inline void perf_ibs_enable_event(struct perf_ibs *perf_ibs, wrmsrl(hwc->config_base, tmp & ~perf_ibs->enable_mask); wrmsrl(hwc->config_base, tmp | perf_ibs->enable_mask); + + if (hwc->extra_reg.reg) + wrmsrl(hwc->extra_reg.reg, hwc->extra_reg.config); } /* @@ -456,6 +468,11 @@ static inline void perf_ibs_enable_event(struct perf_ibs *perf_ibs, static inline void perf_ibs_disable_event(struct perf_ibs *perf_ibs, struct hw_perf_event *hwc, u64 config) { + if (ibs_caps & IBS_CAPS_DIS) { + wrmsrl(hwc->extra_reg.reg, perf_ibs->disable_mask); + return; + } + config &= ~perf_ibs->cnt_mask; if (boot_cpu_data.x86 == 0x10) wrmsrl(hwc->config_base, config); @@ -808,6 +825,7 @@ static struct perf_ibs perf_ibs_fetch = { .check_period = perf_ibs_check_period, }, .msr = MSR_AMD64_IBSFETCHCTL, + .msr2 = MSR_AMD64_IBSFETCHCTL2, .config_mask = IBS_FETCH_MAX_CNT | IBS_FETCH_RAND_EN, .cnt_mask = IBS_FETCH_MAX_CNT, .enable_mask = IBS_FETCH_ENABLE, @@ -833,6 +851,7 @@ static struct perf_ibs perf_ibs_op = { .check_period = perf_ibs_check_period, }, .msr = MSR_AMD64_IBSOPCTL, + .msr2 = MSR_AMD64_IBSOPCTL2, .config_mask = IBS_OP_MAX_CNT, .cnt_mask = IBS_OP_MAX_CNT | IBS_OP_CUR_CNT | IBS_OP_CUR_CNT_RAND, @@ -1390,6 +1409,8 @@ static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs) out: if (throttle) { + if (ibs_caps & IBS_CAPS_DIS) + wrmsrl(hwc->extra_reg.reg, perf_ibs->disable_mask); perf_ibs_stop(event, 0); } else { if (perf_ibs == &perf_ibs_op) { @@ -1463,6 +1484,9 @@ static __init int perf_ibs_fetch_init(void) if (ibs_caps & IBS_CAPS_ZEN4) perf_ibs_fetch.config_mask |= IBS_FETCH_L3MISSONLY; + if (ibs_caps & IBS_CAPS_DIS) + perf_ibs_fetch.disable_mask = IBS_FETCH_2_DIS; + perf_ibs_fetch.pmu.attr_groups = fetch_attr_groups; perf_ibs_fetch.pmu.attr_update = fetch_attr_update; @@ -1484,6 +1508,9 @@ static __init int perf_ibs_op_init(void) if (ibs_caps & IBS_CAPS_ZEN4) perf_ibs_op.config_mask |= IBS_OP_L3MISSONLY; + if (ibs_caps & IBS_CAPS_DIS) + perf_ibs_op.disable_mask = IBS_OP_2_DIS; + perf_ibs_op.pmu.attr_groups = op_attr_groups; perf_ibs_op.pmu.attr_update = op_attr_update; @@ -1730,6 +1757,23 @@ static void clear_APIC_ibs(void) static int x86_pmu_amd_ibs_starting_cpu(unsigned int cpu) { setup_APIC_ibs(); + + if (ibs_caps & IBS_CAPS_DIS) { + /* + * IBS enable sequence: + * CTL[En] = 1; + * CTL2[Dis] = 0; + * + * IBS disable sequence: + * CTL2[Dis] = 1; + * + * Set CTL2[Dis] when CPU comes up. This is needed to make + * enable sequence effective. + */ + wrmsrl(MSR_AMD64_IBSFETCHCTL2, IBS_FETCH_2_DIS); + wrmsrl(MSR_AMD64_IBSOPCTL2, IBS_OP_2_DIS); + } + return 0; } From 6d82dbf7c585de0ac9f433c1776cf26b2716d4e4 Mon Sep 17 00:00:00 2001 From: Ravi Bangoria Date: Mon, 16 Feb 2026 04:25:27 +0000 Subject: [PATCH 21/34] perf/amd/ibs: Enable fetch latency filtering commit 35247fa60b74e1c643423c3bc7c6a59cbca262bb upstream. IBS Fetch on future hardware adds fetch latency filtering which generates interrupt only when FetchLat value exceeds a programmable threshold. Hardware allows threshold in 128-cycle increment (i.e. 128, 256, 384 etc.) from 128 to 1920 cycles. Like the existing IBS filters, samples that fail the latency test are dropped and IBS restarts internally. Since hardware supports threshold in multiple of 128, add a software filter on top to support latency threshold with the granularity of 1 cycle in between [128-1920]. Example: # perf record -e ibs_fetch/fetchlat=128/ -c 10000 -a -- sleep 5 Signed-off-by: Ravi Bangoria Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260216042530.1546-5-ravi.bangoria@amd.com Signed-off-by: suryasaimadhu Signed-off-by: PvsNarasimha --- arch/x86/events/amd/ibs.c | 66 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 65 insertions(+), 1 deletion(-) diff --git a/arch/x86/events/amd/ibs.c b/arch/x86/events/amd/ibs.c index c2aa90f59498c..a90677129d59e 100644 --- a/arch/x86/events/amd/ibs.c +++ b/arch/x86/events/amd/ibs.c @@ -34,6 +34,8 @@ static u32 ibs_caps; /* attr.config1 */ #define IBS_OP_CONFIG1_LDLAT_MASK (0xFFFULL << 0) +#define IBS_FETCH_CONFIG1_FETCHLAT_MASK (0x7FFULL << 0) + /* * IBS states: * @@ -281,6 +283,14 @@ static bool perf_ibs_ldlat_event(struct perf_ibs *perf_ibs, (event->attr.config1 & IBS_OP_CONFIG1_LDLAT_MASK); } +static bool perf_ibs_fetch_lat_event(struct perf_ibs *perf_ibs, + struct perf_event *event) +{ + return perf_ibs == &perf_ibs_fetch && + (ibs_caps & IBS_CAPS_FETCHLAT) && + (event->attr.config1 & IBS_FETCH_CONFIG1_FETCHLAT_MASK); +} + static int perf_ibs_init(struct perf_event *event) { struct hw_perf_event *hwc = &event->hw; @@ -373,6 +383,17 @@ static int perf_ibs_init(struct perf_event *event) config |= IBS_OP_L3MISSONLY | IBS_OP_LDLAT_EN; } + if (perf_ibs_fetch_lat_event(perf_ibs, event)) { + u64 fetchlat = event->attr.config1 & IBS_FETCH_CONFIG1_FETCHLAT_MASK; + + if (fetchlat < 128 || fetchlat > 1920) + return -EINVAL; + fetchlat >>= 7; + + hwc->extra_reg.reg = perf_ibs->msr2; + hwc->extra_reg.config |= fetchlat << IBS_FETCH_2_FETCHLAT_FILTER_SHIFT; + } + /* * If we modify hwc->sample_period, we also need to update * hwc->last_period and hwc->period_left. @@ -661,6 +682,8 @@ PMU_EVENT_ATTR_STRING(ldlat, ibs_op_ldlat_format, "config1:0-11"); PMU_EVENT_ATTR_STRING(zen4_ibs_extensions, zen4_ibs_extensions, "1"); PMU_EVENT_ATTR_STRING(ldlat, ibs_op_ldlat_cap, "1"); PMU_EVENT_ATTR_STRING(dtlb_pgsize, ibs_op_dtlb_pgsize_cap, "1"); +PMU_EVENT_ATTR_STRING(fetchlat, ibs_fetch_lat_format, "config1:0-10"); +PMU_EVENT_ATTR_STRING(fetchlat, ibs_fetch_lat_cap, "1"); static umode_t zen4_ibs_extensions_is_visible(struct kobject *kobj, struct attribute *attr, int i) @@ -668,6 +691,12 @@ zen4_ibs_extensions_is_visible(struct kobject *kobj, struct attribute *attr, int return ibs_caps & IBS_CAPS_ZEN4 ? attr->mode : 0; } +static umode_t +ibs_fetch_lat_is_visible(struct kobject *kobj, struct attribute *attr, int i) +{ + return ibs_caps & IBS_CAPS_FETCHLAT ? attr->mode : 0; +} + static umode_t ibs_op_ldlat_is_visible(struct kobject *kobj, struct attribute *attr, int i) { @@ -696,6 +725,16 @@ static struct attribute *zen4_ibs_extensions_attrs[] = { NULL, }; +static struct attribute *ibs_fetch_lat_format_attrs[] = { + &ibs_fetch_lat_format.attr.attr, + NULL, +}; + +static struct attribute *ibs_fetch_lat_cap_attrs[] = { + &ibs_fetch_lat_cap.attr.attr, + NULL, +}; + static struct attribute *ibs_op_ldlat_cap_attrs[] = { &ibs_op_ldlat_cap.attr.attr, NULL, @@ -723,6 +762,18 @@ static struct attribute_group group_zen4_ibs_extensions = { .is_visible = zen4_ibs_extensions_is_visible, }; +static struct attribute_group group_ibs_fetch_lat_cap = { + .name = "caps", + .attrs = ibs_fetch_lat_cap_attrs, + .is_visible = ibs_fetch_lat_is_visible, +}; + +static struct attribute_group group_ibs_fetch_lat_format = { + .name = "format", + .attrs = ibs_fetch_lat_format_attrs, + .is_visible = ibs_fetch_lat_is_visible, +}; + static struct attribute_group group_ibs_op_ldlat_cap = { .name = "caps", .attrs = ibs_op_ldlat_cap_attrs, @@ -744,6 +795,8 @@ static const struct attribute_group *fetch_attr_groups[] = { static const struct attribute_group *fetch_attr_update[] = { &group_fetch_l3missonly, &group_zen4_ibs_extensions, + &group_ibs_fetch_lat_cap, + &group_ibs_fetch_lat_format, NULL, }; @@ -1187,7 +1240,8 @@ static int perf_ibs_get_offset_max(struct perf_ibs *perf_ibs, { if (event->attr.sample_type & PERF_SAMPLE_RAW || perf_ibs_is_mem_sample_type(perf_ibs, event) || - perf_ibs_ldlat_event(perf_ibs, event)) + perf_ibs_ldlat_event(perf_ibs, event) || + perf_ibs_fetch_lat_event(perf_ibs, event)) return perf_ibs->offset_max; else if (check_rip) return 3; @@ -1329,6 +1383,16 @@ static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs) } } + if (perf_ibs_fetch_lat_event(perf_ibs, event)) { + union ibs_fetch_ctl fetch_ctl; + + fetch_ctl.val = ibs_data.regs[ibs_fetch_msr_idx(MSR_AMD64_IBSFETCHCTL)]; + if (fetch_ctl.fetch_lat < (event->attr.config1 & IBS_FETCH_CONFIG1_FETCHLAT_MASK)) { + throttle = perf_event_account_interrupt(event); + goto out; + } + } + /* * Read IbsBrTarget, IbsOpData4, and IbsExtdCtl separately * depending on their availability. From c6bea8db734009b7aff4ddfa9df2f83cb202ca52 Mon Sep 17 00:00:00 2001 From: Ravi Bangoria Date: Mon, 16 Feb 2026 04:25:28 +0000 Subject: [PATCH 22/34] perf/amd/ibs: Enable RIP bit63 hardware filtering commit 8c63c4af92ac5f041ce437c1f2a31ce3ef03c585 upstream. IBS on future hardware adds the ability to filter IBS events by examining RIP bit 63. Because Linux kernel addresses always have bit 63 set while user-space addresses never do, this capability can be used as a privilege filter. So far, IBS supports privilege filtering in software (swfilt=1), where samples are dropped in the NMI handler. The RIP bit63 hardware filter enables IBS to be usable by unprivileged users without passing swfilt flag. So, swfilt flag will silently be ignored when the hardware filtering capability is present. Example (non-root user): $ perf record -e ibs_op//u -- Signed-off-by: Ravi Bangoria Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260216042530.1546-6-ravi.bangoria@amd.com Signed-off-by: suryasaimadhu Signed-off-by: PvsNarasimha --- arch/x86/events/amd/ibs.c | 46 ++++++++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/arch/x86/events/amd/ibs.c b/arch/x86/events/amd/ibs.c index a90677129d59e..2ce9524172285 100644 --- a/arch/x86/events/amd/ibs.c +++ b/arch/x86/events/amd/ibs.c @@ -320,11 +320,6 @@ static int perf_ibs_init(struct perf_event *event) event->attr.exclude_idle) return -EINVAL; - if (!(event->attr.config2 & IBS_SW_FILTER_MASK) && - (event->attr.exclude_kernel || event->attr.exclude_user || - event->attr.exclude_hv)) - return -EINVAL; - ret = validate_group(event); if (ret) return ret; @@ -337,6 +332,32 @@ static int perf_ibs_init(struct perf_event *event) hwc->extra_reg.reg = perf_ibs->msr2; } + if (ibs_caps & IBS_CAPS_BIT63_FILTER) { + if (perf_ibs == &perf_ibs_fetch) { + if (event->attr.exclude_kernel) { + hwc->extra_reg.config |= IBS_FETCH_2_EXCL_RIP_63_EQ_1; + hwc->extra_reg.reg = perf_ibs->msr2; + } + if (event->attr.exclude_user) { + hwc->extra_reg.config |= IBS_FETCH_2_EXCL_RIP_63_EQ_0; + hwc->extra_reg.reg = perf_ibs->msr2; + } + } else { + if (event->attr.exclude_kernel) { + hwc->extra_reg.config |= IBS_OP_2_EXCL_RIP_63_EQ_1; + hwc->extra_reg.reg = perf_ibs->msr2; + } + if (event->attr.exclude_user) { + hwc->extra_reg.config |= IBS_OP_2_EXCL_RIP_63_EQ_0; + hwc->extra_reg.reg = perf_ibs->msr2; + } + } + } else if (!(event->attr.config2 & IBS_SW_FILTER_MASK) && + (event->attr.exclude_kernel || event->attr.exclude_user || + event->attr.exclude_hv)) { + return -EINVAL; + } + if (hwc->sample_period) { if (config & perf_ibs->cnt_mask) /* raw max_cnt may not be set */ @@ -1276,7 +1297,7 @@ static bool perf_ibs_is_kernel_br_target(struct perf_event *event, op_data.op_brn_ret && kernel_ip(br_target)); } -static bool perf_ibs_swfilt_discard(struct perf_ibs *perf_ibs, struct perf_event *event, +static bool perf_ibs_discard_sample(struct perf_ibs *perf_ibs, struct perf_event *event, struct pt_regs *regs, struct perf_ibs_data *ibs_data, int br_target_idx) { @@ -1431,8 +1452,9 @@ static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs) regs.flags |= PERF_EFLAGS_EXACT; } - if ((event->attr.config2 & IBS_SW_FILTER_MASK) && - perf_ibs_swfilt_discard(perf_ibs, event, ®s, &ibs_data, br_target_idx)) { + if (((ibs_caps & IBS_CAPS_BIT63_FILTER) || + (event->attr.config2 & IBS_SW_FILTER_MASK)) && + perf_ibs_discard_sample(perf_ibs, event, ®s, &ibs_data, br_target_idx)) { throttle = perf_event_account_interrupt(event); goto out; } @@ -1892,6 +1914,14 @@ static __init int amd_ibs_init(void) perf_ibs_pm_init(); +#ifdef CONFIG_X86_32 + /* + * IBS_CAPS_BIT63_FILTER is used for exclude_kernel/user filtering, + * which obviously won't work for 32 bit kernel. + */ + caps &= ~IBS_CAPS_BIT63_FILTER; +#endif + ibs_caps = caps; /* make ibs_caps visible to other cpus: */ smp_mb(); From f5e8464b92867415d8ea53b3b8656b021538d4fd Mon Sep 17 00:00:00 2001 From: Ravi Bangoria Date: Mon, 16 Feb 2026 04:25:29 +0000 Subject: [PATCH 23/34] perf/amd/ibs: Enable streaming store filter commit 8ae68bfec97596a3656ce8d0a7b1240d888eab10 upstream. IBS OP on future hardware supports recording samples only for instructions that does streaming store. Like the existing IBS filters, samples pointing to instruction which does not cause streaming store are discarded and IBS restarts internally. Example: $ perf record -e ibs_op/strmst=1/ -- Signed-off-by: Ravi Bangoria Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260216042530.1546-7-ravi.bangoria@amd.com Signed-off-by: suryasaimadhu Signed-off-by: PvsNarasimha --- arch/x86/events/amd/ibs.c | 51 ++++++++++++++++++++++++++++++++++ arch/x86/include/asm/amd/ibs.h | 3 +- 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/arch/x86/events/amd/ibs.c b/arch/x86/events/amd/ibs.c index 2ce9524172285..d000e1b0347ca 100644 --- a/arch/x86/events/amd/ibs.c +++ b/arch/x86/events/amd/ibs.c @@ -33,6 +33,8 @@ static u32 ibs_caps; /* attr.config1 */ #define IBS_OP_CONFIG1_LDLAT_MASK (0xFFFULL << 0) +#define IBS_OP_CONFIG1_STRMST_MASK (1ULL << 12) +#define IBS_OP_CONFIG1_STRMST_SHIFT (12) #define IBS_FETCH_CONFIG1_FETCHLAT_MASK (0x7FFULL << 0) @@ -291,6 +293,14 @@ static bool perf_ibs_fetch_lat_event(struct perf_ibs *perf_ibs, (event->attr.config1 & IBS_FETCH_CONFIG1_FETCHLAT_MASK); } +static bool perf_ibs_strmst_event(struct perf_ibs *perf_ibs, + struct perf_event *event) +{ + return perf_ibs == &perf_ibs_op && + (ibs_caps & IBS_CAPS_STRMST_RMTSOCKET) && + (event->attr.config1 & IBS_OP_CONFIG1_STRMST_MASK); +} + static int perf_ibs_init(struct perf_event *event) { struct hw_perf_event *hwc = &event->hw; @@ -415,6 +425,15 @@ static int perf_ibs_init(struct perf_event *event) hwc->extra_reg.config |= fetchlat << IBS_FETCH_2_FETCHLAT_FILTER_SHIFT; } + if (perf_ibs_strmst_event(perf_ibs, event)) { + u64 strmst = event->attr.config1 & IBS_OP_CONFIG1_STRMST_MASK; + + strmst >>= IBS_OP_CONFIG1_STRMST_SHIFT; + + hwc->extra_reg.reg = perf_ibs->msr2; + hwc->extra_reg.config |= strmst << IBS_OP_2_STRM_ST_FILTER_SHIFT; + } + /* * If we modify hwc->sample_period, we also need to update * hwc->last_period and hwc->period_left. @@ -705,6 +724,8 @@ PMU_EVENT_ATTR_STRING(ldlat, ibs_op_ldlat_cap, "1"); PMU_EVENT_ATTR_STRING(dtlb_pgsize, ibs_op_dtlb_pgsize_cap, "1"); PMU_EVENT_ATTR_STRING(fetchlat, ibs_fetch_lat_format, "config1:0-10"); PMU_EVENT_ATTR_STRING(fetchlat, ibs_fetch_lat_cap, "1"); +PMU_EVENT_ATTR_STRING(strmst, ibs_op_strmst_format, "config1:12"); +PMU_EVENT_ATTR_STRING(strmst, ibs_op_strmst_cap, "1"); static umode_t zen4_ibs_extensions_is_visible(struct kobject *kobj, struct attribute *attr, int i) @@ -718,6 +739,12 @@ ibs_fetch_lat_is_visible(struct kobject *kobj, struct attribute *attr, int i) return ibs_caps & IBS_CAPS_FETCHLAT ? attr->mode : 0; } +static umode_t +ibs_op_strmst_is_visible(struct kobject *kobj, struct attribute *attr, int i) +{ + return ibs_caps & IBS_CAPS_STRMST_RMTSOCKET ? attr->mode : 0; +} + static umode_t ibs_op_ldlat_is_visible(struct kobject *kobj, struct attribute *attr, int i) { @@ -766,6 +793,11 @@ static struct attribute *ibs_op_dtlb_pgsize_cap_attrs[] = { NULL, }; +static struct attribute *ibs_op_strmst_cap_attrs[] = { + &ibs_op_strmst_cap.attr.attr, + NULL, +}; + static struct attribute_group group_fetch_formats = { .name = "format", .attrs = fetch_attrs, @@ -807,6 +839,12 @@ static struct attribute_group group_ibs_op_dtlb_pgsize_cap = { .is_visible = ibs_op_dtlb_pgsize_is_visible, }; +static struct attribute_group group_ibs_op_strmst_cap = { + .name = "caps", + .attrs = ibs_op_strmst_cap_attrs, + .is_visible = ibs_op_strmst_is_visible, +}; + static const struct attribute_group *fetch_attr_groups[] = { &group_fetch_formats, &empty_caps_group, @@ -852,6 +890,11 @@ static struct attribute *ibs_op_ldlat_format_attrs[] = { NULL, }; +static struct attribute *ibs_op_strmst_format_attrs[] = { + &ibs_op_strmst_format.attr.attr, + NULL, +}; + static struct attribute_group group_cnt_ctl = { .name = "format", .attrs = cnt_ctl_attrs, @@ -876,6 +919,12 @@ static struct attribute_group group_ibs_op_ldlat_format = { .is_visible = ibs_op_ldlat_is_visible, }; +static struct attribute_group group_ibs_op_strmst_format = { + .name = "format", + .attrs = ibs_op_strmst_format_attrs, + .is_visible = ibs_op_strmst_is_visible, +}; + static const struct attribute_group *op_attr_update[] = { &group_cnt_ctl, &group_op_l3missonly, @@ -883,6 +932,8 @@ static const struct attribute_group *op_attr_update[] = { &group_ibs_op_ldlat_cap, &group_ibs_op_ldlat_format, &group_ibs_op_dtlb_pgsize_cap, + &group_ibs_op_strmst_cap, + &group_ibs_op_strmst_format, NULL, }; diff --git a/arch/x86/include/asm/amd/ibs.h b/arch/x86/include/asm/amd/ibs.h index 77f3a589a99ac..9526336300514 100644 --- a/arch/x86/include/asm/amd/ibs.h +++ b/arch/x86/include/asm/amd/ibs.h @@ -96,7 +96,8 @@ union ibs_op_data2 { rmt_node:1, /* 4: destination node */ cache_hit_st:1, /* 5: cache hit state */ data_src_hi:2, /* 6-7: data source high */ - reserved1:56; /* 8-63: reserved */ + strm_st:1, /* 8: streaming store */ + reserved1:55; /* 9-63: reserved */ }; }; From b1e627cdcd77ee6b976fceab1f2ff938df57a44c Mon Sep 17 00:00:00 2001 From: Ravi Bangoria Date: Mon, 16 Feb 2026 04:25:30 +0000 Subject: [PATCH 24/34] perf/amd/ibs: Advertise remote socket capability commit b2ea0f541d354c10186a894e8bf5bf83abf8a517 upstream. IBS OP on future hardware can indicate data source from remote socket as well. Advertise this capability to userspace so that userspace tools can decode IBS data accordingly. Signed-off-by: Ravi Bangoria Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260216042530.1546-8-ravi.bangoria@amd.com Signed-off-by: suryasaimadhu Signed-off-by: PvsNarasimha --- arch/x86/events/amd/ibs.c | 19 +++++++++++++++++++ arch/x86/include/asm/amd/ibs.h | 3 ++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/arch/x86/events/amd/ibs.c b/arch/x86/events/amd/ibs.c index d000e1b0347ca..8b9896fb20a71 100644 --- a/arch/x86/events/amd/ibs.c +++ b/arch/x86/events/amd/ibs.c @@ -726,6 +726,7 @@ PMU_EVENT_ATTR_STRING(fetchlat, ibs_fetch_lat_format, "config1:0-10"); PMU_EVENT_ATTR_STRING(fetchlat, ibs_fetch_lat_cap, "1"); PMU_EVENT_ATTR_STRING(strmst, ibs_op_strmst_format, "config1:12"); PMU_EVENT_ATTR_STRING(strmst, ibs_op_strmst_cap, "1"); +PMU_EVENT_ATTR_STRING(rmtsocket, ibs_op_rmtsocket_cap, "1"); static umode_t zen4_ibs_extensions_is_visible(struct kobject *kobj, struct attribute *attr, int i) @@ -745,6 +746,12 @@ ibs_op_strmst_is_visible(struct kobject *kobj, struct attribute *attr, int i) return ibs_caps & IBS_CAPS_STRMST_RMTSOCKET ? attr->mode : 0; } +static umode_t +ibs_op_rmtsocket_is_visible(struct kobject *kobj, struct attribute *attr, int i) +{ + return ibs_caps & IBS_CAPS_STRMST_RMTSOCKET ? attr->mode : 0; +} + static umode_t ibs_op_ldlat_is_visible(struct kobject *kobj, struct attribute *attr, int i) { @@ -798,6 +805,11 @@ static struct attribute *ibs_op_strmst_cap_attrs[] = { NULL, }; +static struct attribute *ibs_op_rmtsocket_cap_attrs[] = { + &ibs_op_rmtsocket_cap.attr.attr, + NULL, +}; + static struct attribute_group group_fetch_formats = { .name = "format", .attrs = fetch_attrs, @@ -845,6 +857,12 @@ static struct attribute_group group_ibs_op_strmst_cap = { .is_visible = ibs_op_strmst_is_visible, }; +static struct attribute_group group_ibs_op_rmtsocket_cap = { + .name = "caps", + .attrs = ibs_op_rmtsocket_cap_attrs, + .is_visible = ibs_op_rmtsocket_is_visible, +}; + static const struct attribute_group *fetch_attr_groups[] = { &group_fetch_formats, &empty_caps_group, @@ -934,6 +952,7 @@ static const struct attribute_group *op_attr_update[] = { &group_ibs_op_dtlb_pgsize_cap, &group_ibs_op_strmst_cap, &group_ibs_op_strmst_format, + &group_ibs_op_rmtsocket_cap, NULL, }; diff --git a/arch/x86/include/asm/amd/ibs.h b/arch/x86/include/asm/amd/ibs.h index 9526336300514..3c974ab368bdb 100644 --- a/arch/x86/include/asm/amd/ibs.h +++ b/arch/x86/include/asm/amd/ibs.h @@ -97,7 +97,8 @@ union ibs_op_data2 { cache_hit_st:1, /* 5: cache hit state */ data_src_hi:2, /* 6-7: data source high */ strm_st:1, /* 8: streaming store */ - reserved1:55; /* 9-63: reserved */ + rmt_socket:1, /* 9: remote socket */ + reserved1:54; /* 10-63: reserved */ }; }; From ffc33d4bf2ac44e41b20d2d11b615369058668db Mon Sep 17 00:00:00 2001 From: Sun Haiyong Date: Mon, 4 Dec 2023 16:20:55 +0800 Subject: [PATCH 25/34] perf top: Remove needless malloc(0) call that triggers -Walloc-size MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit commit 79baac8acfc60a7a5114f6d60731e28c242ef8ce upstream. GCC 14 introduces a new -Walloc-size included in -Wextra which errors out like: builtin-top.c: In function ‘prompt_integer’: builtin-top.c:360:21: error: allocation of insufficient size ‘0’ for type ‘char’ with size ‘1’ [-Werror=alloc-size] 360 | char *buf = malloc(0), *p; | ^~~~~~ Just set it to NULL, getline() will do the allocation. Signed-off-by: Sun Haiyong Signed-off-by: Yanteng Si Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Ingo Molnar Cc: Jiri Olsa Cc: Mark Rutland Cc: Namhyung Kim Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20231204082055.91877-1-siyanteng@loongson.cn Signed-off-by: Arnaldo Carvalho de Melo Signed-off-by: PvsNarasimha --- tools/perf/builtin-top.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/perf/builtin-top.c b/tools/perf/builtin-top.c index 381274c70a9af..2308c336c01c7 100644 --- a/tools/perf/builtin-top.c +++ b/tools/perf/builtin-top.c @@ -357,7 +357,7 @@ static void perf_top__print_sym_table(struct perf_top *top) static void prompt_integer(int *target, const char *msg) { - char *buf = malloc(0), *p; + char *buf = NULL, *p; size_t dummy = 0; int tmp; From 1e88f438a634b5211e713c8b4d49f06be4beca47 Mon Sep 17 00:00:00 2001 From: Sun Haiyong Date: Sat, 6 Jan 2024 17:41:29 +0800 Subject: [PATCH 26/34] perf tools: Fix calloc() arguments to address error introduced in gcc-14 commit 7bbe8f0071dfa23fcc3b2864ec9f3b1aeb7ab2df upstream. the definition of calloc is as follows: void *calloc(size_t nmemb, size_t size); number of members is in the first parameter and the size is in the second parameter. Fix error messages on gcc 14 20240102: error: 'calloc' sizes specified with 'sizeof' in the earlier argument and not in the later argument [-Werror=calloc-transposed-args] Committer notes: I noticed this on fedora 40 and rawhide. Signed-off-by: Sun Haiyong Acked-by: Namhyung Kim Tested-by: Arnaldo Carvalho de Melo Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Ian Rogers Cc: Ingo Molnar Cc: Jiri Olsa Cc: Mark Rutland Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20240106094129.3337057-1-siyanteng@loongson.cn Signed-off-by: Yanteng Si Signed-off-by: Arnaldo Carvalho de Melo Signed-off-by: PvsNarasimha --- tools/perf/builtin-record.c | 4 ++-- tools/perf/util/hist.c | 4 ++-- tools/perf/util/metricgroup.c | 2 +- tools/perf/util/synthetic-events.c | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tools/perf/builtin-record.c b/tools/perf/builtin-record.c index ddaf38f92898f..67ed865e8f243 100644 --- a/tools/perf/builtin-record.c +++ b/tools/perf/builtin-record.c @@ -4036,8 +4036,8 @@ int cmd_record(int argc, const char **argv) } if (rec->switch_output.num_files) { - rec->switch_output.filenames = calloc(sizeof(char *), - rec->switch_output.num_files); + rec->switch_output.filenames = calloc(rec->switch_output.num_files, + sizeof(char *)); if (!rec->switch_output.filenames) { err = -EINVAL; goto out_opts; diff --git a/tools/perf/util/hist.c b/tools/perf/util/hist.c index aa450cc2648aa..ef0f7aefc4b14 100644 --- a/tools/perf/util/hist.c +++ b/tools/perf/util/hist.c @@ -491,8 +491,8 @@ static int hist_entry__init(struct hist_entry *he, } if (symbol_conf.res_sample) { - he->res_samples = calloc(sizeof(struct res_sample), - symbol_conf.res_sample); + he->res_samples = calloc(symbol_conf.res_sample, + sizeof(struct res_sample)); if (!he->res_samples) goto err_srcline; } diff --git a/tools/perf/util/metricgroup.c b/tools/perf/util/metricgroup.c index bb5faaa25d510..df1595b69528a 100644 --- a/tools/perf/util/metricgroup.c +++ b/tools/perf/util/metricgroup.c @@ -286,7 +286,7 @@ static int setup_metric_events(const char *pmu, struct hashmap *ids, *out_metric_events = NULL; ids_size = hashmap__size(ids); - metric_events = calloc(sizeof(void *), ids_size + 1); + metric_events = calloc(ids_size + 1, sizeof(void *)); if (!metric_events) return -ENOMEM; diff --git a/tools/perf/util/synthetic-events.c b/tools/perf/util/synthetic-events.c index a0579c7d7b9e9..7577e63515df9 100644 --- a/tools/perf/util/synthetic-events.c +++ b/tools/perf/util/synthetic-events.c @@ -1039,11 +1039,11 @@ int perf_event__synthesize_threads(struct perf_tool *tool, if (thread_nr > n) thread_nr = n; - synthesize_threads = calloc(sizeof(pthread_t), thread_nr); + synthesize_threads = calloc(thread_nr, sizeof(pthread_t)); if (synthesize_threads == NULL) goto free_dirent; - args = calloc(sizeof(*args), thread_nr); + args = calloc(thread_nr, sizeof(*args)); if (args == NULL) goto free_threads; From abfb9742f0e38b96ca505d5a7fff387524b9e536 Mon Sep 17 00:00:00 2001 From: Arnaldo Carvalho de Melo Date: Thu, 12 Jun 2025 18:08:40 -0300 Subject: [PATCH 27/34] tools arch amd ibs: Sync ibs.h with the kernel sources commit bb6b4143503750318a3f85975186db899a3caaf2 upstream. To pick up the changes from: 861c6b1185fbb2e3 ("x86/platform/amd: Add standard header guards to ") A small change to tools/perf/check-headers.sh was made to cope with the move of this header done in: 3846389c03a85188 ("x86/platform/amd: Move the header to ") That don't result in any changes in the tools, just address this perf build warning: Warning: Kernel ABI header differences: diff -u tools/arch/x86/include/asm/amd/ibs.h arch/x86/include/asm/amd/ibs.h Cc: Adrian Hunter Cc: Ian Rogers Cc: Ingo Molnar Cc: James Clark Cc: Jiri Olsa Cc: Kan Liang Cc: Namhyung Kim Link: https://lore.kernel.org/r/aEtCi0pup5FEwnzn@x1 Signed-off-by: Arnaldo Carvalho de Melo Signed-off-by: PvsNarasimha --- tools/arch/x86/include/asm/amd-ibs.h | 5 +++++ tools/perf/check-headers.sh | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/arch/x86/include/asm/amd-ibs.h b/tools/arch/x86/include/asm/amd-ibs.h index 93807b437e4de..a73b5c7fbb9a8 100644 --- a/tools/arch/x86/include/asm/amd-ibs.h +++ b/tools/arch/x86/include/asm/amd-ibs.h @@ -1,4 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_X86_AMD_IBS_H +#define _ASM_X86_AMD_IBS_H + /* * From PPR Vol 1 for AMD Family 19h Model 01h B1 * 55898 Rev 0.35 - Feb 5, 2021 @@ -150,3 +153,5 @@ struct perf_ibs_data { }; u64 regs[MSR_AMD64_IBS_REG_COUNT_MAX]; }; + +#endif /* _ASM_X86_AMD_IBS_H */ diff --git a/tools/perf/check-headers.sh b/tools/perf/check-headers.sh index d62732aa15777..a9a5c8ec82b46 100755 --- a/tools/perf/check-headers.sh +++ b/tools/perf/check-headers.sh @@ -160,7 +160,7 @@ done # diff with extra ignore lines check arch/x86/lib/memcpy_64.S '-I "^EXPORT_SYMBOL" -I "^#include " -I"^SYM_FUNC_START\(_LOCAL\)*(memcpy_\(erms\|orig\))" -I"^#include "' check arch/x86/lib/memset_64.S '-I "^EXPORT_SYMBOL" -I "^#include " -I"^SYM_FUNC_START\(_LOCAL\)*(memset_\(erms\|orig\))"' -check arch/x86/include/asm/amd/ibs.h '-I "^#include [<\"]\(asm/\)*msr-index.h"' +check arch/x86/include/asm/amd/ibs.h '-I "^#include .*/msr-index.h"' check arch/arm64/include/asm/cputype.h '-I "^#include [<\"]\(asm/\)*sysreg.h"' check include/uapi/asm-generic/mman.h '-I "^#include <\(uapi/\)*asm-generic/mman-common\(-tools\)*.h>"' check include/uapi/linux/mman.h '-I "^#include <\(uapi/\)*asm/mman.h>"' From 2418e2a702e5510be0d96e57627a9b2bef35f5fb Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Tue, 6 May 2025 13:41:43 +0200 Subject: [PATCH 28/34] tools/arch/x86: Move the header to commit c1ab4ce3cb759f69fb9085a060e568b73e8f5cd8 upstream. Synchronize with what we did with the kernel side header in: 3846389c03a8 ("x86/platform/amd: Move the header to ") Reported-by: Stephen Rothwell Signed-off-by: Ingo Molnar Cc: linux-kernel@vger.kernel.org Signed-off-by: PvsNarasimha --- tools/arch/x86/include/asm/{amd-ibs.h => amd/ibs.h} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename tools/arch/x86/include/asm/{amd-ibs.h => amd/ibs.h} (99%) diff --git a/tools/arch/x86/include/asm/amd-ibs.h b/tools/arch/x86/include/asm/amd/ibs.h similarity index 99% rename from tools/arch/x86/include/asm/amd-ibs.h rename to tools/arch/x86/include/asm/amd/ibs.h index a73b5c7fbb9a8..8f19b05920456 100644 --- a/tools/arch/x86/include/asm/amd-ibs.h +++ b/tools/arch/x86/include/asm/amd/ibs.h @@ -7,7 +7,7 @@ * 55898 Rev 0.35 - Feb 5, 2021 */ -#include "msr-index.h" +#include "../msr-index.h" /* IBS_OP_DATA2 DataSrc */ #define IBS_DATA_SRC_LOC_CACHE 2 From e3f937a22b0d525941a42ca57f553dc6434ca95d Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Mon, 2 Feb 2026 18:43:10 +0000 Subject: [PATCH 29/34] arm64: Convert SCTLR_EL2 to sysreg infrastructure commit 0a35bd285f43c26ccec33872fc6bb679069eaea8 upstream. Convert SCTLR_EL2 to the sysreg infrastructure, as per the 2025-12_rel revision of the Registers.json file. Note that we slightly deviate from the above, as we stick to the ARM ARM M.a definition of SCTLR_EL2[9], which is RES0, in order to avoid dragging the POE2 definitions... Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20260202184329.2724080-2-maz@kernel.org Signed-off-by: Marc Zyngier Signed-off-by: PvsNarasimha --- arch/arm64/include/asm/sysreg.h | 7 --- arch/arm64/tools/sysreg | 69 +++++++++++++++++++++++++++ tools/arch/arm64/include/asm/sysreg.h | 6 --- 3 files changed, 69 insertions(+), 13 deletions(-) diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index 2ca39be924f04..822b2e9f9d84d 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -484,7 +484,6 @@ #define SYS_VPIDR_EL2 sys_reg(3, 4, 0, 0, 0) #define SYS_VMPIDR_EL2 sys_reg(3, 4, 0, 0, 5) -#define SYS_SCTLR_EL2 sys_reg(3, 4, 1, 0, 0) #define SYS_ACTLR_EL2 sys_reg(3, 4, 1, 0, 1) #define SYS_SCTLR2_EL2 sys_reg(3, 4, 1, 0, 3) #define SYS_HCR_EL2 sys_reg(3, 4, 1, 1, 0) @@ -810,12 +809,6 @@ #define SCTLR_ELx_A (BIT(1)) #define SCTLR_ELx_M (BIT(0)) -/* SCTLR_EL2 specific flags. */ -#define SCTLR_EL2_RES1 ((BIT(4)) | (BIT(5)) | (BIT(11)) | (BIT(16)) | \ - (BIT(18)) | (BIT(22)) | (BIT(23)) | (BIT(28)) | \ - (BIT(29))) - -#define SCTLR_EL2_BT (BIT(36)) #ifdef CONFIG_CPU_BIG_ENDIAN #define ENDIAN_SET_EL2 SCTLR_ELx_EE #else diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index 0f9755ed5b6d0..e1ebbc20e6dc3 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -2024,6 +2024,75 @@ Field 1 ZA Field 0 SM EndSysreg +Sysreg SCTLR_EL2 3 4 1 0 0 +Field 63 TIDCP +Field 62 SPINTMASK +Field 61 NMI +Field 60 EnTP2 +Field 59 TCSO +Field 58 TCSO0 +Field 57 EPAN +Field 56 EnALS +Field 55 EnAS0 +Field 54 EnASR +Res0 53:50 +Field 49:46 TWEDEL +Field 45 TWEDEn +Field 44 DSSBS +Field 43 ATA +Field 42 ATA0 +Enum 41:40 TCF + 0b00 NONE + 0b01 SYNC + 0b10 ASYNC + 0b11 ASYMM +EndEnum +Enum 39:38 TCF0 + 0b00 NONE + 0b01 SYNC + 0b10 ASYNC + 0b11 ASYMM +EndEnum +Field 37 ITFSB +Field 36 BT +Field 35 BT0 +Field 34 EnFPM +Field 33 MSCEn +Field 32 CMOW +Field 31 EnIA +Field 30 EnIB +Field 29 LSMAOE +Field 28 nTLSMD +Field 27 EnDA +Field 26 UCI +Field 25 EE +Field 24 E0E +Field 23 SPAN +Field 22 EIS +Field 21 IESB +Field 20 TSCXT +Field 19 WXN +Field 18 nTWE +Res0 17 +Field 16 nTWI +Field 15 UCT +Field 14 DZE +Field 13 EnDB +Field 12 I +Field 11 EOS +Field 10 EnRCTX +Res0 9 +Field 8 SED +Field 7 ITD +Field 6 nAA +Field 5 CP15BEN +Field 4 SA0 +Field 3 SA +Field 2 C +Field 1 A +Field 0 M +EndSysreg + SysregFields HFGxTR_EL2 Field 63 nAMAIR2_EL1 Field 62 nMAIR2_EL1 diff --git a/tools/arch/arm64/include/asm/sysreg.h b/tools/arch/arm64/include/asm/sysreg.h index ccc13e9913760..425aa6ec1d1a8 100644 --- a/tools/arch/arm64/include/asm/sysreg.h +++ b/tools/arch/arm64/include/asm/sysreg.h @@ -504,12 +504,6 @@ #define SCTLR_ELx_A (BIT(1)) #define SCTLR_ELx_M (BIT(0)) -/* SCTLR_EL2 specific flags. */ -#define SCTLR_EL2_RES1 ((BIT(4)) | (BIT(5)) | (BIT(11)) | (BIT(16)) | \ - (BIT(18)) | (BIT(22)) | (BIT(23)) | (BIT(28)) | \ - (BIT(29))) - -#define SCTLR_EL2_BT (BIT(36)) #ifdef CONFIG_CPU_BIG_ENDIAN #define ENDIAN_SET_EL2 SCTLR_ELx_EE #else From 16ecb8b33bed5f203027cd976baa6302503dec9c Mon Sep 17 00:00:00 2001 From: James Clark Date: Mon, 6 Jan 2025 14:24:38 +0000 Subject: [PATCH 30/34] arm64/sysreg/tools: Move TRFCR definitions to sysreg commit c382ee674c8b5005798606267d660cf995218b18 upstream. Convert TRFCR to automatic generation. Add separate definitions for ELx and EL2 as TRFCR_EL1 doesn't have CX. This also mirrors the previous definition so no code change is required. Also add TRFCR_EL12 which will start to be used in a later commit. Unfortunately, to avoid breaking the Perf build with duplicate definition errors, the tools copy of the sysreg.h header needs to be updated at the same time rather than the usual second commit. This is because the generated version of sysreg (arch/arm64/include/generated/asm/sysreg-defs.h), is currently shared and tools/ does not have its own copy. Reviewed-by: Mark Brown Signed-off-by: James Clark Signed-off-by: James Clark Link: https://lore.kernel.org/r/20250106142446.628923-4-james.clark@linaro.org Signed-off-by: Marc Zyngier Signed-off-by: PvsNarasimha --- arch/arm64/include/asm/sysreg.h | 12 --------- arch/arm64/tools/sysreg | 36 +++++++++++++++++++++++++++ tools/arch/arm64/include/asm/sysreg.h | 12 --------- 3 files changed, 36 insertions(+), 24 deletions(-) diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index 822b2e9f9d84d..e50c8de1373b8 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -280,8 +280,6 @@ #define SYS_RGSR_EL1 sys_reg(3, 0, 1, 0, 5) #define SYS_GCR_EL1 sys_reg(3, 0, 1, 0, 6) -#define SYS_TRFCR_EL1 sys_reg(3, 0, 1, 2, 1) - #define SYS_TCR_EL1 sys_reg(3, 0, 2, 0, 2) #define SYS_APIAKEYLO_EL1 sys_reg(3, 0, 2, 1, 0) @@ -498,7 +496,6 @@ #define SYS_VTTBR_EL2 sys_reg(3, 4, 2, 1, 0) #define SYS_VTCR_EL2 sys_reg(3, 4, 2, 1, 2) -#define SYS_TRFCR_EL2 sys_reg(3, 4, 1, 2, 1) #define SYS_VNCR_EL2 sys_reg(3, 4, 2, 2, 0) #define SYS_HAFGRTR_EL2 sys_reg(3, 4, 3, 1, 6) #define SYS_SPSR_EL2 sys_reg(3, 4, 4, 0, 0) @@ -943,15 +940,6 @@ /* Safe value for MPIDR_EL1: Bit31:RES1, Bit30:U:0, Bit24:MT:0 */ #define SYS_MPIDR_SAFE_VAL (BIT(31)) -#define TRFCR_ELx_TS_SHIFT 5 -#define TRFCR_ELx_TS_MASK ((0x3UL) << TRFCR_ELx_TS_SHIFT) -#define TRFCR_ELx_TS_VIRTUAL ((0x1UL) << TRFCR_ELx_TS_SHIFT) -#define TRFCR_ELx_TS_GUEST_PHYSICAL ((0x2UL) << TRFCR_ELx_TS_SHIFT) -#define TRFCR_ELx_TS_PHYSICAL ((0x3UL) << TRFCR_ELx_TS_SHIFT) -#define TRFCR_EL2_CX BIT(3) -#define TRFCR_ELx_ExTRE BIT(1) -#define TRFCR_ELx_E0TRE BIT(0) - /* GIC Hypervisor interface registers */ /* ICH_MISR_EL2 bit definitions */ #define ICH_MISR_EOI (1 << 0) diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index e1ebbc20e6dc3..34f9fbf6460b6 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -1760,6 +1760,22 @@ Sysreg CPACR_EL1 3 0 1 0 2 Fields CPACR_ELx EndSysreg +SysregFields TRFCR_ELx +Res0 63:7 +UnsignedEnum 6:5 TS + 0b0001 VIRTUAL + 0b0010 GUEST_PHYSICAL + 0b0011 PHYSICAL +EndEnum +Res0 4:2 +Field 1 ExTRE +Field 0 E0TRE +EndSysregFields + +Sysreg TRFCR_EL1 3 0 1 2 1 +Fields TRFCR_ELx +EndSysreg + Sysreg SMPRI_EL1 3 0 1 2 4 Res0 63:4 Field 3:0 PRIORITY @@ -2233,6 +2249,22 @@ Field 1 ICIALLU Field 0 ICIALLUIS EndSysreg +Sysreg TRFCR_EL2 3 4 1 2 1 +Res0 63:7 +UnsignedEnum 6:5 TS + 0b0000 USE_TRFCR_EL1_TS + 0b0001 VIRTUAL + 0b0010 GUEST_PHYSICAL + 0b0011 PHYSICAL +EndEnum +Res0 4 +Field 3 CX +Res0 2 +Field 1 E2TRE +Field 0 E0HTRE +EndSysreg + + Sysreg HDFGRTR_EL2 3 4 3 1 4 Field 63 PMBIDR_EL1 Field 62 nPMSNEVFR_EL1 @@ -2470,6 +2502,10 @@ Sysreg ZCR_EL12 3 5 1 2 0 Fields ZCR_ELx EndSysreg +Sysreg TRFCR_EL12 3 5 1 2 1 +Fields TRFCR_ELx +EndSysreg + Sysreg SMCR_EL12 3 5 1 2 6 Fields SMCR_ELx EndSysreg diff --git a/tools/arch/arm64/include/asm/sysreg.h b/tools/arch/arm64/include/asm/sysreg.h index 425aa6ec1d1a8..dc0dfc6c0da85 100644 --- a/tools/arch/arm64/include/asm/sysreg.h +++ b/tools/arch/arm64/include/asm/sysreg.h @@ -170,8 +170,6 @@ #define SYS_RGSR_EL1 sys_reg(3, 0, 1, 0, 5) #define SYS_GCR_EL1 sys_reg(3, 0, 1, 0, 6) -#define SYS_TRFCR_EL1 sys_reg(3, 0, 1, 2, 1) - #define SYS_TCR_EL1 sys_reg(3, 0, 2, 0, 2) #define SYS_APIAKEYLO_EL1 sys_reg(3, 0, 2, 1, 0) @@ -381,7 +379,6 @@ #define SYS_VTTBR_EL2 sys_reg(3, 4, 2, 1, 0) #define SYS_VTCR_EL2 sys_reg(3, 4, 2, 1, 2) -#define SYS_TRFCR_EL2 sys_reg(3, 4, 1, 2, 1) #define SYS_HDFGRTR_EL2 sys_reg(3, 4, 3, 1, 4) #define SYS_HDFGWTR_EL2 sys_reg(3, 4, 3, 1, 5) #define SYS_HAFGRTR_EL2 sys_reg(3, 4, 3, 1, 6) @@ -620,15 +617,6 @@ /* Safe value for MPIDR_EL1: Bit31:RES1, Bit30:U:0, Bit24:MT:0 */ #define SYS_MPIDR_SAFE_VAL (BIT(31)) -#define TRFCR_ELx_TS_SHIFT 5 -#define TRFCR_ELx_TS_MASK ((0x3UL) << TRFCR_ELx_TS_SHIFT) -#define TRFCR_ELx_TS_VIRTUAL ((0x1UL) << TRFCR_ELx_TS_SHIFT) -#define TRFCR_ELx_TS_GUEST_PHYSICAL ((0x2UL) << TRFCR_ELx_TS_SHIFT) -#define TRFCR_ELx_TS_PHYSICAL ((0x3UL) << TRFCR_ELx_TS_SHIFT) -#define TRFCR_EL2_CX BIT(3) -#define TRFCR_ELx_ExTRE BIT(1) -#define TRFCR_ELx_E0TRE BIT(0) - /* GIC Hypervisor interface registers */ /* ICH_MISR_EL2 bit definitions */ #define ICH_MISR_EOI (1 << 0) From df6d74cb406d050b3dd3c1a5bfefec6e1e295521 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Fri, 1 May 2026 12:21:44 +0100 Subject: [PATCH 31/34] KVM: arm64: Make EL2 exception entry and exit context-synchronization events commit d7396a72eae795d7f968fb451237b6ac1616d712 upstream. SCTLR_EL2.EIS and SCTLR_EL2.EOS control whether exception entry and exit at EL2 are Context Synchronisation Events (CSEs). Per ARM DDI 0487 M.b D24.2.175 (p. D24-9754): - !FEAT_ExS: the bit is RES1, so the entry/exit is unconditionally a CSE. - FEAT_ExS: the reset value is architecturally UNKNOWN; software must set the bit to make the entry/exit a CSE. INIT_SCTLR_EL2_MMU_ON in arch/arm64/include/asm/sysreg.h sets neither bit. KVM/arm64 hot paths rely on ERET from EL2 being a CSE, and on synchronous EL1->EL2 entry being a CSE, to elide explicit ISBs after MSRs to context-switching system registers (HCR_EL2, ZCR_EL2, ptrauth keys, etc.). On FEAT_ExS hardware those reliances are not architecturally backed unless EOS=1 (and, for entry, EIS=1). Until commit 0a35bd285f43 ("arm64: Convert SCTLR_EL2 to sysreg infrastructure"), SCTLR_EL2_RES1 was a hand-rolled mask that included BIT(11) (EOS) and BIT(22) (EIS), so INIT_SCTLR_EL2_MMU_ON was setting both unconditionally. The conversion made SCTLR_EL2_RES1 auto-generated; because the sysreg tooling only models unconditionally-RES1 fields and EIS/EOS are RES1 only when FEAT_ExS is absent, the auto-generated mask is UL(0). The seven other bits dropped from the old mask (positions 4, 5, 16, 18, 23, 28, 29) are unconditionally RES1 in the E2H=0 SCTLR_EL2 layout per DDI 0487 M.b D24.2.175, so dropping them is harmless. EIS and EOS are the only bits whose semantics changed for FEAT_ExS hardware and where the kernel relies on the value being 1. Make the guarantee explicit: include SCTLR_ELx_EIS | SCTLR_ELx_EOS in INIT_SCTLR_EL2_MMU_ON so that EL2 exception entry and exit are unconditionally CSEs regardless of whether FEAT_ExS is implemented. This matches the pairing in arch/arm64/kvm/config.c which treats EIS and EOS together as RES1 under !FEAT_ExS. Fixes: 0a35bd285f43 ("arm64: Convert SCTLR_EL2 to sysreg infrastructure") Reviewed-by: Yuan Yao Assisted-by: Gemini:gemini-3.1-pro review-prompts Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260501112149.2824881-2-tabba@google.com Signed-off-by: Marc Zyngier Signed-off-by: PvsNarasimha --- arch/arm64/include/asm/sysreg.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index e50c8de1373b8..c76c5fee759af 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -815,7 +815,7 @@ #define INIT_SCTLR_EL2_MMU_ON \ (SCTLR_ELx_M | SCTLR_ELx_C | SCTLR_ELx_SA | SCTLR_ELx_I | \ SCTLR_ELx_IESB | SCTLR_ELx_WXN | ENDIAN_SET_EL2 | \ - SCTLR_ELx_ITFSB | SCTLR_EL2_RES1) + SCTLR_ELx_ITFSB | SCTLR_ELx_EIS | SCTLR_ELx_EOS | SCTLR_EL2_RES1) #define INIT_SCTLR_EL2_MMU_OFF \ (SCTLR_EL2_RES1 | ENDIAN_SET_EL2) From 3e3f3ebefe5aef532abceaec9a58e7a4b9f2b1dd Mon Sep 17 00:00:00 2001 From: James Clark Date: Mon, 6 Jan 2025 14:24:42 +0000 Subject: [PATCH 32/34] coresight: Pass guest TRFCR value to KVM commit aaf69eff6cdb8613ff1f6a520821f769dc92f969 upstream. Currently the userspace and kernel filters for guests are never set, so no trace will be generated for them. Add support for tracing guests by passing the desired TRFCR value to KVM so it can be applied to the guest. By writing either E1TRE or E0TRE, filtering on either guest kernel or guest userspace is also supported. And if both E1TRE and E0TRE are cleared when exclude_guest is set, that option is supported too. This change also brings exclude_host support which is difficult to add as a separate commit without excess churn and resulting in no trace at all. cpu_prohibit_trace() gets moved to TRBE because the ETM driver doesn't need the read, it already has the base TRFCR value. TRBE only needs the read to disable it and then restore. Testing ======= The addresses were counted with the following: $ perf report -D | grep -Eo 'EL2|EL1|EL0' | sort | uniq -c Guest kernel only: $ perf record -e cs_etm//Gk -a -- true 535 EL1 1 EL2 Guest user only (only 5 addresses because the guest runs slowly in the model): $ perf record -e cs_etm//Gu -a -- true 5 EL0 Host kernel only: $ perf record -e cs_etm//Hk -a -- true 3501 EL2 Host userspace only: $ perf record -e cs_etm//Hu -a -- true 408 EL0 1 EL2 Signed-off-by: James Clark Link: https://lore.kernel.org/r/20250106142446.628923-8-james.clark@linaro.org Signed-off-by: Marc Zyngier Signed-off-by: PvsNarasimha --- .../coresight/coresight-etm4x-core.c | 49 ++++++++++++++++--- drivers/hwtracing/coresight/coresight-etm4x.h | 2 +- drivers/hwtracing/coresight/coresight-priv.h | 3 ++ .../coresight/coresight-self-hosted-trace.h | 9 ---- drivers/hwtracing/coresight/coresight-trbe.c | 10 ++++ 5 files changed, 56 insertions(+), 17 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index c5928f63475a0..9691c8897ecf4 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -268,10 +269,28 @@ struct etm4_enable_arg { */ static void etm4x_prohibit_trace(struct etmv4_drvdata *drvdata) { + u64 trfcr; + /* If the CPU doesn't support FEAT_TRF, nothing to do */ if (!drvdata->trfcr) return; - cpu_prohibit_trace(); + + trfcr = drvdata->trfcr & ~(TRFCR_ELx_ExTRE | TRFCR_ELx_E0TRE); + + write_trfcr(trfcr); + kvm_tracing_set_el1_configuration(trfcr); +} + +static u64 etm4x_get_kern_user_filter(struct etmv4_drvdata *drvdata) +{ + u64 trfcr = drvdata->trfcr; + + if (drvdata->config.mode & ETM_MODE_EXCL_KERN) + trfcr &= ~TRFCR_ELx_ExTRE; + if (drvdata->config.mode & ETM_MODE_EXCL_USER) + trfcr &= ~TRFCR_ELx_E0TRE; + + return trfcr; } /* @@ -286,18 +305,28 @@ static void etm4x_prohibit_trace(struct etmv4_drvdata *drvdata) */ static void etm4x_allow_trace(struct etmv4_drvdata *drvdata) { - u64 trfcr = drvdata->trfcr; + u64 trfcr, guest_trfcr; /* If the CPU doesn't support FEAT_TRF, nothing to do */ - if (!trfcr) + if (!drvdata->trfcr) return; - if (drvdata->config.mode & ETM_MODE_EXCL_KERN) - trfcr &= ~TRFCR_ELx_ExTRE; - if (drvdata->config.mode & ETM_MODE_EXCL_USER) - trfcr &= ~TRFCR_ELx_E0TRE; + if (drvdata->config.mode & ETM_MODE_EXCL_HOST) + trfcr = drvdata->trfcr & ~(TRFCR_ELx_ExTRE | TRFCR_ELx_E0TRE); + else + trfcr = etm4x_get_kern_user_filter(drvdata); write_trfcr(trfcr); + + /* Set filters for guests and pass to KVM */ + if (drvdata->config.mode & ETM_MODE_EXCL_GUEST) + guest_trfcr = drvdata->trfcr & ~(TRFCR_ELx_ExTRE | TRFCR_ELx_E0TRE); + else + guest_trfcr = etm4x_get_kern_user_filter(drvdata); + + /* TRFCR_EL1 doesn't have CX so mask it out. */ + guest_trfcr &= ~TRFCR_EL2_CX; + kvm_tracing_set_el1_configuration(guest_trfcr); } #ifdef CONFIG_ETM4X_IMPDEF_FEATURE @@ -705,6 +734,12 @@ static int etm4_parse_event_config(struct coresight_device *csdev, if (attr->exclude_user) config->mode = ETM_MODE_EXCL_USER; + if (attr->exclude_host) + config->mode |= ETM_MODE_EXCL_HOST; + + if (attr->exclude_guest) + config->mode |= ETM_MODE_EXCL_GUEST; + /* Always start from the default config */ etm4_set_default_config(config); diff --git a/drivers/hwtracing/coresight/coresight-etm4x.h b/drivers/hwtracing/coresight/coresight-etm4x.h index d8b8102d790cc..8c3e12554d958 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x.h +++ b/drivers/hwtracing/coresight/coresight-etm4x.h @@ -818,7 +818,7 @@ enum etm_impdef_type { * @s_ex_level: Secure ELs where tracing is supported. */ struct etmv4_config { - u32 mode; + u64 mode; u32 pe_sel; u32 cfg; u32 eventctrl0; diff --git a/drivers/hwtracing/coresight/coresight-priv.h b/drivers/hwtracing/coresight/coresight-priv.h index b758a42ed8c73..c8d1274f4adb5 100644 --- a/drivers/hwtracing/coresight/coresight-priv.h +++ b/drivers/hwtracing/coresight/coresight-priv.h @@ -40,6 +40,9 @@ #define ETM_MODE_EXCL_KERN BIT(30) #define ETM_MODE_EXCL_USER BIT(31) +#define ETM_MODE_EXCL_HOST BIT(32) +#define ETM_MODE_EXCL_GUEST BIT(33) + struct cs_pair_attribute { struct device_attribute attr; u32 lo_off; diff --git a/drivers/hwtracing/coresight/coresight-self-hosted-trace.h b/drivers/hwtracing/coresight/coresight-self-hosted-trace.h index 53840a2c41f2c..303d71911870f 100644 --- a/drivers/hwtracing/coresight/coresight-self-hosted-trace.h +++ b/drivers/hwtracing/coresight/coresight-self-hosted-trace.h @@ -21,13 +21,4 @@ static inline void write_trfcr(u64 val) isb(); } -static inline u64 cpu_prohibit_trace(void) -{ - u64 trfcr = read_trfcr(); - - /* Prohibit tracing at EL0 & the kernel EL */ - write_trfcr(trfcr & ~(TRFCR_ELx_ExTRE | TRFCR_ELx_E0TRE)); - /* Return the original value of the TRFCR */ - return trfcr; -} #endif /* __CORESIGHT_SELF_HOSTED_TRACE_H */ diff --git a/drivers/hwtracing/coresight/coresight-trbe.c b/drivers/hwtracing/coresight/coresight-trbe.c index a564bd5c8b62d..d6bf9b4898c42 100644 --- a/drivers/hwtracing/coresight/coresight-trbe.c +++ b/drivers/hwtracing/coresight/coresight-trbe.c @@ -1110,6 +1110,16 @@ static bool is_perf_trbe(struct perf_output_handle *handle) return true; } +static u64 cpu_prohibit_trace(void) +{ + u64 trfcr = read_trfcr(); + + /* Prohibit tracing at EL0 & the kernel EL */ + write_trfcr(trfcr & ~(TRFCR_ELx_ExTRE | TRFCR_ELx_E0TRE)); + /* Return the original value of the TRFCR */ + return trfcr; +} + static irqreturn_t arm_trbe_irq_handler(int irq, void *dev) { struct perf_output_handle **handle_ptr = dev; From ccf4097dff8e8f03bed02f82579e25e94978e966 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Sun, 12 Jan 2025 13:08:59 +0000 Subject: [PATCH 33/34] arm64/sysreg: Get rid of TRFCR_ELx SysregFields commit 01009b06a6b52d8439c55b530633a971c13b6cb2 upstream. There is no such thing as TRFCR_ELx in the architecture. What we have is TRFCR_EL1, for which TRFCR_EL12 is an accessor. Rename TRFCR_ELx_* to TRFCR_EL1_*, and fix the bit of code using these names. Similarly, TRFCR_EL12 is redefined as a mapping to TRFCR_EL1. Reviewed-by: James Clark Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/87cygsqgkh.wl-maz@kernel.org Cc: Suzuki K Poulose Cc: Mark Brown Cc: Will Deacon Cc: Catalin Marinas Signed-off-by: PvsNarasimha --- arch/arm64/tools/sysreg | 8 ++------ .../hwtracing/coresight/coresight-etm4x-core.c | 16 ++++++++-------- .../hwtracing/coresight/coresight-etm4x-sysfs.c | 10 +++++----- drivers/hwtracing/coresight/coresight-trbe.c | 2 +- 4 files changed, 16 insertions(+), 20 deletions(-) diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index 34f9fbf6460b6..2e41e043031bb 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -1760,7 +1760,7 @@ Sysreg CPACR_EL1 3 0 1 0 2 Fields CPACR_ELx EndSysreg -SysregFields TRFCR_ELx +Sysreg TRFCR_EL1 3 0 1 2 1 Res0 63:7 UnsignedEnum 6:5 TS 0b0001 VIRTUAL @@ -1770,10 +1770,6 @@ EndEnum Res0 4:2 Field 1 ExTRE Field 0 E0TRE -EndSysregFields - -Sysreg TRFCR_EL1 3 0 1 2 1 -Fields TRFCR_ELx EndSysreg Sysreg SMPRI_EL1 3 0 1 2 4 @@ -2503,7 +2499,7 @@ Fields ZCR_ELx EndSysreg Sysreg TRFCR_EL12 3 5 1 2 1 -Fields TRFCR_ELx +Fields TRFCR_EL1 EndSysreg Sysreg SMCR_EL12 3 5 1 2 6 diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index 9691c8897ecf4..18604ad5082e3 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -275,7 +275,7 @@ static void etm4x_prohibit_trace(struct etmv4_drvdata *drvdata) if (!drvdata->trfcr) return; - trfcr = drvdata->trfcr & ~(TRFCR_ELx_ExTRE | TRFCR_ELx_E0TRE); + trfcr = drvdata->trfcr & ~(TRFCR_EL1_ExTRE | TRFCR_EL1_E0TRE); write_trfcr(trfcr); kvm_tracing_set_el1_configuration(trfcr); @@ -286,9 +286,9 @@ static u64 etm4x_get_kern_user_filter(struct etmv4_drvdata *drvdata) u64 trfcr = drvdata->trfcr; if (drvdata->config.mode & ETM_MODE_EXCL_KERN) - trfcr &= ~TRFCR_ELx_ExTRE; + trfcr &= ~TRFCR_EL1_ExTRE; if (drvdata->config.mode & ETM_MODE_EXCL_USER) - trfcr &= ~TRFCR_ELx_E0TRE; + trfcr &= ~TRFCR_EL1_E0TRE; return trfcr; } @@ -312,7 +312,7 @@ static void etm4x_allow_trace(struct etmv4_drvdata *drvdata) return; if (drvdata->config.mode & ETM_MODE_EXCL_HOST) - trfcr = drvdata->trfcr & ~(TRFCR_ELx_ExTRE | TRFCR_ELx_E0TRE); + trfcr = drvdata->trfcr & ~(TRFCR_EL1_ExTRE | TRFCR_EL1_E0TRE); else trfcr = etm4x_get_kern_user_filter(drvdata); @@ -320,7 +320,7 @@ static void etm4x_allow_trace(struct etmv4_drvdata *drvdata) /* Set filters for guests and pass to KVM */ if (drvdata->config.mode & ETM_MODE_EXCL_GUEST) - guest_trfcr = drvdata->trfcr & ~(TRFCR_ELx_ExTRE | TRFCR_ELx_E0TRE); + guest_trfcr = drvdata->trfcr & ~(TRFCR_EL1_ExTRE | TRFCR_EL1_E0TRE); else guest_trfcr = etm4x_get_kern_user_filter(drvdata); @@ -1253,9 +1253,9 @@ static void cpu_detect_trace_filtering(struct etmv4_drvdata *drvdata) * tracing at the kernel EL and EL0, forcing to use the * virtual time as the timestamp. */ - trfcr = (TRFCR_ELx_TS_VIRTUAL | - TRFCR_ELx_ExTRE | - TRFCR_ELx_E0TRE); + trfcr = (TRFCR_EL1_TS_VIRTUAL | + TRFCR_EL1_ExTRE | + TRFCR_EL1_E0TRE); /* If we are running at EL2, allow tracing the CONTEXTIDR_EL2. */ if (is_kernel_in_hyp_mode()) diff --git a/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c b/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c index a9f19629f3f84..c767f8ae4cf1d 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c @@ -2319,11 +2319,11 @@ static ssize_t ts_source_show(struct device *dev, goto out; } - switch (drvdata->trfcr & TRFCR_ELx_TS_MASK) { - case TRFCR_ELx_TS_VIRTUAL: - case TRFCR_ELx_TS_GUEST_PHYSICAL: - case TRFCR_ELx_TS_PHYSICAL: - val = FIELD_GET(TRFCR_ELx_TS_MASK, drvdata->trfcr); + switch (drvdata->trfcr & TRFCR_EL1_TS_MASK) { + case TRFCR_EL1_TS_VIRTUAL: + case TRFCR_EL1_TS_GUEST_PHYSICAL: + case TRFCR_EL1_TS_PHYSICAL: + val = FIELD_GET(TRFCR_EL1_TS_MASK, drvdata->trfcr); break; default: val = -1; diff --git a/drivers/hwtracing/coresight/coresight-trbe.c b/drivers/hwtracing/coresight/coresight-trbe.c index d6bf9b4898c42..2a38bb58770ed 100644 --- a/drivers/hwtracing/coresight/coresight-trbe.c +++ b/drivers/hwtracing/coresight/coresight-trbe.c @@ -1115,7 +1115,7 @@ static u64 cpu_prohibit_trace(void) u64 trfcr = read_trfcr(); /* Prohibit tracing at EL0 & the kernel EL */ - write_trfcr(trfcr & ~(TRFCR_ELx_ExTRE | TRFCR_ELx_E0TRE)); + write_trfcr(trfcr & ~(TRFCR_EL1_ExTRE | TRFCR_EL1_E0TRE)); /* Return the original value of the TRFCR */ return trfcr; } From 2ad7f78067479e37075865c7bae7ae43e2aa9936 Mon Sep 17 00:00:00 2001 From: Leo Yan Date: Mon, 19 May 2025 18:49:44 +0100 Subject: [PATCH 34/34] coresight: etm4x: Fix timestamp bit field handling commit ee811bc733be5c57a2bfecdf2f6f5d4db466200a upstream. Timestamps in the trace data appear as all zeros on recent kernels, although the feature works correctly on old kernels (e.g., v6.12). Since commit c382ee674c8b ("arm64/sysreg/tools: Move TRFCR definitions to sysreg"), the TRFCR_ELx_TS_{VIRTUAL|GUEST_PHYSICAL|PHYSICAL} macros were updated to remove the bit shift. As a result, the driver no longer shifts bits when operates the timestamp field. Fix this by using the FIELD_PREP() and FIELD_GET() helpers. Reported-by: Tamas Zsoldos Fixes: c382ee674c8b ("arm64/sysreg/tools: Move TRFCR definitions to sysreg") Signed-off-by: Leo Yan Reviewed-by: James Clark Signed-off-by: Suzuki K Poulose Link: https://lore.kernel.org/r/20250519174945.2245271-2-leo.yan@arm.com Signed-off-by: PvsNarasimha --- drivers/hwtracing/coresight/coresight-etm4x-core.c | 2 +- drivers/hwtracing/coresight/coresight-etm4x-sysfs.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index 18604ad5082e3..0496f6519b560 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -1253,7 +1253,7 @@ static void cpu_detect_trace_filtering(struct etmv4_drvdata *drvdata) * tracing at the kernel EL and EL0, forcing to use the * virtual time as the timestamp. */ - trfcr = (TRFCR_EL1_TS_VIRTUAL | + trfcr = (FIELD_PREP(TRFCR_EL1_TS_MASK, TRFCR_EL1_TS_VIRTUAL) | TRFCR_EL1_ExTRE | TRFCR_EL1_E0TRE); diff --git a/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c b/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c index c767f8ae4cf1d..9c152344d9c09 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c @@ -2319,11 +2319,11 @@ static ssize_t ts_source_show(struct device *dev, goto out; } - switch (drvdata->trfcr & TRFCR_EL1_TS_MASK) { + val = FIELD_GET(TRFCR_EL1_TS_MASK, drvdata->trfcr); + switch (val) { case TRFCR_EL1_TS_VIRTUAL: case TRFCR_EL1_TS_GUEST_PHYSICAL: case TRFCR_EL1_TS_PHYSICAL: - val = FIELD_GET(TRFCR_EL1_TS_MASK, drvdata->trfcr); break; default: val = -1;