From 124421089d066b4c61aceb888b4b5fcd8bf64f6c Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Sat, 9 Dec 2023 01:02:51 +0000 Subject: [PATCH 01/85] arm64/sysreg: Add definition for ID_AA64PFR2_EL1 commit 9fb5dc53a1176402905a7dde6cd812bc01ce6831 upstream. DDI0601 2023-09 defines a new system register ID_AA64PFR2_EL1 which enumerates FPMR and some new MTE features. Add a definition of this register. Signed-off-by: Mark Brown Reviewed-by: Fuad Tabba Link: https://lore.kernel.org/r/20231209-b4-arm64-sysreg-additions-v1-5-45284e538474@kernel.org Signed-off-by: Will Deacon Signed-off-by: Bin Huang --- arch/arm64/tools/sysreg | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index fd72b60f65a9..6a3ce97784e0 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -1002,6 +1002,27 @@ UnsignedEnum 3:0 BT EndEnum EndSysreg +Sysreg ID_AA64PFR2_EL1 3 0 0 4 2 +Res0 63:36 +UnsignedEnum 35:32 FPMR + 0b0000 NI + 0b0001 IMP +EndEnum +Res0 31:12 +UnsignedEnum 11:8 MTEFAR + 0b0000 NI + 0b0001 IMP +EndEnum +UnsignedEnum 7:4 MTESTOREONLY + 0b0000 NI + 0b0001 IMP +EndEnum +UnsignedEnum 3:0 MTEPERM + 0b0000 NI + 0b0001 IMP +EndEnum +EndSysreg + Sysreg ID_AA64ZFR0_EL1 3 0 0 4 4 Res0 63:60 UnsignedEnum 59:56 F64MM From 723d6885e52a53971634c6b8c8346de70d1f963d Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Sat, 9 Dec 2023 01:02:52 +0000 Subject: [PATCH 02/85] arm64/sysreg: Update ID_AA64ISAR2_EL1 defintion for DDI0601 2023-09 commit 6e3dcfd139755d95f2c0d1f865f2e093d2b35c91 upstream. DDI0601 2023-09 defines some new fields in previously RES0 space in ID_AA64ISAR2_EL1, together with one new enum value. Update the system register definition to reflect this. Signed-off-by: Mark Brown Reviewed-by: Fuad Tabba Link: https://lore.kernel.org/r/20231209-b4-arm64-sysreg-additions-v1-6-45284e538474@kernel.org Signed-off-by: Will Deacon Signed-off-by: Bin Huang --- arch/arm64/tools/sysreg | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index 6a3ce97784e0..e3ceaf0f723b 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -1359,7 +1359,14 @@ EndEnum EndSysreg Sysreg ID_AA64ISAR2_EL1 3 0 0 6 2 -Res0 63:56 +UnsignedEnum 63:60 ATS1A + 0b0000 NI + 0b0001 IMP +EndEnum +UnsignedEnum 59:56 LUT + 0b0000 NI + 0b0001 IMP +EndEnum UnsignedEnum 55:52 CSSC 0b0000 NI 0b0001 IMP @@ -1368,7 +1375,19 @@ UnsignedEnum 51:48 RPRFM 0b0000 NI 0b0001 IMP EndEnum -Res0 47:32 +Res0 47:44 +UnsignedEnum 43:40 PRFMSLC + 0b0000 NI + 0b0001 IMP +EndEnum +UnsignedEnum 39:36 SYSINSTR_128 + 0b0000 NI + 0b0001 IMP +EndEnum +UnsignedEnum 35:32 SYSREG_128 + 0b0000 NI + 0b0001 IMP +EndEnum UnsignedEnum 31:28 CLRBHB 0b0000 NI 0b0001 IMP @@ -1392,6 +1411,7 @@ UnsignedEnum 15:12 APA3 0b0011 PAuth2 0b0100 FPAC 0b0101 FPACCOMBINE + 0b0110 PAuth_LR EndEnum UnsignedEnum 11:8 GPA3 0b0000 NI From 0d03de55493a9590bf77061e677ce29a8d50cadd Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Sat, 9 Dec 2023 01:02:53 +0000 Subject: [PATCH 03/85] arm64/sysreg: Add definition for ID_AA64ISAR3_EL1 commit b5aefb668701c2b019011f6f8fe29814b8529ecd upstream. DDI0601 2023-09 adds a new system register ID_AA64ISAR3_EL1 enumerating new floating point and TLB invalidation features. Add a defintion for it. Signed-off-by: Mark Brown Reviewed-by: Fuad Tabba Link: https://lore.kernel.org/r/20231209-b4-arm64-sysreg-additions-v1-7-45284e538474@kernel.org Signed-off-by: Will Deacon Signed-off-by: Bin Huang --- arch/arm64/tools/sysreg | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index e3ceaf0f723b..f838838d42b9 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -1427,6 +1427,23 @@ UnsignedEnum 3:0 WFxT EndEnum EndSysreg +Sysreg ID_AA64ISAR3_EL1 3 0 0 6 3 +Res0 63:12 +UnsignedEnum 11:8 TLBIW + 0b0000 NI + 0b0001 IMP +EndEnum +UnsignedEnum 7:4 FAMINMAX + 0b0000 NI + 0b0001 IMP +EndEnum +UnsignedEnum 3:0 CPA + 0b0000 NI + 0b0001 IMP + 0b0010 CPA2 +EndEnum +EndSysreg + Sysreg ID_AA64MMFR0_EL1 3 0 0 7 0 UnsignedEnum 63:60 ECV 0b0000 NI From 24fc9aad70a24ba4c93fa329b920ff29362643fe Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Sat, 9 Dec 2023 01:02:54 +0000 Subject: [PATCH 04/85] arm64/sysreg: Add definition for ID_AA64FPFR0_EL1 commit 9e4f409b07df14443ae4840f17f07e5025435e3d upstream. DDI0601 2023-09 defines a new feature register ID_AA64FPFR0_EL1 which enumerates a number of FP8 related features. Add a definition for it. Signed-off-by: Mark Brown Reviewed-by: Fuad Tabba Link: https://lore.kernel.org/r/20231209-b4-arm64-sysreg-additions-v1-8-45284e538474@kernel.org Signed-off-by: Will Deacon Signed-off-by: Bin Huang --- arch/arm64/tools/sysreg | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index f838838d42b9..e3184413f5b0 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -1127,6 +1127,35 @@ EndEnum Res0 31:0 EndSysreg +Sysreg ID_AA64FPFR0_EL1 3 0 0 4 7 +Res0 63:32 +UnsignedEnum 31 F8CVT + 0b0 NI + 0b1 IMP +EndEnum +UnsignedEnum 30 F8FMA + 0b0 NI + 0b1 IMP +EndEnum +UnsignedEnum 29 F8DP4 + 0b0 NI + 0b1 IMP +EndEnum +UnsignedEnum 28 F8DP2 + 0b0 NI + 0b1 IMP +EndEnum +Res0 27:2 +UnsignedEnum 1 F8E4M3 + 0b0 NI + 0b1 IMP +EndEnum +UnsignedEnum 0 F8E5M2 + 0b0 NI + 0b1 IMP +EndEnum +EndSysreg + Sysreg ID_AA64DFR0_EL1 3 0 0 5 0 Enum 63:60 HPMN0 0b0000 UNPREDICTABLE From fece9c8c834214049a47f9caa4669441130fe36c Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Sat, 9 Dec 2023 01:02:55 +0000 Subject: [PATCH 05/85] arm64/sysreg: Update ID_AA64SMFR0_EL1 definition for DDI0601 2023-09 commit 8afe582d77000ad244b66ed278aedc4ab5ee1634 upstream. The 2023-09 release of DDI0601 defines a number of new feature enumeration fields in ID_AA64SMFR0_EL1. Add these fields. Signed-off-by: Mark Brown Reviewed-by: Fuad Tabba Link: https://lore.kernel.org/r/20231209-b4-arm64-sysreg-additions-v1-9-45284e538474@kernel.org Signed-off-by: Will Deacon Signed-off-by: Bin Huang --- arch/arm64/tools/sysreg | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index e3184413f5b0..6ae482e38260 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -1075,7 +1075,11 @@ UnsignedEnum 63 FA64 0b0 NI 0b1 IMP EndEnum -Res0 62:60 +Res0 62:61 +UnsignedEnum 60 LUTv2 + 0b0 NI + 0b1 IMP +EndEnum UnsignedEnum 59:56 SMEver 0b0000 SME 0b0001 SME2 @@ -1103,7 +1107,14 @@ UnsignedEnum 42 F16F16 0b0 NI 0b1 IMP EndEnum -Res0 41:40 +UnsignedEnum 41 F8F16 + 0b0 NI + 0b1 IMP +EndEnum +UnsignedEnum 40 F8F32 + 0b0 NI + 0b1 IMP +EndEnum UnsignedEnum 39:36 I8I32 0b0000 NI 0b1111 IMP @@ -1124,7 +1135,20 @@ UnsignedEnum 32 F32F32 0b0 NI 0b1 IMP EndEnum -Res0 31:0 +Res0 31 +UnsignedEnum 30 SF8FMA + 0b0 NI + 0b1 IMP +EndEnum +UnsignedEnum 29 SF8DP4 + 0b0 NI + 0b1 IMP +EndEnum +UnsignedEnum 28 SF8DP2 + 0b0 NI + 0b1 IMP +EndEnum +Res0 27:0 EndSysreg Sysreg ID_AA64FPFR0_EL1 3 0 0 4 7 From 3fdec6497a108c1d7666a118b2d09a6302b0c761 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Sat, 9 Dec 2023 01:02:56 +0000 Subject: [PATCH 06/85] arm64/sysreg: Update SCTLR_EL1 for DDI0601 2023-09 commit a6052284a9f9bcfb982edfe00044ecdfdf72eaa7 upstream. DDI0601 2023-09 defines some new fields in SCTLR_EL1 controlling new MTE and floating point features. Update our sysreg definition to reflect these. Signed-off-by: Mark Brown Reviewed-by: Fuad Tabba Link: https://lore.kernel.org/r/20231209-b4-arm64-sysreg-additions-v1-10-45284e538474@kernel.org Signed-off-by: Will Deacon Signed-off-by: Bin Huang --- arch/arm64/tools/sysreg | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index 6ae482e38260..658049ac40e7 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -1826,7 +1826,8 @@ Field 63 TIDCP Field 62 SPINTMASK Field 61 NMI Field 60 EnTP2 -Res0 59:58 +Field 59 TCSO +Field 58 TCSO0 Field 57 EPAN Field 56 EnALS Field 55 EnAS0 @@ -1855,7 +1856,7 @@ EndEnum Field 37 ITFSB Field 36 BT1 Field 35 BT0 -Res0 34 +Field 34 EnFPM Field 33 MSCEn Field 32 CMOW Field 31 EnIA From 4aa9c59cb20fe58e27b53f0407ea439cbf63032e Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Sat, 9 Dec 2023 01:02:57 +0000 Subject: [PATCH 07/85] arm64/sysreg: Update HCRX_EL2 definition for DDI0601 2023-09 commit 126cb3a60d35cc2ce7db090b087e00ff85b12cfc upstream. DDI0601 2023-09 defines new fields in HCRX_EL2 controlling access to new system registers, update our definition of HCRX_EL2 to reflect this. Signed-off-by: Mark Brown Reviewed-by: Fuad Tabba Link: https://lore.kernel.org/r/20231209-b4-arm64-sysreg-additions-v1-11-45284e538474@kernel.org Signed-off-by: Will Deacon Signed-off-by: Bin Huang --- arch/arm64/tools/sysreg | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index 658049ac40e7..790bb2f54d28 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -2544,7 +2544,9 @@ Fields ZCR_ELx EndSysreg Sysreg HCRX_EL2 3 4 1 2 2 -Res0 63:23 +Res0 63:25 +Field 24 PACMEn +Field 23 EnFPM Field 22 GCSEn Field 21 EnIDCP128 Field 20 EnSDERR From 54a33dac184575b7bea5402a8c36d995492cfeb1 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Sat, 9 Dec 2023 01:02:58 +0000 Subject: [PATCH 08/85] arm64/sysreg: Add definition for FPMR commit e3a649ecf8b9253cb1d05ceb085544472b06446f upstream. DDI0601 2023-09 defines a new sysrem register FPMR (Floating Point Mode Register) which configures the new FP8 features. Add a definition of this register. Signed-off-by: Mark Brown Reviewed-by: Fuad Tabba Link: https://lore.kernel.org/r/20231209-b4-arm64-sysreg-additions-v1-12-45284e538474@kernel.org Signed-off-by: Will Deacon Signed-off-by: Bin Huang --- arch/arm64/tools/sysreg | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index 790bb2f54d28..0de3d0020fe9 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -2185,6 +2185,29 @@ Field 1 ZA Field 0 SM EndSysreg +Sysreg FPMR 3 3 4 4 2 +Res0 63:38 +Field 37:32 LSCALE2 +Field 31:24 NSCALE +Res0 23 +Field 22:16 LSCALE +Field 15 OSC +Field 14 OSM +Res0 13:9 +UnsignedEnum 8:6 F8D + 0b000 E5M2 + 0b001 E4M3 +EndEnum +UnsignedEnum 5:3 F8S2 + 0b000 E5M2 + 0b001 E4M3 +EndEnum +UnsignedEnum 2:0 F8S1 + 0b000 E5M2 + 0b001 E4M3 +EndEnum +EndSysreg + Sysreg SCTLR_EL2 3 4 1 0 0 Field 63 TIDCP Field 62 SPINTMASK From ae20126164885c68a0e76195a2092948e4fd1cd5 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Wed, 9 Apr 2025 17:40:07 +0100 Subject: [PATCH 09/85] arm64/fpsimd: Add fpsimd_save_and_flush_current_state() commit d3a181588df9401d319fe2dc24bf1a364a687cc2 upstream. When the current task's FPSIMD/SVE/SME state may be live on *any* CPU in the system, special care must be taken when manipulating that state, as this manipulation can race with preemption and/or asynchronous usage of FPSIMD/SVE/SME (e.g. kernel-mode NEON in softirq handlers). Even when manipulation is is protected with get_cpu_fpsimd_context() and get_cpu_fpsimd_context(), the logic necessary when the state is live on the current CPU can be wildly different from the logic necessary when the state is not live on the current CPU. A number of historical and extant issues result from failing to handle these cases consistetntly and/or correctly. To make it easier to get such manipulation correct, add a new fpsimd_save_and_flush_current_state() helper function, which ensures that the current task's state has been saved to memory and any stale state on any CPU has been "flushed" such that is not live on any CPU in the system. This will allow code to safely manipulate the saved state without risk of races. Subsequent patches will use the new function. Signed-off-by: Mark Rutland Cc: Marc Zyngier Cc: Mark Brown Cc: Will Deacon Reviewed-by: Mark Brown Link: https://lore.kernel.org/r/20250409164010.3480271-11-mark.rutland@arm.com Signed-off-by: Catalin Marinas [ bh: veLinux 6.6.151 uses ownership-aware fpsimd_save() as a substitute for the unavailable upstream fpsimd_save_user_state() helper. ] Signed-off-by: Bin Huang --- arch/arm64/include/asm/fpsimd.h | 1 + arch/arm64/kernel/fpsimd.c | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h index 1a549100deb7..9bc692cd0690 100644 --- a/arch/arm64/include/asm/fpsimd.h +++ b/arch/arm64/include/asm/fpsimd.h @@ -101,6 +101,7 @@ DECLARE_PER_CPU(struct cpu_fp_state, fpsimd_last_state); extern void fpsimd_bind_state_to_cpu(struct cpu_fp_state *fp_state); extern void fpsimd_flush_task_state(struct task_struct *target); +extern void fpsimd_save_and_flush_current_state(void); extern void fpsimd_save_and_flush_cpu_state(void); static inline bool thread_sm_enabled(struct thread_struct *thread) diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c index 1ec38ee07dbd..210c6181cb26 100644 --- a/arch/arm64/kernel/fpsimd.c +++ b/arch/arm64/kernel/fpsimd.c @@ -1856,6 +1856,17 @@ void fpsimd_flush_task_state(struct task_struct *t) barrier(); } +void fpsimd_save_and_flush_current_state(void) +{ + if (!system_supports_fpsimd()) + return; + + get_cpu_fpsimd_context(); + fpsimd_save(); + fpsimd_flush_task_state(current); + put_cpu_fpsimd_context(); +} + /* * Invalidate any task's FPSIMD state that is present on this cpu. * The FPSIMD context should be acquired with get_cpu_fpsimd_context() From 973a9143beddea773884b58ad673e62b8d115a56 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Wed, 9 Apr 2025 17:40:08 +0100 Subject: [PATCH 10/85] arm64/fpsimd: signal32: Always save+flush state early commit 3aa4d74438afa1324513a7a6bc30f57e8727ad86 upstream. There are several issues with the way the native signal handling code manipulates FPSIMD/SVE/SME state. To fix those issues, subsequent patches will rework the native signal handling code to always save+flush the current task's FPSIMD/SVE/SME state before manipulating that state. In preparation for those changes, rework the compat signal handling code to save+flush the current task's FPSIMD state before manipulating it. Subsequent patches will remove fpsimd_signal_preserve_current_state() and fpsimd_update_current_state(). Compat tasks can only have FPSIMD state, and cannot have any SVE or SME state. Thus, the SVE state manipulation present in fpsimd_signal_preserve_current_state() and fpsimd_update_current_state() is not necessary, and it is safe to directly manipulate current->thread.uw.fpsimd_state once it has been saved+flushed. Use fpsimd_save_and_flush_current_state() to save+flush the state for both signal delivery and signal return, before the state is manipulated in any way. While it would be safe for compat_restore_vfp_context() to use fpsimd_flush_task_state(current), there are several extant issues in the native signal code resulting from incorrect use of fpsimd_flush_task_state(), and for consistency it is preferable to use fpsimd_save_and_flush_current_state(). Signed-off-by: Mark Rutland Cc: Marc Zyngier Cc: Mark Brown Cc: Will Deacon Reviewed-by: Mark Brown Link: https://lore.kernel.org/r/20250409164010.3480271-12-mark.rutland@arm.com Signed-off-by: Catalin Marinas Signed-off-by: Bin Huang --- arch/arm64/kernel/signal32.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/arch/arm64/kernel/signal32.c b/arch/arm64/kernel/signal32.c index bbd542704730..19c4987efbfb 100644 --- a/arch/arm64/kernel/signal32.c +++ b/arch/arm64/kernel/signal32.c @@ -103,7 +103,7 @@ static int compat_preserve_vfp_context(struct compat_vfp_sigframe __user *frame) * Note that this also saves V16-31, which aren't visible * in AArch32. */ - fpsimd_signal_preserve_current_state(); + fpsimd_save_and_flush_current_state(); /* Place structure header on the stack */ __put_user_error(magic, &frame->magic, err); @@ -169,14 +169,17 @@ static int compat_restore_vfp_context(struct compat_vfp_sigframe __user *frame) fpsimd.fpsr = fpscr & VFP_FPSCR_STAT_MASK; fpsimd.fpcr = fpscr & VFP_FPSCR_CTRL_MASK; + if (err) + return -EFAULT; + /* * We don't need to touch the exception register, so * reload the hardware state. */ - if (!err) - fpsimd_update_current_state(&fpsimd); + fpsimd_save_and_flush_current_state(); + current->thread.uw.fpsimd_state = fpsimd; - return err ? -EFAULT : 0; + return 0; } static int compat_restore_sigframe(struct pt_regs *regs, From 446a96e3786e01ecec72e4a9db51c027209ace69 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Fri, 15 Sep 2023 13:18:06 +0100 Subject: [PATCH 11/85] arm64/sve: Report FEAT_SVE_B16B16 to userspace commit 5d5b4e8c2d9ec12a5cc013b82c1873cd387e0ddf upstream. SVE 2.1 introduced a new feature FEAT_SVE_B16B16 which adds instructions supporting the BFloat16 floating point format. Report this to userspace through the ID registers and hwcap. Reported-by: Peter Maydell Signed-off-by: Mark Brown Link: https://lore.kernel.org/r/20230915-arm64-zfr-b16b16-el0-v1-1-f9aba807bdb5@kernel.org Signed-off-by: Catalin Marinas [ bh: veLinux 6.6.151 uses HWCAP_CAP_MATCH_ID() for SVE_B16B16 capability checks, requiring both system_supports_sve() and ID-register feature.] Signed-off-by: Bin Huang --- Documentation/arch/arm64/cpu-feature-registers.rst | 2 ++ Documentation/arch/arm64/elf_hwcaps.rst | 3 +++ arch/arm64/include/asm/hwcap.h | 1 + arch/arm64/include/uapi/asm/hwcap.h | 1 + arch/arm64/kernel/cpufeature.c | 3 +++ arch/arm64/kernel/cpuinfo.c | 1 + arch/arm64/tools/sysreg | 6 +++++- 7 files changed, 16 insertions(+), 1 deletion(-) diff --git a/Documentation/arch/arm64/cpu-feature-registers.rst b/Documentation/arch/arm64/cpu-feature-registers.rst index de6d8a4790e2..44f9bd78539d 100644 --- a/Documentation/arch/arm64/cpu-feature-registers.rst +++ b/Documentation/arch/arm64/cpu-feature-registers.rst @@ -268,6 +268,8 @@ infrastructure: +------------------------------+---------+---------+ | SHA3 | [35-32] | y | +------------------------------+---------+---------+ + | B16B16 | [27-24] | y | + +------------------------------+---------+---------+ | BF16 | [23-20] | y | +------------------------------+---------+---------+ | BitPerm | [19-16] | y | diff --git a/Documentation/arch/arm64/elf_hwcaps.rst b/Documentation/arch/arm64/elf_hwcaps.rst index c6e988a07c54..9a256701dc62 100644 --- a/Documentation/arch/arm64/elf_hwcaps.rst +++ b/Documentation/arch/arm64/elf_hwcaps.rst @@ -320,6 +320,9 @@ HWCAP2_MOPS HWCAP2_HBC Functionality implied by ID_AA64ISAR2_EL1.BC == 0b0001. +HWCAP2_SVE_B16B16 + Functionality implied by ID_AA64ZFR0_EL1.B16B16 == 0b0001. + HWCAP3_LS64 Functionality implied by ID_AA64ISAR1_EL1.LS64 == 0b0001. diff --git a/arch/arm64/include/asm/hwcap.h b/arch/arm64/include/asm/hwcap.h index 69fd710c27a3..064c86b0dc99 100644 --- a/arch/arm64/include/asm/hwcap.h +++ b/arch/arm64/include/asm/hwcap.h @@ -139,6 +139,7 @@ #define KERNEL_HWCAP_SME_F16F16 __khwcap2_feature(SME_F16F16) #define KERNEL_HWCAP_MOPS __khwcap2_feature(MOPS) #define KERNEL_HWCAP_HBC __khwcap2_feature(HBC) +#define KERNEL_HWCAP_SVE_B16B16 __khwcap2_feature(SVE_B16B16) #define __khwcap3_feature(x) (const_ilog2(HWCAP3_ ## x) + 128) #define KERNEL_HWCAP_LS64 __khwcap3_feature(LS64) diff --git a/arch/arm64/include/uapi/asm/hwcap.h b/arch/arm64/include/uapi/asm/hwcap.h index 4cabccfd7d84..bd613c7df482 100644 --- a/arch/arm64/include/uapi/asm/hwcap.h +++ b/arch/arm64/include/uapi/asm/hwcap.h @@ -104,6 +104,7 @@ #define HWCAP2_SME_F16F16 (1UL << 42) #define HWCAP2_MOPS (1UL << 43) #define HWCAP2_HBC (1UL << 44) +#define HWCAP2_SVE_B16B16 (1UL << 45) /* * HWCAP3 flags - for AT_HWCAP3 diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 7da7b9473561..3c0b7ae3797e 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -280,6 +280,8 @@ static const struct arm64_ftr_bits ftr_id_aa64zfr0[] = { FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_EL1_SM4_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE), FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_EL1_SHA3_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE), + FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_EL1_B16B16_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE), FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_EL1_BF16_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE), @@ -2994,6 +2996,7 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, AES, IMP, CAP_HWCAP, KERNEL_HWCAP_SVEAES), HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, AES, PMULL128, CAP_HWCAP, KERNEL_HWCAP_SVEPMULL), HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, BitPerm, IMP, CAP_HWCAP, KERNEL_HWCAP_SVEBITPERM), + HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, B16B16, IMP, CAP_HWCAP, KERNEL_HWCAP_SVE_B16B16), HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, BF16, IMP, CAP_HWCAP, KERNEL_HWCAP_SVEBF16), HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, BF16, EBF16, CAP_HWCAP, KERNEL_HWCAP_SVE_EBF16), HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, SHA3, IMP, CAP_HWCAP, KERNEL_HWCAP_SVESHA3), diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c index 21a18237fc4b..b5358e3881f0 100644 --- a/arch/arm64/kernel/cpuinfo.c +++ b/arch/arm64/kernel/cpuinfo.c @@ -129,6 +129,7 @@ static const char *const hwcap_str[] = { [KERNEL_HWCAP_SME_F16F16] = "smef16f16", [KERNEL_HWCAP_MOPS] = "mops", [KERNEL_HWCAP_HBC] = "hbc", + [KERNEL_HWCAP_SVE_B16B16] = "sveb16b16", }; #ifdef CONFIG_COMPAT diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index 0de3d0020fe9..03de36189a0d 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -1047,7 +1047,11 @@ UnsignedEnum 35:32 SHA3 0b0000 NI 0b0001 IMP EndEnum -Res0 31:24 +Res0 31:28 +UnsignedEnum 27:24 B16B16 + 0b0000 NI + 0b0001 IMP +EndEnum UnsignedEnum 23:20 BF16 0b0000 NI 0b0001 IMP From 646c5292f1cebaec7d07c69c94d7a29a123bb813 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Fri, 15 Sep 2023 13:18:07 +0100 Subject: [PATCH 12/85] kselftest/arm64: Verify HWCAP2_SVE_B16B16 commit 3accaef1f61e0c5531ae9e92ff0d66981cdf1516 upstream. Validate that SVE B16B16 support is reported correctly and consistently to userspace. Signed-off-by: Mark Brown Link: https://lore.kernel.org/r/20230915-arm64-zfr-b16b16-el0-v1-2-f9aba807bdb5@kernel.org Signed-off-by: Catalin Marinas Signed-off-by: Bin Huang --- tools/testing/selftests/arm64/abi/hwcap.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tools/testing/selftests/arm64/abi/hwcap.c b/tools/testing/selftests/arm64/abi/hwcap.c index 311a2a65f7cf..55b9053afaae 100644 --- a/tools/testing/selftests/arm64/abi/hwcap.c +++ b/tools/testing/selftests/arm64/abi/hwcap.c @@ -226,6 +226,12 @@ static void sveaes_sigill(void) asm volatile(".inst 0x4522e400" : : : "z0"); } +static void sveb16b16_sigill(void) +{ + /* BFADD ZA.H[W0, 0], {Z0.H-Z1.H} */ + asm volatile(".inst 0xC1E41C00" : : : ); +} + static void svepmull_sigill(void) { /* PMULLB Z0.Q, Z0.D, Z0.D */ @@ -493,6 +499,13 @@ static const struct hwcap_data { .cpuinfo = "sveaes", .sigill_fn = sveaes_sigill, }, + { + .name = "SVE2 B16B16", + .at_hwcap = AT_HWCAP2, + .hwcap_bit = HWCAP2_SVE_B16B16, + .cpuinfo = "sveb16b16", + .sigill_fn = sveb16b16_sigill, + }, { .name = "SVE2 PMULL", .at_hwcap = AT_HWCAP2, From eb84abbb4c67965176833f31a5d2183a5309e10b Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Wed, 6 Mar 2024 23:14:46 +0000 Subject: [PATCH 13/85] arm64/cpufeature: Hook new identification registers up to cpufeature commit cc9f69a3dad3b64b299dc2d5f95935fe16cb8b79 upstream. The 2023 architecture extensions have defined several new ID registers, hook them up to the cpufeature code so we can add feature checks and hwcaps based on their contents. Signed-off-by: Mark Brown Link: https://lore.kernel.org/r/20240306-arm64-2023-dpisa-v5-1-c568edc8ed7f@kernel.org Signed-off-by: Catalin Marinas Signed-off-by: Bin Huang --- arch/arm64/include/asm/cpu.h | 3 +++ arch/arm64/kernel/cpufeature.c | 28 ++++++++++++++++++++++++++++ arch/arm64/kernel/cpuinfo.c | 3 +++ 3 files changed, 34 insertions(+) diff --git a/arch/arm64/include/asm/cpu.h b/arch/arm64/include/asm/cpu.h index 887bd0d992bb..8d0f07cf4b90 100644 --- a/arch/arm64/include/asm/cpu.h +++ b/arch/arm64/include/asm/cpu.h @@ -52,14 +52,17 @@ struct cpuinfo_arm64 { u64 reg_id_aa64isar0; u64 reg_id_aa64isar1; u64 reg_id_aa64isar2; + u64 reg_id_aa64isar3; u64 reg_id_aa64mmfr0; u64 reg_id_aa64mmfr1; u64 reg_id_aa64mmfr2; u64 reg_id_aa64mmfr3; u64 reg_id_aa64pfr0; u64 reg_id_aa64pfr1; + u64 reg_id_aa64pfr2; u64 reg_id_aa64zfr0; u64 reg_id_aa64smfr0; + u64 reg_id_aa64fpfr0; struct cpuinfo_32bit aarch32; diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 3c0b7ae3797e..25cb5bd18180 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -236,6 +236,10 @@ static const struct arm64_ftr_bits ftr_id_aa64isar2[] = { ARM64_FTR_END, }; +static const struct arm64_ftr_bits ftr_id_aa64isar3[] = { + ARM64_FTR_END, +}; + static const struct arm64_ftr_bits ftr_id_aa64pfr0[] = { ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_CSV3_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_CSV2_SHIFT, 4, 0), @@ -269,6 +273,10 @@ static const struct arm64_ftr_bits ftr_id_aa64pfr1[] = { ARM64_FTR_END, }; +static const struct arm64_ftr_bits ftr_id_aa64pfr2[] = { + ARM64_FTR_END, +}; + static const struct arm64_ftr_bits ftr_id_aa64zfr0[] = { ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE), FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_EL1_F64MM_SHIFT, 4, 0), @@ -321,6 +329,10 @@ static const struct arm64_ftr_bits ftr_id_aa64smfr0[] = { ARM64_FTR_END, }; +static const struct arm64_ftr_bits ftr_id_aa64fpfr0[] = { + ARM64_FTR_END, +}; + static const struct arm64_ftr_bits ftr_id_aa64mmfr0[] = { ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_ECV_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_FGT_SHIFT, 4, 0), @@ -717,10 +729,12 @@ static const struct __ftr_reg_entry { &id_aa64pfr0_override), ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64PFR1_EL1, ftr_id_aa64pfr1, &id_aa64pfr1_override), + ARM64_FTR_REG(SYS_ID_AA64PFR2_EL1, ftr_id_aa64pfr2), ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64ZFR0_EL1, ftr_id_aa64zfr0, &id_aa64zfr0_override), ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64SMFR0_EL1, ftr_id_aa64smfr0, &id_aa64smfr0_override), + ARM64_FTR_REG(SYS_ID_AA64FPFR0_EL1, ftr_id_aa64fpfr0), /* Op1 = 0, CRn = 0, CRm = 5 */ ARM64_FTR_REG(SYS_ID_AA64DFR0_EL1, ftr_id_aa64dfr0), @@ -732,6 +746,7 @@ static const struct __ftr_reg_entry { &id_aa64isar1_override), ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64ISAR2_EL1, ftr_id_aa64isar2, &id_aa64isar2_override), + ARM64_FTR_REG(SYS_ID_AA64ISAR3_EL1, ftr_id_aa64isar3), /* Op1 = 0, CRn = 0, CRm = 7 */ ARM64_FTR_REG(SYS_ID_AA64MMFR0_EL1, ftr_id_aa64mmfr0), @@ -1031,14 +1046,17 @@ void __init init_cpu_features(struct cpuinfo_arm64 *info) init_cpu_ftr_reg(SYS_ID_AA64ISAR0_EL1, info->reg_id_aa64isar0); init_cpu_ftr_reg(SYS_ID_AA64ISAR1_EL1, info->reg_id_aa64isar1); init_cpu_ftr_reg(SYS_ID_AA64ISAR2_EL1, info->reg_id_aa64isar2); + init_cpu_ftr_reg(SYS_ID_AA64ISAR3_EL1, info->reg_id_aa64isar3); init_cpu_ftr_reg(SYS_ID_AA64MMFR0_EL1, info->reg_id_aa64mmfr0); init_cpu_ftr_reg(SYS_ID_AA64MMFR1_EL1, info->reg_id_aa64mmfr1); init_cpu_ftr_reg(SYS_ID_AA64MMFR2_EL1, info->reg_id_aa64mmfr2); init_cpu_ftr_reg(SYS_ID_AA64MMFR3_EL1, info->reg_id_aa64mmfr3); init_cpu_ftr_reg(SYS_ID_AA64PFR0_EL1, info->reg_id_aa64pfr0); init_cpu_ftr_reg(SYS_ID_AA64PFR1_EL1, info->reg_id_aa64pfr1); + init_cpu_ftr_reg(SYS_ID_AA64PFR2_EL1, info->reg_id_aa64pfr2); init_cpu_ftr_reg(SYS_ID_AA64ZFR0_EL1, info->reg_id_aa64zfr0); init_cpu_ftr_reg(SYS_ID_AA64SMFR0_EL1, info->reg_id_aa64smfr0); + init_cpu_ftr_reg(SYS_ID_AA64FPFR0_EL1, info->reg_id_aa64fpfr0); if (id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0)) init_32bit_cpu_features(&info->aarch32); @@ -1276,6 +1294,8 @@ void update_cpu_features(int cpu, info->reg_id_aa64isar1, boot->reg_id_aa64isar1); taint |= check_update_ftr_reg(SYS_ID_AA64ISAR2_EL1, cpu, info->reg_id_aa64isar2, boot->reg_id_aa64isar2); + taint |= check_update_ftr_reg(SYS_ID_AA64ISAR3_EL1, cpu, + info->reg_id_aa64isar3, boot->reg_id_aa64isar3); /* * Differing PARange support is fine as long as all peripherals and @@ -1295,12 +1315,17 @@ void update_cpu_features(int cpu, info->reg_id_aa64pfr0, boot->reg_id_aa64pfr0); taint |= check_update_ftr_reg(SYS_ID_AA64PFR1_EL1, cpu, info->reg_id_aa64pfr1, boot->reg_id_aa64pfr1); + taint |= check_update_ftr_reg(SYS_ID_AA64PFR2_EL1, cpu, + info->reg_id_aa64pfr2, boot->reg_id_aa64pfr2); taint |= check_update_ftr_reg(SYS_ID_AA64ZFR0_EL1, cpu, info->reg_id_aa64zfr0, boot->reg_id_aa64zfr0); taint |= check_update_ftr_reg(SYS_ID_AA64SMFR0_EL1, cpu, info->reg_id_aa64smfr0, boot->reg_id_aa64smfr0); + taint |= check_update_ftr_reg(SYS_ID_AA64FPFR0_EL1, cpu, + info->reg_id_aa64fpfr0, boot->reg_id_aa64fpfr0); + if (IS_ENABLED(CONFIG_ARM64_SVE) && id_aa64pfr0_sve(read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1))) { @@ -1420,8 +1445,10 @@ u64 __read_sysreg_by_encoding(u32 sys_id) read_sysreg_case(SYS_ID_AA64PFR0_EL1); read_sysreg_case(SYS_ID_AA64PFR1_EL1); + read_sysreg_case(SYS_ID_AA64PFR2_EL1); read_sysreg_case(SYS_ID_AA64ZFR0_EL1); read_sysreg_case(SYS_ID_AA64SMFR0_EL1); + read_sysreg_case(SYS_ID_AA64FPFR0_EL1); read_sysreg_case(SYS_ID_AA64DFR0_EL1); read_sysreg_case(SYS_ID_AA64DFR1_EL1); read_sysreg_case(SYS_ID_AA64MMFR0_EL1); @@ -1431,6 +1458,7 @@ u64 __read_sysreg_by_encoding(u32 sys_id) read_sysreg_case(SYS_ID_AA64ISAR0_EL1); read_sysreg_case(SYS_ID_AA64ISAR1_EL1); read_sysreg_case(SYS_ID_AA64ISAR2_EL1); + read_sysreg_case(SYS_ID_AA64ISAR3_EL1); read_sysreg_case(SYS_CNTFRQ_EL0); read_sysreg_case(SYS_CTR_EL0); diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c index b5358e3881f0..56cc3ba8545e 100644 --- a/arch/arm64/kernel/cpuinfo.c +++ b/arch/arm64/kernel/cpuinfo.c @@ -448,14 +448,17 @@ static void __cpuinfo_store_cpu(struct cpuinfo_arm64 *info) info->reg_id_aa64isar0 = read_cpuid(ID_AA64ISAR0_EL1); info->reg_id_aa64isar1 = read_cpuid(ID_AA64ISAR1_EL1); info->reg_id_aa64isar2 = read_cpuid(ID_AA64ISAR2_EL1); + info->reg_id_aa64isar3 = read_cpuid(ID_AA64ISAR3_EL1); info->reg_id_aa64mmfr0 = read_cpuid(ID_AA64MMFR0_EL1); info->reg_id_aa64mmfr1 = read_cpuid(ID_AA64MMFR1_EL1); info->reg_id_aa64mmfr2 = read_cpuid(ID_AA64MMFR2_EL1); info->reg_id_aa64mmfr3 = read_cpuid(ID_AA64MMFR3_EL1); info->reg_id_aa64pfr0 = read_cpuid(ID_AA64PFR0_EL1); info->reg_id_aa64pfr1 = read_cpuid(ID_AA64PFR1_EL1); + info->reg_id_aa64pfr2 = read_cpuid(ID_AA64PFR2_EL1); info->reg_id_aa64zfr0 = read_cpuid(ID_AA64ZFR0_EL1); info->reg_id_aa64smfr0 = read_cpuid(ID_AA64SMFR0_EL1); + info->reg_id_aa64fpfr0 = read_cpuid(ID_AA64FPFR0_EL1); if (id_aa64pfr1_mte(info->reg_id_aa64pfr1)) info->reg_gmid = read_cpuid(GMID_EL1); From 8d1eae29dbb5d4dc3c6180c5a35f90296ff00e39 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Wed, 6 Mar 2024 23:14:47 +0000 Subject: [PATCH 14/85] arm64/fpsimd: Enable host kernel access to FPMR commit b6c0b424cb91a864e62533a6520743ddcdde5270 upstream. FEAT_FPMR provides a new generally accessible architectural register FPMR. This is only accessible to EL0 and EL1 when HCRX_EL2.EnFPM is set to 1, do this when the host is running. The guest part will be done along with context switching the new register and exposing it via guest management. Acked-by: Marc Zyngier Signed-off-by: Mark Brown Link: https://lore.kernel.org/r/20240306-arm64-2023-dpisa-v5-2-c568edc8ed7f@kernel.org Signed-off-by: Catalin Marinas Signed-off-by: Bin Huang --- arch/arm64/include/asm/kvm_arm.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h index 5e8a0835470d..60101b2f51e0 100644 --- a/arch/arm64/include/asm/kvm_arm.h +++ b/arch/arm64/include/asm/kvm_arm.h @@ -103,7 +103,7 @@ #define HCR_HOST_VHE_FLAGS (HCR_RW | HCR_TGE | HCR_E2H) #define HCRX_GUEST_FLAGS (HCRX_EL2_SMPME | HCRX_EL2_TCR2En) -#define HCRX_HOST_FLAGS (HCRX_EL2_MSCEn | HCRX_EL2_TCR2En) +#define HCRX_HOST_FLAGS (HCRX_EL2_MSCEn | HCRX_EL2_TCR2En | HCRX_EL2_EnFPM) /* TCR_EL2 Registers bits */ #define TCR_EL2_DS (1UL << 32) From c154713de4bfdef3004414f936a8fd6baaf78073 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Wed, 6 Mar 2024 23:14:48 +0000 Subject: [PATCH 15/85] arm64/fpsimd: Support FEAT_FPMR commit 203f2b95a882dc46dd9873562167db69a1f61711 upstream. FEAT_FPMR defines a new EL0 accessible register FPMR use to configure the FP8 related features added to the architecture at the same time. Detect support for this register and context switch it for EL0 when present. Due to the sharing of responsibility for saving floating point state between the host kernel and KVM FP8 support is not yet implemented in KVM and a stub similar to that used for SVCR is provided for FPMR in order to avoid bisection issues. To make it easier to share host state with the hypervisor we store FPMR as a hardened usercopy field in uw (along with some padding). Signed-off-by: Mark Brown Link: https://lore.kernel.org/r/20240306-arm64-2023-dpisa-v5-3-c568edc8ed7f@kernel.org Signed-off-by: Catalin Marinas Signed-off-by: Bin Huang --- arch/arm64/include/asm/cpufeature.h | 5 +++++ arch/arm64/include/asm/fpsimd.h | 2 ++ arch/arm64/include/asm/kvm_host.h | 1 + arch/arm64/include/asm/processor.h | 4 ++++ arch/arm64/kernel/cpufeature.c | 9 +++++++++ arch/arm64/kernel/fpsimd.c | 13 +++++++++++++ arch/arm64/kvm/fpsimd.c | 1 + arch/arm64/tools/cpucaps | 1 + 8 files changed, 36 insertions(+) diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index 559fb477f07c..4f9b0886b747 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -766,6 +766,11 @@ static __always_inline bool system_supports_tpidr2(void) return system_supports_sme(); } +static __always_inline bool system_supports_fpmr(void) +{ + return alternative_has_cap_unlikely(ARM64_HAS_FPMR); +} + static __always_inline bool system_supports_cnp(void) { return alternative_has_cap_unlikely(ARM64_HAS_CNP); diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h index 9bc692cd0690..7eb37449a316 100644 --- a/arch/arm64/include/asm/fpsimd.h +++ b/arch/arm64/include/asm/fpsimd.h @@ -90,6 +90,7 @@ struct cpu_fp_state { void *sve_state; void *sme_state; u64 *svcr; + u64 *fpmr; unsigned int sve_vl; unsigned int sme_vl; enum fp_type *fp_type; @@ -158,6 +159,7 @@ extern void cpu_enable_sve(const struct arm64_cpu_capabilities *__unused); extern void cpu_enable_sme(const struct arm64_cpu_capabilities *__unused); extern void cpu_enable_sme2(const struct arm64_cpu_capabilities *__unused); extern void cpu_enable_fa64(const struct arm64_cpu_capabilities *__unused); +extern void cpu_enable_fpmr(const struct arm64_cpu_capabilities *__unused); extern u64 read_zcr_features(void); extern u64 read_smcr_features(void); diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 582ba39dba04..12037774bf02 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -480,6 +480,7 @@ struct kvm_vcpu_arch { enum fp_type fp_type; unsigned int sve_max_vl; u64 svcr; + u64 fpmr; /* Stage 2 paging state used by the hardware on next switch */ struct kvm_s2_mmu *hw_mmu; diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h index e5bc54522e71..62d932d05bff 100644 --- a/arch/arm64/include/asm/processor.h +++ b/arch/arm64/include/asm/processor.h @@ -155,6 +155,8 @@ struct thread_struct { struct { unsigned long tp_value; /* TLS register */ unsigned long tp2_value; + u64 fpmr; + unsigned long pad; struct user_fpsimd_state fpsimd_state; } uw; @@ -250,6 +252,8 @@ static inline void arch_thread_struct_whitelist(unsigned long *offset, BUILD_BUG_ON(sizeof_field(struct thread_struct, uw) != sizeof_field(struct thread_struct, uw.tp_value) + sizeof_field(struct thread_struct, uw.tp2_value) + + sizeof_field(struct thread_struct, uw.fpmr) + + sizeof_field(struct thread_struct, uw.pad) + sizeof_field(struct thread_struct, uw.fpsimd_state)); *offset = offsetof(struct thread_struct, uw); diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 25cb5bd18180..de4f95e22c76 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -274,6 +274,7 @@ static const struct arm64_ftr_bits ftr_id_aa64pfr1[] = { }; static const struct arm64_ftr_bits ftr_id_aa64pfr2[] = { + ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR2_EL1_FPMR_SHIFT, 4, 0), ARM64_FTR_END, }; @@ -2899,6 +2900,14 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .cpu_enable = cpu_enable_ls64_v, ARM64_CPUID_FIELDS(ID_AA64ISAR1_EL1, LS64, LS64_V) }, + { + .desc = "FPMR", + .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .capability = ARM64_HAS_FPMR, + .matches = has_cpuid_feature, + .cpu_enable = cpu_enable_fpmr, + ARM64_CPUID_FIELDS(ID_AA64PFR2_EL1, FPMR, IMP) + }, {}, }; diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c index 210c6181cb26..cd5d362a3e50 100644 --- a/arch/arm64/kernel/fpsimd.c +++ b/arch/arm64/kernel/fpsimd.c @@ -385,6 +385,9 @@ static void task_fpsimd_load(void) WARN_ON(!system_supports_fpsimd()); WARN_ON(!have_cpu_fpsimd_context()); + if (system_supports_fpmr()) + write_sysreg_s(current->thread.uw.fpmr, SYS_FPMR); + if (system_supports_sve() || system_supports_sme()) { switch (current->thread.fp_type) { case FP_STATE_FPSIMD: @@ -472,6 +475,9 @@ static void fpsimd_save(void) if (test_thread_flag(TIF_FOREIGN_FPSTATE)) return; + if (system_supports_fpmr()) + *(last->fpmr) = read_sysreg_s(SYS_FPMR); + /* * Save SVE state if it is live. * @@ -719,6 +725,12 @@ static void sve_to_fpsimd(struct task_struct *task) } } +void cpu_enable_fpmr(const struct arm64_cpu_capabilities *__always_unused p) +{ + write_sysreg_s(read_sysreg_s(SYS_SCTLR_EL1) | SCTLR_EL1_EnFPM_MASK, + SYS_SCTLR_EL1); +} + #ifdef CONFIG_ARM64_SVE /* * Call __sve_free() directly only if you know task can't be scheduled @@ -1728,6 +1740,7 @@ static void fpsimd_bind_task_to_cpu(void) last->sve_vl = task_get_sve_vl(current); last->sme_vl = task_get_sme_vl(current); last->svcr = ¤t->thread.svcr; + last->fpmr = ¤t->thread.uw.fpmr; last->fp_type = ¤t->thread.fp_type; last->to_save = FP_STATE_CURRENT; current->thread.fpsimd_cpu = smp_processor_id(); diff --git a/arch/arm64/kvm/fpsimd.c b/arch/arm64/kvm/fpsimd.c index e57db49ea468..143a7a0c2f9d 100644 --- a/arch/arm64/kvm/fpsimd.c +++ b/arch/arm64/kvm/fpsimd.c @@ -126,6 +126,7 @@ void kvm_arch_vcpu_ctxsync_fp(struct kvm_vcpu *vcpu) fp_state.sve_vl = vcpu->arch.sve_max_vl; fp_state.sme_state = NULL; fp_state.svcr = &vcpu->arch.svcr; + fp_state.fpmr = &vcpu->arch.fpmr; fp_state.fp_type = &vcpu->arch.fp_type; if (vcpu_has_sve(vcpu)) diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index 79bdb98e62bd..5de1f4836ba9 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -26,6 +26,7 @@ HAS_ECV HAS_ECV_CNTPOFF HAS_EPAN HAS_EVT +HAS_FPMR HAS_FGT HAS_FPSIMD HAS_GENERIC_AUTH From 061e877ec9c4623af0d173066a581fe49d246054 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Wed, 6 Mar 2024 23:14:49 +0000 Subject: [PATCH 16/85] arm64/signal: Add FPMR signal handling commit 8c46def44409fc914278630b7ba5ac142ab7c4f4 upstream. Expose FPMR in the signal context on systems where it is supported. The kernel validates the exact size of the FPSIMD registers so we can't readily add it to fpsimd_context without disruption. Signed-off-by: Mark Brown Link: https://lore.kernel.org/r/20240306-arm64-2023-dpisa-v5-4-c568edc8ed7f@kernel.org Signed-off-by: Catalin Marinas Signed-off-by: Bin Huang --- arch/arm64/include/uapi/asm/sigcontext.h | 8 ++++ arch/arm64/kernel/signal.c | 59 ++++++++++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/arch/arm64/include/uapi/asm/sigcontext.h b/arch/arm64/include/uapi/asm/sigcontext.h index 8f003db7a696..57f76d82077e 100644 --- a/arch/arm64/include/uapi/asm/sigcontext.h +++ b/arch/arm64/include/uapi/asm/sigcontext.h @@ -152,6 +152,14 @@ struct tpidr2_context { __u64 tpidr2; }; +/* FPMR context */ +#define FPMR_MAGIC 0x46504d52 + +struct fpmr_context { + struct _aarch64_ctx head; + __u64 fpmr; +}; + #define ZA_MAGIC 0x54366345 struct za_context { diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c index 3dec83dfe16c..7763134802c7 100644 --- a/arch/arm64/kernel/signal.c +++ b/arch/arm64/kernel/signal.c @@ -61,6 +61,7 @@ struct rt_sigframe_user_layout { unsigned long tpidr2_offset; unsigned long za_offset; unsigned long zt_offset; + unsigned long fpmr_offset; unsigned long extra_offset; unsigned long end_offset; }; @@ -183,6 +184,8 @@ struct user_ctxs { u32 za_size; struct zt_context __user *zt; u32 zt_size; + struct fpmr_context __user *fpmr; + u32 fpmr_size; }; static int preserve_fpsimd_context(struct fpsimd_context __user *ctx) @@ -238,6 +241,33 @@ static int restore_fpsimd_context(struct user_ctxs *user) return 0; } +static int preserve_fpmr_context(struct fpmr_context __user *ctx) +{ + int err = 0; + + current->thread.uw.fpmr = read_sysreg_s(SYS_FPMR); + + __put_user_error(FPMR_MAGIC, &ctx->head.magic, err); + __put_user_error(sizeof(*ctx), &ctx->head.size, err); + __put_user_error(current->thread.uw.fpmr, &ctx->fpmr, err); + + return err; +} + +static int restore_fpmr_context(struct user_ctxs *user) +{ + u64 fpmr; + int err = 0; + + if (user->fpmr_size != sizeof(*user->fpmr)) + return -EINVAL; + + __get_user_error(fpmr, &user->fpmr->fpmr, err); + if (!err) + write_sysreg_s(fpmr, SYS_FPMR); + + return err; +} #ifdef CONFIG_ARM64_SVE @@ -617,6 +647,7 @@ static int parse_user_sigframe(struct user_ctxs *user, user->tpidr2 = NULL; user->za = NULL; user->zt = NULL; + user->fpmr = NULL; if (!IS_ALIGNED((unsigned long)base, 16)) goto invalid; @@ -711,6 +742,17 @@ static int parse_user_sigframe(struct user_ctxs *user, user->zt_size = size; break; + case FPMR_MAGIC: + if (!system_supports_fpmr()) + goto invalid; + + if (user->fpmr) + goto invalid; + + user->fpmr = (struct fpmr_context __user *)head; + user->fpmr_size = size; + break; + case EXTRA_MAGIC: if (have_extra_context) goto invalid; @@ -833,6 +875,9 @@ static int restore_sigframe(struct pt_regs *regs, if (err == 0 && system_supports_tpidr2() && user.tpidr2) err = restore_tpidr2_context(&user); + if (err == 0 && system_supports_fpmr() && user.fpmr) + err = restore_fpmr_context(&user); + if (err == 0 && system_supports_sme() && user.za) err = restore_za_context(&user); @@ -955,6 +1000,13 @@ static int setup_sigframe_layout(struct rt_sigframe_user_layout *user, } } + if (system_supports_fpmr()) { + err = sigframe_alloc(user, &user->fpmr_offset, + sizeof(struct fpmr_context)); + if (err) + return err; + } + return sigframe_alloc_end(user); } @@ -1010,6 +1062,13 @@ static int setup_sigframe(struct rt_sigframe_user_layout *user, err |= preserve_tpidr2_context(tpidr2_ctx); } + /* FPMR if supported */ + if (system_supports_fpmr() && err == 0) { + struct fpmr_context __user *fpmr_ctx = + apply_user_offset(user, user->fpmr_offset); + err |= preserve_fpmr_context(fpmr_ctx); + } + /* ZA state if present */ if (system_supports_sme() && err == 0 && user->za_offset) { struct za_context __user *za_ctx = From 4170e5f1cb0ef9018067790a5f7e687c1415b863 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Wed, 6 Mar 2024 23:14:50 +0000 Subject: [PATCH 17/85] arm64/ptrace: Expose FPMR via ptrace commit 4035c22ef7d43a6c00d6a6584c60e902b95b46af upstream. Add a new regset to expose FPMR via ptrace. It is not added to the FPSIMD registers since that structure is exposed elsewhere without any allowance for extension we don't add there. Signed-off-by: Mark Brown Link: https://lore.kernel.org/r/20240306-arm64-2023-dpisa-v5-5-c568edc8ed7f@kernel.org Signed-off-by: Catalin Marinas Signed-off-by: Bin Huang --- arch/arm64/kernel/ptrace.c | 42 ++++++++++++++++++++++++++++++++++++++ include/uapi/linux/elf.h | 1 + 2 files changed, 43 insertions(+) diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c index c30f5a70bd18..0ed89720d847 100644 --- a/arch/arm64/kernel/ptrace.c +++ b/arch/arm64/kernel/ptrace.c @@ -697,6 +697,39 @@ static int tls_set(struct task_struct *target, const struct user_regset *regset, return ret; } +static int fpmr_get(struct task_struct *target, const struct user_regset *regset, + struct membuf to) +{ + if (!system_supports_fpmr()) + return -EINVAL; + + if (target == current) + fpsimd_preserve_current_state(); + + return membuf_store(&to, target->thread.uw.fpmr); +} + +static int fpmr_set(struct task_struct *target, const struct user_regset *regset, + unsigned int pos, unsigned int count, + const void *kbuf, const void __user *ubuf) +{ + int ret; + unsigned long fpmr; + + if (!system_supports_fpmr()) + return -EINVAL; + + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &fpmr, 0, count); + if (ret) + return ret; + + target->thread.uw.fpmr = fpmr; + + fpsimd_flush_task_state(target); + + return 0; +} + static int system_call_get(struct task_struct *target, const struct user_regset *regset, struct membuf to) @@ -1419,6 +1452,7 @@ enum aarch64_regset { REGSET_HW_BREAK, REGSET_HW_WATCH, #endif + REGSET_FPMR, REGSET_SYSTEM_CALL, #ifdef CONFIG_ARM64_SVE REGSET_SVE, @@ -1497,6 +1531,14 @@ static const struct user_regset aarch64_regsets[] = { .regset_get = system_call_get, .set = system_call_set, }, + [REGSET_FPMR] = { + .core_note_type = NT_ARM_FPMR, + .n = 1, + .size = sizeof(u64), + .align = sizeof(u64), + .regset_get = fpmr_get, + .set = fpmr_set, + }, #ifdef CONFIG_ARM64_SVE [REGSET_SVE] = { /* Scalable Vector Extension */ .core_note_type = NT_ARM_SVE, diff --git a/include/uapi/linux/elf.h b/include/uapi/linux/elf.h index 9b731976ce2f..4c8b4641a306 100644 --- a/include/uapi/linux/elf.h +++ b/include/uapi/linux/elf.h @@ -440,6 +440,7 @@ typedef struct elf64_shdr { #define NT_ARM_SSVE 0x40b /* ARM Streaming SVE registers */ #define NT_ARM_ZA 0x40c /* ARM SME ZA registers */ #define NT_ARM_ZT 0x40d /* ARM SME ZT registers */ +#define NT_ARM_FPMR 0x40e /* ARM floating point mode register */ #define NT_ARC_V2 0x600 /* ARCv2 accumulator/extra registers */ #define NT_VMCOREDD 0x700 /* Vmcore Device Dump Note */ #define NT_MIPS_DSP 0x800 /* MIPS DSP ASE registers */ From 3781bafc2ffb3d7973883eb805291e57f0fc41a4 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Wed, 6 Mar 2024 23:14:51 +0000 Subject: [PATCH 18/85] arm64/hwcap: Define hwcaps for 2023 DPISA features commit c1932cac7902a8b0f7355515917dedc5412eb15d upstream. The 2023 architecture extensions include a large number of floating point features, most of which simply add new instructions. Add hwcaps so that userspace can enumerate these features. Signed-off-by: Mark Brown Link: https://lore.kernel.org/r/20240306-arm64-2023-dpisa-v5-6-c568edc8ed7f@kernel.org Signed-off-by: Catalin Marinas [ bh: veLinux 6.6.151 uses HWCAP_CAP_MATCH_ID() for SME capability checks and retains upstream matchers elsewhere. ] Signed-off-by: Bin Huang --- Documentation/arch/arm64/elf_hwcaps.rst | 49 +++++++++++++++++++++++++ arch/arm64/include/asm/hwcap.h | 15 ++++++++ arch/arm64/include/uapi/asm/hwcap.h | 15 ++++++++ arch/arm64/kernel/cpufeature.c | 35 ++++++++++++++++++ arch/arm64/kernel/cpuinfo.c | 15 ++++++++ 5 files changed, 129 insertions(+) diff --git a/Documentation/arch/arm64/elf_hwcaps.rst b/Documentation/arch/arm64/elf_hwcaps.rst index 9a256701dc62..79802e7707b2 100644 --- a/Documentation/arch/arm64/elf_hwcaps.rst +++ b/Documentation/arch/arm64/elf_hwcaps.rst @@ -329,6 +329,55 @@ HWCAP3_LS64 HWCAP3_LS64_V Functionality implied by ID_AA64ISAR1_EL1.LS64 == 0b0010. +HWCAP2_FPMR + Functionality implied by ID_AA64PFR2_EL1.FMR == 0b0001. + +HWCAP2_LUT + Functionality implied by ID_AA64ISAR2_EL1.LUT == 0b0001. + +HWCAP2_FAMINMAX + Functionality implied by ID_AA64ISAR3_EL1.FAMINMAX == 0b0001. + +HWCAP2_F8CVT + Functionality implied by ID_AA64FPFR0_EL1.F8CVT == 0b1. + +HWCAP2_F8FMA + Functionality implied by ID_AA64FPFR0_EL1.F8FMA == 0b1. + +HWCAP2_F8DP4 + Functionality implied by ID_AA64FPFR0_EL1.F8DP4 == 0b1. + +HWCAP2_F8DP2 + Functionality implied by ID_AA64FPFR0_EL1.F8DP2 == 0b1. + +HWCAP2_F8E4M3 + Functionality implied by ID_AA64FPFR0_EL1.F8E4M3 == 0b1. + +HWCAP2_F8E5M2 + Functionality implied by ID_AA64FPFR0_EL1.F8E5M2 == 0b1. + +HWCAP2_SME_LUTV2 + Functionality implied by ID_AA64SMFR0_EL1.LUTv2 == 0b1. + +HWCAP2_SME_F8F16 + Functionality implied by ID_AA64SMFR0_EL1.F8F16 == 0b1. + +HWCAP2_SME_F8F32 + Functionality implied by ID_AA64SMFR0_EL1.F8F32 == 0b1. + +HWCAP2_SME_SF8FMA + Functionality implied by ID_AA64SMFR0_EL1.SF8FMA == 0b1. + +HWCAP2_SME_SF8DP4 + Functionality implied by ID_AA64SMFR0_EL1.SF8DP4 == 0b1. + +HWCAP2_SME_SF8DP2 + Functionality implied by ID_AA64SMFR0_EL1.SF8DP2 == 0b1. + +HWCAP2_SME_SF8DP4 + Functionality implied by ID_AA64SMFR0_EL1.SF8DP4 == 0b1. + + 4. Unused AT_HWCAP bits ----------------------- diff --git a/arch/arm64/include/asm/hwcap.h b/arch/arm64/include/asm/hwcap.h index 064c86b0dc99..92e6cd25ccfb 100644 --- a/arch/arm64/include/asm/hwcap.h +++ b/arch/arm64/include/asm/hwcap.h @@ -140,6 +140,21 @@ #define KERNEL_HWCAP_MOPS __khwcap2_feature(MOPS) #define KERNEL_HWCAP_HBC __khwcap2_feature(HBC) #define KERNEL_HWCAP_SVE_B16B16 __khwcap2_feature(SVE_B16B16) +#define KERNEL_HWCAP_FPMR __khwcap2_feature(FPMR) +#define KERNEL_HWCAP_LUT __khwcap2_feature(LUT) +#define KERNEL_HWCAP_FAMINMAX __khwcap2_feature(FAMINMAX) +#define KERNEL_HWCAP_F8CVT __khwcap2_feature(F8CVT) +#define KERNEL_HWCAP_F8FMA __khwcap2_feature(F8FMA) +#define KERNEL_HWCAP_F8DP4 __khwcap2_feature(F8DP4) +#define KERNEL_HWCAP_F8DP2 __khwcap2_feature(F8DP2) +#define KERNEL_HWCAP_F8E4M3 __khwcap2_feature(F8E4M3) +#define KERNEL_HWCAP_F8E5M2 __khwcap2_feature(F8E5M2) +#define KERNEL_HWCAP_SME_LUTV2 __khwcap2_feature(SME_LUTV2) +#define KERNEL_HWCAP_SME_F8F16 __khwcap2_feature(SME_F8F16) +#define KERNEL_HWCAP_SME_F8F32 __khwcap2_feature(SME_F8F32) +#define KERNEL_HWCAP_SME_SF8FMA __khwcap2_feature(SME_SF8FMA) +#define KERNEL_HWCAP_SME_SF8DP4 __khwcap2_feature(SME_SF8DP4) +#define KERNEL_HWCAP_SME_SF8DP2 __khwcap2_feature(SME_SF8DP2) #define __khwcap3_feature(x) (const_ilog2(HWCAP3_ ## x) + 128) #define KERNEL_HWCAP_LS64 __khwcap3_feature(LS64) diff --git a/arch/arm64/include/uapi/asm/hwcap.h b/arch/arm64/include/uapi/asm/hwcap.h index bd613c7df482..49fb2d8b5375 100644 --- a/arch/arm64/include/uapi/asm/hwcap.h +++ b/arch/arm64/include/uapi/asm/hwcap.h @@ -105,6 +105,21 @@ #define HWCAP2_MOPS (1UL << 43) #define HWCAP2_HBC (1UL << 44) #define HWCAP2_SVE_B16B16 (1UL << 45) +#define HWCAP2_FPMR (1UL << 48) +#define HWCAP2_LUT (1UL << 49) +#define HWCAP2_FAMINMAX (1UL << 50) +#define HWCAP2_F8CVT (1UL << 51) +#define HWCAP2_F8FMA (1UL << 52) +#define HWCAP2_F8DP4 (1UL << 53) +#define HWCAP2_F8DP2 (1UL << 54) +#define HWCAP2_F8E4M3 (1UL << 55) +#define HWCAP2_F8E5M2 (1UL << 56) +#define HWCAP2_SME_LUTV2 (1UL << 57) +#define HWCAP2_SME_F8F16 (1UL << 58) +#define HWCAP2_SME_F8F32 (1UL << 59) +#define HWCAP2_SME_SF8FMA (1UL << 60) +#define HWCAP2_SME_SF8DP4 (1UL << 61) +#define HWCAP2_SME_SF8DP2 (1UL << 62) /* * HWCAP3 flags - for AT_HWCAP3 diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index de4f95e22c76..fa32aebdd45e 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -222,6 +222,7 @@ static const struct arm64_ftr_bits ftr_id_aa64isar1[] = { }; static const struct arm64_ftr_bits ftr_id_aa64isar2[] = { + ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_EL1_LUT_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_EL1_CSSC_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_EL1_RPRFM_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_EL1_CLRBHB_SHIFT, 4, 0), @@ -237,6 +238,7 @@ static const struct arm64_ftr_bits ftr_id_aa64isar2[] = { }; static const struct arm64_ftr_bits ftr_id_aa64isar3[] = { + ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR3_EL1_FAMINMAX_SHIFT, 4, 0), ARM64_FTR_END, }; @@ -305,6 +307,8 @@ static const struct arm64_ftr_bits ftr_id_aa64zfr0[] = { static const struct arm64_ftr_bits ftr_id_aa64smfr0[] = { ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_FA64_SHIFT, 1, 0), + ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), + FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_LUTv2_SHIFT, 1, 0), ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_SMEver_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), @@ -317,6 +321,10 @@ static const struct arm64_ftr_bits ftr_id_aa64smfr0[] = { FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_B16B16_SHIFT, 1, 0), ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_F16F16_SHIFT, 1, 0), + ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), + FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_F8F16_SHIFT, 1, 0), + ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), + FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_F8F32_SHIFT, 1, 0), ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_I8I32_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), @@ -327,10 +335,22 @@ static const struct arm64_ftr_bits ftr_id_aa64smfr0[] = { FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_BI32I32_SHIFT, 1, 0), ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_F32F32_SHIFT, 1, 0), + ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), + FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_SF8FMA_SHIFT, 1, 0), + ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), + FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_SF8DP4_SHIFT, 1, 0), + ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), + FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_SF8DP2_SHIFT, 1, 0), ARM64_FTR_END, }; static const struct arm64_ftr_bits ftr_id_aa64fpfr0[] = { + ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8CVT_SHIFT, 1, 0), + ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8FMA_SHIFT, 1, 0), + ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8DP4_SHIFT, 1, 0), + ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8DP2_SHIFT, 1, 0), + ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8E4M3_SHIFT, 1, 0), + ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8E5M2_SHIFT, 1, 0), ARM64_FTR_END, }; @@ -3011,6 +3031,7 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { HWCAP_CAP(ID_AA64PFR0_EL1, AdvSIMD, IMP, CAP_HWCAP, KERNEL_HWCAP_ASIMD), HWCAP_CAP(ID_AA64PFR0_EL1, AdvSIMD, FP16, CAP_HWCAP, KERNEL_HWCAP_ASIMDHP), HWCAP_CAP(ID_AA64PFR0_EL1, DIT, IMP, CAP_HWCAP, KERNEL_HWCAP_DIT), + HWCAP_CAP(ID_AA64PFR2_EL1, FPMR, IMP, CAP_HWCAP, KERNEL_HWCAP_FPMR), HWCAP_CAP(ID_AA64ISAR1_EL1, DPB, IMP, CAP_HWCAP, KERNEL_HWCAP_DCPOP), HWCAP_CAP(ID_AA64ISAR1_EL1, DPB, DPB2, CAP_HWCAP, KERNEL_HWCAP_DCPODP), HWCAP_CAP(ID_AA64ISAR1_EL1, JSCVT, IMP, CAP_HWCAP, KERNEL_HWCAP_JSCVT), @@ -3025,6 +3046,8 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { HWCAP_CAP(ID_AA64ISAR1_EL1, I8MM, IMP, CAP_HWCAP, KERNEL_HWCAP_I8MM), HWCAP_CAP(ID_AA64ISAR1_EL1, LS64, LS64, CAP_HWCAP, KERNEL_HWCAP_LS64), HWCAP_CAP(ID_AA64ISAR1_EL1, LS64, LS64_V, CAP_HWCAP, KERNEL_HWCAP_LS64_V), + HWCAP_CAP(ID_AA64ISAR2_EL1, LUT, IMP, CAP_HWCAP, KERNEL_HWCAP_LUT), + HWCAP_CAP(ID_AA64ISAR3_EL1, FAMINMAX, IMP, CAP_HWCAP, KERNEL_HWCAP_FAMINMAX), HWCAP_CAP(ID_AA64MMFR2_EL1, AT, IMP, CAP_HWCAP, KERNEL_HWCAP_USCAT), #ifdef CONFIG_ARM64_SVE HWCAP_CAP(ID_AA64PFR0_EL1, SVE, IMP, CAP_HWCAP, KERNEL_HWCAP_SVE), @@ -3065,6 +3088,7 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { #ifdef CONFIG_ARM64_SME HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64PFR1_EL1, SME, IMP, CAP_HWCAP, KERNEL_HWCAP_SME), HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, FA64, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_FA64), + HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, LUTv2, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_LUTV2), HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SMEver, SME2p1, CAP_HWCAP, KERNEL_HWCAP_SME2P1), HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SMEver, SME2, CAP_HWCAP, KERNEL_HWCAP_SME2), HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, I16I64, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_I16I64), @@ -3072,12 +3096,23 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, I16I32, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_I16I32), HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, B16B16, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_B16B16), HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, F16F16, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_F16F16), + HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, F8F16, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_F8F16), + HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, F8F32, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_F8F32), HWCAP_CAP(ID_MATCH_ID(has_sme_feature, AA64SMFR0_EL1, I8I32, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_I8I32), HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, F16F32, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_F16F32), HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, B16F32, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_B16F32), HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, BI32I32, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_BI32I32), HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, F32F32, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_F32F32), + HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SF8FMA, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_SF8FMA), + HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SF8DP4, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_SF8DP4), + HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SF8DP2, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_SF8DP2), #endif /* CONFIG_ARM64_SME */ + HWCAP_CAP(ID_AA64FPFR0_EL1, F8CVT, IMP, CAP_HWCAP, KERNEL_HWCAP_F8CVT), + HWCAP_CAP(ID_AA64FPFR0_EL1, F8FMA, IMP, CAP_HWCAP, KERNEL_HWCAP_F8FMA), + HWCAP_CAP(ID_AA64FPFR0_EL1, F8DP4, IMP, CAP_HWCAP, KERNEL_HWCAP_F8DP4), + HWCAP_CAP(ID_AA64FPFR0_EL1, F8DP2, IMP, CAP_HWCAP, KERNEL_HWCAP_F8DP2), + HWCAP_CAP(ID_AA64FPFR0_EL1, F8E4M3, IMP, CAP_HWCAP, KERNEL_HWCAP_F8E4M3), + HWCAP_CAP(ID_AA64FPFR0_EL1, F8E5M2, IMP, CAP_HWCAP, KERNEL_HWCAP_F8E5M2), {}, }; diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c index 56cc3ba8545e..e7910b3e7fb4 100644 --- a/arch/arm64/kernel/cpuinfo.c +++ b/arch/arm64/kernel/cpuinfo.c @@ -130,6 +130,21 @@ static const char *const hwcap_str[] = { [KERNEL_HWCAP_MOPS] = "mops", [KERNEL_HWCAP_HBC] = "hbc", [KERNEL_HWCAP_SVE_B16B16] = "sveb16b16", + [KERNEL_HWCAP_FPMR] = "fpmr", + [KERNEL_HWCAP_LUT] = "lut", + [KERNEL_HWCAP_FAMINMAX] = "faminmax", + [KERNEL_HWCAP_F8CVT] = "f8cvt", + [KERNEL_HWCAP_F8FMA] = "f8fma", + [KERNEL_HWCAP_F8DP4] = "f8dp4", + [KERNEL_HWCAP_F8DP2] = "f8dp2", + [KERNEL_HWCAP_F8E4M3] = "f8e4m3", + [KERNEL_HWCAP_F8E5M2] = "f8e5m2", + [KERNEL_HWCAP_SME_LUTV2] = "smelutv2", + [KERNEL_HWCAP_SME_F8F16] = "smef8f16", + [KERNEL_HWCAP_SME_F8F32] = "smef8f32", + [KERNEL_HWCAP_SME_SF8FMA] = "smesf8fma", + [KERNEL_HWCAP_SME_SF8DP4] = "smesf8dp4", + [KERNEL_HWCAP_SME_SF8DP2] = "smesf8dp2", }; #ifdef CONFIG_COMPAT From 71eef2b9c297eff819c81ccda45c3abf8390b03b Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Wed, 6 Mar 2024 23:14:52 +0000 Subject: [PATCH 19/85] kselftest/arm64: Handle FPMR context in generic signal frame parser commit f4dcccdda5867bb68f48046e86e5a7ccaae10d27 upstream. Teach the generic signal frame parsing code about the newly added FPMR frame, avoiding warnings every time one is generated. Signed-off-by: Mark Brown Link: https://lore.kernel.org/r/20240306-arm64-2023-dpisa-v5-7-c568edc8ed7f@kernel.org Signed-off-by: Catalin Marinas Signed-off-by: Bin Huang --- .../testing/selftests/arm64/signal/testcases/testcases.c | 8 ++++++++ .../testing/selftests/arm64/signal/testcases/testcases.h | 1 + 2 files changed, 9 insertions(+) diff --git a/tools/testing/selftests/arm64/signal/testcases/testcases.c b/tools/testing/selftests/arm64/signal/testcases/testcases.c index 9f580b55b388..674b88cc8c39 100644 --- a/tools/testing/selftests/arm64/signal/testcases/testcases.c +++ b/tools/testing/selftests/arm64/signal/testcases/testcases.c @@ -209,6 +209,14 @@ bool validate_reserved(ucontext_t *uc, size_t resv_sz, char **err) zt = (struct zt_context *)head; new_flags |= ZT_CTX; break; + case FPMR_MAGIC: + if (flags & FPMR_CTX) + *err = "Multiple FPMR_MAGIC"; + else if (head->size != + sizeof(struct fpmr_context)) + *err = "Bad size for fpmr_context"; + new_flags |= FPMR_CTX; + break; case EXTRA_MAGIC: if (flags & EXTRA_CTX) *err = "Multiple EXTRA_MAGIC"; diff --git a/tools/testing/selftests/arm64/signal/testcases/testcases.h b/tools/testing/selftests/arm64/signal/testcases/testcases.h index a08ab0d6207a..7727126347e0 100644 --- a/tools/testing/selftests/arm64/signal/testcases/testcases.h +++ b/tools/testing/selftests/arm64/signal/testcases/testcases.h @@ -19,6 +19,7 @@ #define ZA_CTX (1 << 2) #define EXTRA_CTX (1 << 3) #define ZT_CTX (1 << 4) +#define FPMR_CTX (1 << 5) #define KSFT_BAD_MAGIC 0xdeadbeef From 484c0f7105effe6c1a4ae88fc5f4274103bdc484 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Wed, 6 Mar 2024 23:14:53 +0000 Subject: [PATCH 20/85] kselftest/arm64: Add basic FPMR test commit 7bcebadda045bead18f6f7050af59af388c4507c upstream. Verify that a FPMR frame is generated on systems that support FPMR and not generated otherwise. Signed-off-by: Mark Brown Link: https://lore.kernel.org/r/20240306-arm64-2023-dpisa-v5-8-c568edc8ed7f@kernel.org Signed-off-by: Catalin Marinas Signed-off-by: Bin Huang --- .../testing/selftests/arm64/signal/.gitignore | 1 + .../arm64/signal/testcases/fpmr_siginfo.c | 82 +++++++++++++++++++ 2 files changed, 83 insertions(+) create mode 100644 tools/testing/selftests/arm64/signal/testcases/fpmr_siginfo.c diff --git a/tools/testing/selftests/arm64/signal/.gitignore b/tools/testing/selftests/arm64/signal/.gitignore index 839e3a252629..1ce5b5eac386 100644 --- a/tools/testing/selftests/arm64/signal/.gitignore +++ b/tools/testing/selftests/arm64/signal/.gitignore @@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only mangle_* fake_sigreturn_* +fpmr_* sme_* ssve_* sve_* diff --git a/tools/testing/selftests/arm64/signal/testcases/fpmr_siginfo.c b/tools/testing/selftests/arm64/signal/testcases/fpmr_siginfo.c new file mode 100644 index 000000000000..e9d24685e741 --- /dev/null +++ b/tools/testing/selftests/arm64/signal/testcases/fpmr_siginfo.c @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2023 ARM Limited + * + * Verify that the FPMR register context in signal frames is set up as + * expected. + */ + +#include +#include +#include +#include +#include +#include + +#include "test_signals_utils.h" +#include "testcases.h" + +static union { + ucontext_t uc; + char buf[1024 * 128]; +} context; + +#define SYS_FPMR "S3_3_C4_C4_2" + +static uint64_t get_fpmr(void) +{ + uint64_t val; + + asm volatile ( + "mrs %0, " SYS_FPMR "\n" + : "=r"(val) + : + : "cc"); + + return val; +} + +int fpmr_present(struct tdescr *td, siginfo_t *si, ucontext_t *uc) +{ + struct _aarch64_ctx *head = GET_BUF_RESV_HEAD(context); + struct fpmr_context *fpmr_ctx; + size_t offset; + bool in_sigframe; + bool have_fpmr; + __u64 orig_fpmr; + + have_fpmr = getauxval(AT_HWCAP2) & HWCAP2_FPMR; + if (have_fpmr) + orig_fpmr = get_fpmr(); + + if (!get_current_context(td, &context.uc, sizeof(context))) + return 1; + + fpmr_ctx = (struct fpmr_context *) + get_header(head, FPMR_MAGIC, td->live_sz, &offset); + + in_sigframe = fpmr_ctx != NULL; + + fprintf(stderr, "FPMR sigframe %s on system %s FPMR\n", + in_sigframe ? "present" : "absent", + have_fpmr ? "with" : "without"); + + td->pass = (in_sigframe == have_fpmr); + + if (have_fpmr && fpmr_ctx) { + if (fpmr_ctx->fpmr != orig_fpmr) { + fprintf(stderr, "FPMR in frame is %llx, was %llx\n", + fpmr_ctx->fpmr, orig_fpmr); + td->pass = false; + } + } + + return 0; +} + +struct tdescr tde = { + .name = "FPMR", + .descr = "Validate that FPMR is present as expected", + .timeout = 3, + .run = fpmr_present, +}; From ba736dbcc1fb7be6a34880f3afdce9bb7f4220c9 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Wed, 6 Mar 2024 23:14:54 +0000 Subject: [PATCH 21/85] kselftest/arm64: Add 2023 DPISA hwcap test coverage commit 44d10c27bd75b88b50d0c5cf2c3fe92933c39f1e upstream. Add the hwcaps added for the 2023 DPISA extensions to the hwcaps test program. Signed-off-by: Mark Brown Link: https://lore.kernel.org/r/20240306-arm64-2023-dpisa-v5-9-c568edc8ed7f@kernel.org Signed-off-by: Catalin Marinas Signed-off-by: Bin Huang --- tools/testing/selftests/arm64/abi/hwcap.c | 217 ++++++++++++++++++++++ 1 file changed, 217 insertions(+) diff --git a/tools/testing/selftests/arm64/abi/hwcap.c b/tools/testing/selftests/arm64/abi/hwcap.c index 55b9053afaae..ed66253e8db5 100644 --- a/tools/testing/selftests/arm64/abi/hwcap.c +++ b/tools/testing/selftests/arm64/abi/hwcap.c @@ -58,11 +58,46 @@ static void cssc_sigill(void) asm volatile(".inst 0xdac01c00" : : : "x0"); } +static void f8cvt_sigill(void) +{ + /* FSCALE V0.4H, V0.4H, V0.4H */ + asm volatile(".inst 0x2ec03c00"); +} + +static void f8dp2_sigill(void) +{ + /* FDOT V0.4H, V0.4H, V0.5H */ + asm volatile(".inst 0xe40fc00"); +} + +static void f8dp4_sigill(void) +{ + /* FDOT V0.2S, V0.2S, V0.2S */ + asm volatile(".inst 0xe00fc00"); +} + +static void f8fma_sigill(void) +{ + /* FMLALB V0.8H, V0.16B, V0.16B */ + asm volatile(".inst 0xec0fc00"); +} + +static void faminmax_sigill(void) +{ + /* FAMIN V0.4H, V0.4H, V0.4H */ + asm volatile(".inst 0x2ec01c00"); +} + static void fp_sigill(void) { asm volatile("fmov s0, #1"); } +static void fpmr_sigill(void) +{ + asm volatile("mrs x0, S3_3_C4_C4_2" : : : "x0"); +} + static void ilrcpc_sigill(void) { /* LDAPUR W0, [SP, #8] */ @@ -81,6 +116,12 @@ static void lrcpc_sigill(void) asm volatile(".inst 0xb8bfc3e0" : : : ); } +static void lut_sigill(void) +{ + /* LUTI2 V0.16B, { V0.16B }, V[0] */ + asm volatile(".inst 0x4e801000"); +} + static void mops_sigill(void) { char dst[1], src[1]; @@ -202,6 +243,78 @@ static void smef16f16_sigill(void) asm volatile("msr S0_3_C4_C6_3, xzr" : : : ); } +static void smef8f16_sigill(void) +{ + /* SMSTART */ + asm volatile("msr S0_3_C4_C7_3, xzr" : : : ); + + /* FDOT ZA.H[W0, 0], Z0.B-Z1.B, Z0.B-Z1.B */ + asm volatile(".inst 0xc1a01020" : : : ); + + /* SMSTOP */ + asm volatile("msr S0_3_C4_C6_3, xzr" : : : ); +} + +static void smef8f32_sigill(void) +{ + /* SMSTART */ + asm volatile("msr S0_3_C4_C7_3, xzr" : : : ); + + /* FDOT ZA.S[W0, 0], { Z0.B-Z1.B }, Z0.B[0] */ + asm volatile(".inst 0xc1500038" : : : ); + + /* SMSTOP */ + asm volatile("msr S0_3_C4_C6_3, xzr" : : : ); +} + +static void smelutv2_sigill(void) +{ + /* SMSTART */ + asm volatile("msr S0_3_C4_C7_3, xzr" : : : ); + + /* LUTI4 { Z0.B-Z3.B }, ZT0, { Z0-Z1 } */ + asm volatile(".inst 0xc08b0000" : : : ); + + /* SMSTOP */ + asm volatile("msr S0_3_C4_C6_3, xzr" : : : ); +} + +static void smesf8dp2_sigill(void) +{ + /* SMSTART */ + asm volatile("msr S0_3_C4_C7_3, xzr" : : : ); + + /* FDOT Z0.H, Z0.B, Z0.B[0] */ + asm volatile(".inst 0x64204400" : : : ); + + /* SMSTOP */ + asm volatile("msr S0_3_C4_C6_3, xzr" : : : ); +} + +static void smesf8dp4_sigill(void) +{ + /* SMSTART */ + asm volatile("msr S0_3_C4_C7_3, xzr" : : : ); + + /* FDOT Z0.S, Z0.B, Z0.B[0] */ + asm volatile(".inst 0xc1a41C00" : : : ); + + /* SMSTOP */ + asm volatile("msr S0_3_C4_C6_3, xzr" : : : ); +} + +static void smesf8fma_sigill(void) +{ + /* SMSTART */ + asm volatile("msr S0_3_C4_C7_3, xzr" : : : ); + + /* FMLALB V0.8H, V0.16B, V0.16B */ + asm volatile(".inst 0xec0fc00"); + + /* SMSTOP */ + asm volatile("msr S0_3_C4_C6_3, xzr" : : : ); +} + static void sve_sigill(void) { /* RDVL x0, #0 */ @@ -326,6 +439,53 @@ static const struct hwcap_data { .cpuinfo = "cssc", .sigill_fn = cssc_sigill, }, + { + .name = "F8CVT", + .at_hwcap = AT_HWCAP2, + .hwcap_bit = HWCAP2_F8CVT, + .cpuinfo = "f8cvt", + .sigill_fn = f8cvt_sigill, + }, + { + .name = "F8DP4", + .at_hwcap = AT_HWCAP2, + .hwcap_bit = HWCAP2_F8DP4, + .cpuinfo = "f8dp4", + .sigill_fn = f8dp4_sigill, + }, + { + .name = "F8DP2", + .at_hwcap = AT_HWCAP2, + .hwcap_bit = HWCAP2_F8DP2, + .cpuinfo = "f8dp4", + .sigill_fn = f8dp2_sigill, + }, + { + .name = "F8E5M2", + .at_hwcap = AT_HWCAP2, + .hwcap_bit = HWCAP2_F8E5M2, + .cpuinfo = "f8e5m2", + }, + { + .name = "F8E4M3", + .at_hwcap = AT_HWCAP2, + .hwcap_bit = HWCAP2_F8E4M3, + .cpuinfo = "f8e4m3", + }, + { + .name = "F8FMA", + .at_hwcap = AT_HWCAP2, + .hwcap_bit = HWCAP2_F8FMA, + .cpuinfo = "f8fma", + .sigill_fn = f8fma_sigill, + }, + { + .name = "FAMINMAX", + .at_hwcap = AT_HWCAP2, + .hwcap_bit = HWCAP2_FAMINMAX, + .cpuinfo = "faminmax", + .sigill_fn = faminmax_sigill, + }, { .name = "FP", .at_hwcap = AT_HWCAP, @@ -333,6 +493,14 @@ static const struct hwcap_data { .cpuinfo = "fp", .sigill_fn = fp_sigill, }, + { + .name = "FPMR", + .at_hwcap = AT_HWCAP2, + .hwcap_bit = HWCAP2_FPMR, + .cpuinfo = "fpmr", + .sigill_fn = fpmr_sigill, + .sigill_reliable = true, + }, { .name = "JSCVT", .at_hwcap = AT_HWCAP, @@ -370,6 +538,13 @@ static const struct hwcap_data { .sigbus_fn = uscat_sigbus, .sigbus_reliable = true, }, + { + .name = "LUT", + .at_hwcap = AT_HWCAP2, + .hwcap_bit = HWCAP2_LUT, + .cpuinfo = "lut", + .sigill_fn = lut_sigill, + }, { .name = "MOPS", .at_hwcap = AT_HWCAP2, @@ -470,6 +645,48 @@ static const struct hwcap_data { .cpuinfo = "smef16f16", .sigill_fn = smef16f16_sigill, }, + { + .name = "SME F8F16", + .at_hwcap = AT_HWCAP2, + .hwcap_bit = HWCAP2_SME_F8F16, + .cpuinfo = "smef8f16", + .sigill_fn = smef8f16_sigill, + }, + { + .name = "SME F8F32", + .at_hwcap = AT_HWCAP2, + .hwcap_bit = HWCAP2_SME_F8F32, + .cpuinfo = "smef8f32", + .sigill_fn = smef8f32_sigill, + }, + { + .name = "SME LUTV2", + .at_hwcap = AT_HWCAP2, + .hwcap_bit = HWCAP2_SME_LUTV2, + .cpuinfo = "smelutv2", + .sigill_fn = smelutv2_sigill, + }, + { + .name = "SME SF8FMA", + .at_hwcap = AT_HWCAP2, + .hwcap_bit = HWCAP2_SME_SF8FMA, + .cpuinfo = "smesf8fma", + .sigill_fn = smesf8fma_sigill, + }, + { + .name = "SME SF8DP2", + .at_hwcap = AT_HWCAP2, + .hwcap_bit = HWCAP2_SME_SF8DP2, + .cpuinfo = "smesf8dp2", + .sigill_fn = smesf8dp2_sigill, + }, + { + .name = "SME SF8DP4", + .at_hwcap = AT_HWCAP2, + .hwcap_bit = HWCAP2_SME_SF8DP4, + .cpuinfo = "smesf8dp4", + .sigill_fn = smesf8dp4_sigill, + }, { .name = "SVE", .at_hwcap = AT_HWCAP, From 83c22791f6461a76b5049ddf8d0e687d5323563b Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Thu, 5 Dec 2024 12:16:53 +0000 Subject: [PATCH 22/85] arm64: ptrace: fix partial SETREGSET for NT_ARM_FPMR commit f5d71291841aecfe5d8435da2dfa7f58ccd18bc8 upstream. Currently fpmr_set() doesn't initialize the temporary 'fpmr' variable, and a SETREGSET call with a length of zero will leave this uninitialized. Consequently an arbitrary value will be written back to target->thread.uw.fpmr, potentially leaking up to 64 bits of memory from the kernel stack. The read is limited to a specific slot on the stack, and the issue does not provide a write mechanism. Fix this by initializing the temporary value before copying the regset from userspace, as for other regsets (e.g. NT_PRSTATUS, NT_PRFPREG, NT_ARM_SYSTEM_CALL). In the case of a zero-length write, the existing contents of FPMR will be retained. Before this patch: | # ./fpmr-test | Attempting to write NT_ARM_FPMR::fpmr = 0x900d900d900d900d | SETREGSET(nt=0x40e, len=8) wrote 8 bytes | | Attempting to read NT_ARM_FPMR::fpmr | GETREGSET(nt=0x40e, len=8) read 8 bytes | Read NT_ARM_FPMR::fpmr = 0x900d900d900d900d | | Attempting to write NT_ARM_FPMR (zero length) | SETREGSET(nt=0x40e, len=0) wrote 0 bytes | | Attempting to read NT_ARM_FPMR::fpmr | GETREGSET(nt=0x40e, len=8) read 8 bytes | Read NT_ARM_FPMR::fpmr = 0xffff800083963d50 After this patch: | # ./fpmr-test | Attempting to write NT_ARM_FPMR::fpmr = 0x900d900d900d900d | SETREGSET(nt=0x40e, len=8) wrote 8 bytes | | Attempting to read NT_ARM_FPMR::fpmr | GETREGSET(nt=0x40e, len=8) read 8 bytes | Read NT_ARM_FPMR::fpmr = 0x900d900d900d900d | | Attempting to write NT_ARM_FPMR (zero length) | SETREGSET(nt=0x40e, len=0) wrote 0 bytes | | Attempting to read NT_ARM_FPMR::fpmr | GETREGSET(nt=0x40e, len=8) read 8 bytes | Read NT_ARM_FPMR::fpmr = 0x900d900d900d900d Fixes: 4035c22ef7d4 ("arm64/ptrace: Expose FPMR via ptrace") Cc: # 6.9.x Signed-off-by: Mark Rutland Cc: Mark Brown Cc: Will Deacon Reviewed-by: Mark Brown Link: https://lore.kernel.org/r/20241205121655.1824269-3-mark.rutland@arm.com Signed-off-by: Catalin Marinas Signed-off-by: Bin Huang --- arch/arm64/kernel/ptrace.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c index 0ed89720d847..3d9f4b05bfce 100644 --- a/arch/arm64/kernel/ptrace.c +++ b/arch/arm64/kernel/ptrace.c @@ -719,6 +719,8 @@ static int fpmr_set(struct task_struct *target, const struct user_regset *regset if (!system_supports_fpmr()) return -EINVAL; + fpmr = target->thread.uw.fpmr; + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &fpmr, 0, count); if (ret) return ret; From 2a943c969392825060418cc0af664a455a19b04c Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Wed, 9 Apr 2025 17:40:03 +0100 Subject: [PATCH 23/85] arm64/fpsimd: Don't corrupt FPMR when streaming mode changes commit e5fa85fce08b21ed41643cb7968bf66bbd0532e3 upstream. When the effective value of PSTATE.SM is changed from 0 to 1 or from 1 to 0 by any method, an entry or exit to/from streaming SVE mode is performed, and hardware automatically resets a number of registers. As of ARM DDI 0487 L.a, this means: * All implemented bits of the SVE vector registers are set to zero. * All implemented bits of the SVE predicate registers are set to zero. * All implemented bits of FFR are set to zero, if FFR is implemented in the new mode. * FPSR is set to 0x0000_0000_0800_009f. * FPMR is set to 0, if FPMR is implemented. Currently task_fpsimd_load() restores FPMR before restoring SVCR (which is an accessor for PSTATE.{SM,ZA}), and so the restored value of FPMR may be clobbered if the restored value of PSTATE.SM happens to differ from the initial value of PSTATE.SM. Fix this by moving the restore of FPMR later. Note: this was originally posted as [1]. Fixes: 203f2b95a882 ("arm64/fpsimd: Support FEAT_FPMR") Signed-off-by: Mark Brown Link: https://lore.kernel.org/linux-arm-kernel/20241204-arm64-sme-reenable-v2-2-bae87728251d@kernel.org/ [ Rutland: rewrite commit message ] Signed-off-by: Mark Rutland Link: https://lore.kernel.org/r/20250409164010.3480271-7-mark.rutland@arm.com Signed-off-by: Catalin Marinas Signed-off-by: Bin Huang --- arch/arm64/kernel/fpsimd.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c index cd5d362a3e50..fdf2768161bb 100644 --- a/arch/arm64/kernel/fpsimd.c +++ b/arch/arm64/kernel/fpsimd.c @@ -385,9 +385,6 @@ static void task_fpsimd_load(void) WARN_ON(!system_supports_fpsimd()); WARN_ON(!have_cpu_fpsimd_context()); - if (system_supports_fpmr()) - write_sysreg_s(current->thread.uw.fpmr, SYS_FPMR); - if (system_supports_sve() || system_supports_sme()) { switch (current->thread.fp_type) { case FP_STATE_FPSIMD: @@ -439,6 +436,9 @@ static void task_fpsimd_load(void) restore_ffr = system_supports_fa64(); } + if (system_supports_fpmr()) + write_sysreg_s(current->thread.uw.fpmr, SYS_FPMR); + if (restore_sve_regs) { WARN_ON_ONCE(current->thread.fp_type != FP_STATE_SVE); sve_load_state(sve_pffr(¤t->thread), From 7ee62e4595fc344a3b681719bb503007f9e47a85 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Wed, 9 Apr 2025 17:40:05 +0100 Subject: [PATCH 24/85] arm64/fpsimd: Reset FPMR upon exec() commit a90878f297d3dba906a6261deccb1bd4a791ba52 upstream. An exec() is expected to reset all FPSIMD/SVE/SME state, and barring special handling of the vector lengths, the state is expected to reset to zero. This reset is handled in fpsimd_flush_thread(), which the core exec() code calls via flush_thread(). When support was added for FPMR, no logic was added to fpsimd_flush_thread() to reset the FPMR value, and thus it is erroneously inherited across an exec(). Add the missing reset of FPMR. Fixes: 203f2b95a882 ("arm64/fpsimd: Support FEAT_FPMR") Signed-off-by: Mark Rutland Cc: Marc Zyngier Cc: Mark Brown Cc: Will Deacon Reviewed-by: Mark Brown Link: https://lore.kernel.org/r/20250409164010.3480271-9-mark.rutland@arm.com Signed-off-by: Catalin Marinas Signed-off-by: Bin Huang --- arch/arm64/kernel/fpsimd.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c index fdf2768161bb..3fcaf054edf7 100644 --- a/arch/arm64/kernel/fpsimd.c +++ b/arch/arm64/kernel/fpsimd.c @@ -1691,6 +1691,9 @@ void fpsimd_flush_thread(void) current->thread.svcr = 0; } + if (system_supports_fpmr()) + current->thread.uw.fpmr = 0; + current->thread.fp_type = FP_STATE_FPSIMD; put_cpu_fpsimd_context(); From 12dc55fe741d7691c37a8400f16ba8bf904c1187 Mon Sep 17 00:00:00 2001 From: Andre Przywara Date: Fri, 16 Aug 2024 16:32:45 +0100 Subject: [PATCH 25/85] kselftest/arm64: hwcap: fix f8dp2 cpuinfo name commit b0d80dbc378d52155c9ecf9579986edccceed3aa upstream. The F8DP2 DPISA extension has a separate cpuinfo field, named accordingly. Change the erroneously placed name of "f8dp4" to "f8dp2". Fixes: 44d10c27bd75 ("kselftest/arm64: Add 2023 DPISA hwcap test coverage") Signed-off-by: Andre Przywara Reviewed-by: Mark Brown Link: https://lore.kernel.org/r/20240816153251.2833702-3-andre.przywara@arm.com Signed-off-by: Catalin Marinas Signed-off-by: Bin Huang --- tools/testing/selftests/arm64/abi/hwcap.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/arm64/abi/hwcap.c b/tools/testing/selftests/arm64/abi/hwcap.c index ed66253e8db5..c4f300d356d7 100644 --- a/tools/testing/selftests/arm64/abi/hwcap.c +++ b/tools/testing/selftests/arm64/abi/hwcap.c @@ -457,7 +457,7 @@ static const struct hwcap_data { .name = "F8DP2", .at_hwcap = AT_HWCAP2, .hwcap_bit = HWCAP2_F8DP2, - .cpuinfo = "f8dp4", + .cpuinfo = "f8dp2", .sigill_fn = f8dp2_sigill, }, { From 3c65fd886ca731d9a7b3b98843674ee41f34e4f6 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Mon, 28 Oct 2024 20:22:31 +0000 Subject: [PATCH 26/85] kselftest/arm64: Fix encoding for SVE B16B16 test commit 69c0d824779843b51ca2339b2163db4d3b40c54c upstream. The test for SVE_B16B16 had a cut'n'paste of a SME instruction, fix it with a relevant SVE instruction. Fixes: 44d10c27bd75 ("kselftest/arm64: Add 2023 DPISA hwcap test coverage") Signed-off-by: Mark Brown Link: https://lore.kernel.org/r/20241028-arm64-b16b16-test-v1-1-59a4a7449bdf@kernel.org Signed-off-by: Catalin Marinas Signed-off-by: Bin Huang --- tools/testing/selftests/arm64/abi/hwcap.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/arm64/abi/hwcap.c b/tools/testing/selftests/arm64/abi/hwcap.c index c4f300d356d7..dd821046ebd5 100644 --- a/tools/testing/selftests/arm64/abi/hwcap.c +++ b/tools/testing/selftests/arm64/abi/hwcap.c @@ -341,8 +341,8 @@ static void sveaes_sigill(void) static void sveb16b16_sigill(void) { - /* BFADD ZA.H[W0, 0], {Z0.H-Z1.H} */ - asm volatile(".inst 0xC1E41C00" : : : ); + /* BFADD Z0.H, Z0.H, Z0.H */ + asm volatile(".inst 0x65000000" : : : ); } static void svepmull_sigill(void) From a381a5522e02a446dd044a517a782e54e98c0574 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Wed, 9 Apr 2025 17:40:09 +0100 Subject: [PATCH 27/85] arm64/fpsimd: signal: Always save+flush state early commit 929fa99b1215966fc8a6ccc2e14b92e36c3c42f3 upstream. There are several issues with the way the native signal handling code manipulates FPSIMD/SVE/SME state, described in detail below. These issues largely result from races with preemption and inconsistent handling of live state vs saved state. Known issues with native FPSIMD/SVE/SME state management include: * On systems with FPMR, the code to save/restore the FPMR accesses the register while it is not owned by the current task. Consequently, this may corrupt the FPMR of the current task and/or may corrupt the FPMR of an unrelated task. The FPMR save/restore has been broken since it was introduced in commit: 8c46def44409fc91 ("arm64/signal: Add FPMR signal handling") * On systems with SME, setup_return() modifies both the live register state and the saved state register state regardless of whether the task's state is live, and without holding the cpu fpsimd context. Consequently: - This may corrupt the state an unrelated task which has PSTATE.SM set and/or PSTATE.ZA set. - The task may enter the signal handler in streaming mode, and or with ZA storage enabled unexpectedly. - The task may enter the signal handler in non-streaming SVE mode with stale SVE register state, which may have been inherited from streaming SVE mode unexpectedly. Where the streaming and non-streaming vector lengths differ, this may be packed into registers arbitrarily. This logic has been broken since it was introduced in commit: 40a8e87bb32855b3 ("arm64/sme: Disable ZA and streaming mode when handling signals") Further incorrect manipulation of state was added in commits: ea64baacbc36a0d5 ("arm64/signal: Flush FPSIMD register state when disabling streaming mode") baa8515281b30861 ("arm64/fpsimd: Track the saved FPSIMD state type separately to TIF_SVE") * Several restoration functions use fpsimd_flush_task_state() to discard the live FPSIMD/SVE/SME while the in-memory copy is stale. When a subset of the FPSIMD/SVE/SME state is restored, the remainder may be non-deterministically reset to a stale snapshot from some arbitrary point in the past. This non-deterministic discarding was introduced in commit: 8cd969d28fd2848d ("arm64/sve: Signal handling support") As of that commit, when TIF_SVE was initially clear, failure to restore the SVE signal frame could reset the FPSIMD registers to a stale snapshot. The pattern of discarding unsaved state was subsequently copied into restoration functions for some new state in commits: 39782210eb7e8763 ("arm64/sme: Implement ZA signal handling") ee072cf708048c0d ("arm64/sme: Implement signal handling for ZT") * On systems with SME/SME2, the entire FPSIMD/SVE/SME state may be loaded onto the CPU redundantly. Either restore_fpsimd_context() or restore_sve_fpsimd_context() will load the entire FPSIMD/SVE/SME state via fpsimd_update_current_state() before restore_za_context() and restore_zt_context() each discard the state via fpsimd_flush_task_state(). This is purely redundant work, and not a functional bug. To fix these issues, rework the native signal handling code to always save+flush the current task's FPSIMD/SVE/SME state before manipulating that state. This avoids races with preemption and ensures that state is manipulated consistently regardless of whether it happened to be live prior to manipulation. This largely involes: * Using fpsimd_save_and_flush_current_state() to save+flush the state for both signal delivery and signal return, before the state is manipulated in any way. * Removing fpsimd_signal_preserve_current_state() and updating preserve_fpsimd_context() to explicitly ensure that the FPSIMD state is up-to-date, as preserve_fpsimd_context() is the only consumer of the FPSIMD state during signal delivery. * Modifying fpsimd_update_current_state() to not reload the FPSIMD state onto the CPU. Ideally we'd remove fpsimd_update_current_state() entirely, but I've left that for subsequent patches as there are a number of of other problems with the FPSIMD<->SVE conversion helpers that should be addressed at the same time. For now I've removed the misleading comment. For setup_return(), we need to decide (for ABI reasons) whether signal delivery should have all the side-effects of an SMSTOP. For now I've left a TODO comment, as there are other questions in this area that I'll address with subsequent patches. Fixes: 8c46def44409 ("arm64/signal: Add FPMR signal handling") Fixes: 40a8e87bb328 ("arm64/sme: Disable ZA and streaming mode when handling signals") Fixes: ea64baacbc36 ("arm64/signal: Flush FPSIMD register state when disabling streaming mode") Fixes: baa8515281b3 ("arm64/fpsimd: Track the saved FPSIMD state type separately to TIF_SVE") Fixes: 8cd969d28fd2 ("arm64/sve: Signal handling support") Fixes: 39782210eb7e ("arm64/sme: Implement ZA signal handling") Fixes: ee072cf70804 ("arm64/sme: Implement signal handling for ZT") Signed-off-by: Mark Rutland Cc: Marc Zyngier Cc: Mark Brown Cc: Will Deacon Reviewed-by: Mark Brown Link: https://lore.kernel.org/r/20250409164010.3480271-13-mark.rutland@arm.com Signed-off-by: Catalin Marinas [ bh: veLinux 6.6.151 adapts the upstream saved-state transition to the target save+flush model while preserving SVE/SME safeguards. ] Signed-off-by: Bin Huang --- arch/arm64/include/asm/fpsimd.h | 1 - arch/arm64/kernel/fpsimd.c | 28 -------------- arch/arm64/kernel/signal.c | 66 ++++++--------------------------- 3 files changed, 12 insertions(+), 83 deletions(-) diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h index 7eb37449a316..ad80cf8de27a 100644 --- a/arch/arm64/include/asm/fpsimd.h +++ b/arch/arm64/include/asm/fpsimd.h @@ -79,7 +79,6 @@ extern void fpsimd_load_state(struct user_fpsimd_state *state); extern void fpsimd_thread_switch(struct task_struct *next); extern void fpsimd_flush_thread(void); -extern void fpsimd_signal_preserve_current_state(void); extern void fpsimd_preserve_current_state(void); extern void fpsimd_restore_current_state(void); extern void fpsimd_update_current_state(struct user_fpsimd_state const *state); diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c index 3fcaf054edf7..e7ccb5da4be0 100644 --- a/arch/arm64/kernel/fpsimd.c +++ b/arch/arm64/kernel/fpsimd.c @@ -1715,18 +1715,6 @@ void fpsimd_preserve_current_state(void) put_cpu_fpsimd_context(); } -/* - * Like fpsimd_preserve_current_state(), but ensure that - * current->thread.uw.fpsimd_state is updated so that it can be copied to - * the signal frame. - */ -void fpsimd_signal_preserve_current_state(void) -{ - fpsimd_preserve_current_state(); - if (current->thread.fp_type == FP_STATE_SVE) - sve_to_fpsimd(current); -} - /* * Associate current's FPSIMD context with this cpu * The caller must have ownership of the cpu FPSIMD context before calling @@ -1819,30 +1807,14 @@ void fpsimd_restore_current_state(void) put_cpu_fpsimd_context(); } -/* - * Load an updated userland FPSIMD state for 'current' from memory and set the - * flag that indicates that the FPSIMD register contents are the most recent - * FPSIMD state of 'current'. This is used by the signal code to restore the - * register state when returning from a signal handler in FPSIMD only cases, - * any SVE context will be discarded. - */ void fpsimd_update_current_state(struct user_fpsimd_state const *state) { if (WARN_ON(!system_supports_fpsimd())) return; - get_cpu_fpsimd_context(); - current->thread.uw.fpsimd_state = *state; if (current->thread.fp_type == FP_STATE_SVE) fpsimd_to_sve(current); - - task_fpsimd_load(); - fpsimd_bind_task_to_cpu(); - - clear_thread_flag(TIF_FOREIGN_FPSTATE); - - put_cpu_fpsimd_context(); } /* diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c index 7763134802c7..14142c234918 100644 --- a/arch/arm64/kernel/signal.c +++ b/arch/arm64/kernel/signal.c @@ -194,6 +194,8 @@ static int preserve_fpsimd_context(struct fpsimd_context __user *ctx) ¤t->thread.uw.fpsimd_state; int err; + sve_sync_to_fpsimd(current); + /* copy the FP and status/control registers */ err = __copy_to_user(ctx->vregs, fpsimd->vregs, sizeof(fpsimd->vregs)); __put_user_error(fpsimd->fpsr, &ctx->fpsr, err); @@ -245,8 +247,6 @@ static int preserve_fpmr_context(struct fpmr_context __user *ctx) { int err = 0; - current->thread.uw.fpmr = read_sysreg_s(SYS_FPMR); - __put_user_error(FPMR_MAGIC, &ctx->head.magic, err); __put_user_error(sizeof(*ctx), &ctx->head.size, err); __put_user_error(current->thread.uw.fpmr, &ctx->fpmr, err); @@ -264,7 +264,7 @@ static int restore_fpmr_context(struct user_ctxs *user) __get_user_error(fpmr, &user->fpmr->fpmr, err); if (!err) - write_sysreg_s(fpmr, SYS_FPMR); + current->thread.uw.fpmr = fpmr; return err; } @@ -298,11 +298,6 @@ static int preserve_sve_context(struct sve_context __user *ctx) err |= __copy_to_user(&ctx->__reserved, reserved, sizeof(reserved)); if (vq) { - /* - * This assumes that the SVE state has already been saved to - * the task struct by calling the function - * fpsimd_signal_preserve_current_state(). - */ err |= __copy_to_user((char __user *)ctx + SVE_SIG_REGS_OFFSET, current->thread.sve_state, SVE_SIG_REGS_SIZE(vq)); @@ -365,16 +360,6 @@ static int restore_sve_fpsimd_context(struct user_ctxs *user) if (user->sve_size < SVE_SIG_CONTEXT_SIZE(vq)) return -EINVAL; - /* - * Careful: we are about __copy_from_user() directly into - * thread.sve_state with preemption enabled, so protection is - * needed to prevent a racing context switch from writing stale - * registers back over the new data. - */ - - fpsimd_flush_task_state(current); - /* From now, fpsimd_thread_switch() won't touch thread.sve_state */ - if (sm) { sme_alloc(current, false); if (!current->thread.sme_state) @@ -479,11 +464,6 @@ static int preserve_za_context(struct za_context __user *ctx) err |= __copy_to_user(&ctx->__reserved, reserved, sizeof(reserved)); if (vq) { - /* - * This assumes that the ZA state has already been saved to - * the task struct by calling the function - * fpsimd_signal_preserve_current_state(). - */ err |= __copy_to_user((char __user *)ctx + ZA_SIG_REGS_OFFSET, current->thread.sme_state, ZA_SIG_REGS_SIZE(vq)); @@ -518,16 +498,6 @@ static int restore_za_context(struct user_ctxs *user) if (user->za_size < ZA_SIG_CONTEXT_SIZE(vq)) return -EINVAL; - /* - * Careful: we are about __copy_from_user() directly into - * thread.sme_state with preemption enabled, so protection is - * needed to prevent a racing context switch from writing stale - * registers back over the new data. - */ - - fpsimd_flush_task_state(current); - /* From now, fpsimd_thread_switch() won't touch thread.sve_state */ - sve_alloc(current, false); if (!current->thread.sve_state) return -ENOMEM; @@ -569,11 +539,6 @@ static int preserve_zt_context(struct zt_context __user *ctx) BUILD_BUG_ON(sizeof(ctx->__reserved) != sizeof(reserved)); err |= __copy_to_user(&ctx->__reserved, reserved, sizeof(reserved)); - /* - * This assumes that the ZT state has already been saved to - * the task struct by calling the function - * fpsimd_signal_preserve_current_state(). - */ err |= __copy_to_user((char __user *)ctx + ZT_SIG_REGS_OFFSET, thread_zt_state(¤t->thread), ZT_SIG_REGS_SIZE(1)); @@ -599,16 +564,6 @@ static int restore_zt_context(struct user_ctxs *user) if (nregs != 1) return -EINVAL; - /* - * Careful: we are about __copy_from_user() directly into - * thread.zt_state with preemption enabled, so protection is - * needed to prevent a racing context switch from writing stale - * registers back over the new data. - */ - - fpsimd_flush_task_state(current); - /* From now, fpsimd_thread_switch() won't touch ZT in thread state */ - err = __copy_from_user(thread_zt_state(¤t->thread), (char __user const *)user->zt + ZT_SIG_REGS_OFFSET, @@ -858,6 +813,8 @@ static int restore_sigframe(struct pt_regs *regs, */ forget_syscall(regs); + fpsimd_save_and_flush_current_state(); + err |= !valid_user_regs(®s->user_regs, current); if (err == 0) err = parse_user_sigframe(&user, sf); @@ -1191,8 +1148,11 @@ static void setup_return(struct pt_regs *regs, struct k_sigaction *ka, /* * If we were in streaming mode the saved register * state was SVE but we will exit SM and use the - * FPSIMD register state - flush the saved FPSIMD - * register state in case it gets loaded. + * FPSIMD register state. + * + * TODO: decide if this should behave as SMSTOP (e.g. reset + * FPSR + FPMR), or whether this should only clear the scalable + * registers + ZA state. */ if (current->thread.svcr & SVCR_SM_MASK) { memset(¤t->thread.uw.fpsimd_state, 0, @@ -1200,9 +1160,7 @@ static void setup_return(struct pt_regs *regs, struct k_sigaction *ka, current->thread.fp_type = FP_STATE_FPSIMD; } - current->thread.svcr &= ~(SVCR_ZA_MASK | - SVCR_SM_MASK); - sme_smstop(); + current->thread.svcr &= ~(SVCR_ZA_MASK | SVCR_SM_MASK); } if (ka->sa.sa_flags & SA_RESTORER) @@ -1220,7 +1178,7 @@ static int setup_rt_frame(int usig, struct ksignal *ksig, sigset_t *set, struct rt_sigframe __user *frame; int err = 0; - fpsimd_signal_preserve_current_state(); + fpsimd_save_and_flush_current_state(); if (get_sigframe(&user, ksig, regs)) return 1; From 939cf75a5baaeb8825547b6abd634823cc904f52 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Wed, 30 Apr 2025 18:32:40 +0100 Subject: [PATCH 28/85] arm64/fpsimd: Avoid warning when sve_to_fpsimd() is unused commit f699c66691fb7e08a5a631c5baf5f2a19b7a6468 upstream. Historically fpsimd_to_sve() and sve_to_fpsimd() were (conditionally) called by functions which were defined regardless of CONFIG_ARM64_SVE. Hence it was necessary that both fpsimd_to_sve() and sve_to_fpsimd() were always defined and not guarded by ifdeffery. As a result of the removal of fpsimd_signal_preserve_current_state() in commit: 929fa99b1215966f ("arm64/fpsimd: signal: Always save+flush state early") ... sve_to_fpsimd() has no callers when CONFIG_ARM64_SVE=n, resulting in a build-time warnign that it is unused: | arch/arm64/kernel/fpsimd.c:676:13: warning: unused function 'sve_to_fpsimd' [-Wunused-function] | 676 | static void sve_to_fpsimd(struct task_struct *task) | | ^~~~~~~~~~~~~ | 1 warning generated. In contrast, fpsimd_to_sve() still has callers which are defined when CONFIG_ARM64_SVE=n, and it would be awkward to hide this behind ifdeffery and/or to use stub functions. For now, suppress the warning by marking both fpsimd_to_sve() and sve_to_fpsimd() as 'static inline', as we usually do for stub functions. The compiler will no longer warn if either function is unused. Aside from suppressing the warning, there should be no functional change as a result of this patch. Link: https://lore.kernel.org/linux-arm-kernel/20250429194600.GA26883@willie-the-truck/ Reported-by: Will Deacon Fixes: 929fa99b1215 ("arm64/fpsimd: signal: Always save+flush state early") Signed-off-by: Mark Rutland Cc: Marc Zyngier Cc: Mark Brown Cc: Will Deacon Link: https://lore.kernel.org/r/20250430173240.4023627-1-mark.rutland@arm.com Signed-off-by: Catalin Marinas Signed-off-by: Bin Huang --- arch/arm64/kernel/fpsimd.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c index e7ccb5da4be0..5ee298c70261 100644 --- a/arch/arm64/kernel/fpsimd.c +++ b/arch/arm64/kernel/fpsimd.c @@ -682,7 +682,7 @@ static void __fpsimd_to_sve(void *sst, struct user_fpsimd_state const *fst, * task->thread.uw.fpsimd_state must be up to date before calling this * function. */ -static void fpsimd_to_sve(struct task_struct *task) +static inline void fpsimd_to_sve(struct task_struct *task) { unsigned int vq; void *sst = task->thread.sve_state; @@ -706,7 +706,7 @@ static void fpsimd_to_sve(struct task_struct *task) * bytes of allocated kernel memory. * task->thread.sve_state must be up to date before calling this function. */ -static void sve_to_fpsimd(struct task_struct *task) +static inline void sve_to_fpsimd(struct task_struct *task) { unsigned int vq, vl; void const *sst = task->thread.sve_state; From 72631a6882d543c0b468e14a4ec6fe5c39c2ebf9 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 8 Dec 2023 12:32:20 +0100 Subject: [PATCH 29/85] arm64: fpsimd: Drop unneeded 'busy' flag commit 9b19700e623f96222c69ecb2adecb1a3e3664cc0 upstream. Kernel mode NEON will preserve the user mode FPSIMD state by saving it into the task struct before clobbering the registers. In order to avoid the need for preserving kernel mode state too, we disallow nested use of kernel mode NEON, i..e, use in softirq context while the interrupted task context was using kernel mode NEON too. Originally, this policy was implemented using a per-CPU flag which was exposed via may_use_simd(), requiring the users of the kernel mode NEON to deal with the possibility that it might return false, and having NEON and non-NEON code paths. This policy was changed by commit 13150149aa6ded1 ("arm64: fpsimd: run kernel mode NEON with softirqs disabled"), and now, softirq processing is disabled entirely instead, and so may_use_simd() can never fail when called from task or softirq context. This means we can drop the fpsimd_context_busy flag entirely, and instead, ensure that we disable softirq processing in places where we formerly relied on the flag for preventing races in the FPSIMD preserve routines. Signed-off-by: Ard Biesheuvel Reviewed-by: Mark Brown Tested-by: Geert Uytterhoeven Link: https://lore.kernel.org/r/20231208113218.3001940-7-ardb@google.com [will: Folded in fix from CAMj1kXFhzbJRyWHELCivQW1yJaF=p07LLtbuyXYX3G1WtsdyQg@mail.gmail.com] Signed-off-by: Will Deacon [bh: veLinux 6.6.151 keeps IRQ handling and drops obsolete busy-flag ops.] Signed-off-by: Bin Huang --- arch/arm64/include/asm/simd.h | 11 +------- arch/arm64/kernel/fpsimd.c | 51 +++++++---------------------------- 2 files changed, 11 insertions(+), 51 deletions(-) diff --git a/arch/arm64/include/asm/simd.h b/arch/arm64/include/asm/simd.h index 6a75d7ecdcaa..8e86c9e70e48 100644 --- a/arch/arm64/include/asm/simd.h +++ b/arch/arm64/include/asm/simd.h @@ -12,8 +12,6 @@ #include #include -DECLARE_PER_CPU(bool, fpsimd_context_busy); - #ifdef CONFIG_KERNEL_MODE_NEON /* @@ -28,17 +26,10 @@ static __must_check inline bool may_use_simd(void) /* * We must make sure that the SVE has been initialized properly * before using the SIMD in kernel. - * fpsimd_context_busy is only set while preemption is disabled, - * and is clear whenever preemption is enabled. Since - * this_cpu_read() is atomic w.r.t. preemption, fpsimd_context_busy - * cannot change under our feet -- if it's set we cannot be - * migrated, and if it's clear we cannot be migrated to a CPU - * where it is set. */ return !WARN_ON(!system_capabilities_finalized()) && system_supports_fpsimd() && - !in_hardirq() && !irqs_disabled() && !in_nmi() && - !this_cpu_read(fpsimd_context_busy); + !in_hardirq() && !irqs_disabled() && !in_nmi(); } #else /* ! CONFIG_KERNEL_MODE_NEON */ diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c index 5ee298c70261..068010995a4c 100644 --- a/arch/arm64/kernel/fpsimd.c +++ b/arch/arm64/kernel/fpsimd.c @@ -85,13 +85,13 @@ * softirq kicks in. Upon vcpu_put(), KVM will save the vcpu FP state and * flag the register state as invalid. * - * In order to allow softirq handlers to use FPSIMD, kernel_neon_begin() may - * save the task's FPSIMD context back to task_struct from softirq context. - * To prevent this from racing with the manipulation of the task's FPSIMD state - * from task context and thereby corrupting the state, it is necessary to - * protect any manipulation of a task's fpsimd_state or TIF_FOREIGN_FPSTATE - * flag with {, __}get_cpu_fpsimd_context(). This will still allow softirqs to - * run but prevent them to use FPSIMD. + * In order to allow softirq handlers to use FPSIMD, kernel_neon_begin() may be + * called from softirq context, which will save the task's FPSIMD context back + * to task_struct. To prevent this from racing with the manipulation of the + * task's FPSIMD state from task context and thereby corrupting the state, it + * is necessary to protect any manipulation of a task's fpsimd_state or + * TIF_FOREIGN_FPSTATE flag with get_cpu_fpsimd_context(), which will suspend + * softirq servicing entirely until put_cpu_fpsimd_context() is called. * * For a certain task, the sequence may look something like this: * - the task gets scheduled in; if both the task's fpsimd_cpu field @@ -209,27 +209,14 @@ static inline void sme_free(struct task_struct *t) { } #endif -DEFINE_PER_CPU(bool, fpsimd_context_busy); -EXPORT_PER_CPU_SYMBOL(fpsimd_context_busy); - static void fpsimd_bind_task_to_cpu(void); -static void __get_cpu_fpsimd_context(void) -{ - bool busy = __this_cpu_xchg(fpsimd_context_busy, true); - - WARN_ON(busy); -} - /* * Claim ownership of the CPU FPSIMD context for use by the calling context. * * The caller may freely manipulate the FPSIMD context metadata until * put_cpu_fpsimd_context() is called. * - * The double-underscore version must only be called if you know the task - * can't be preempted. - * * On RT kernels local_bh_disable() is not sufficient because it only * serializes soft interrupt related sections via a local lock, but stays * preemptible. Disabling preemption is the right choice here as bottom @@ -242,14 +229,6 @@ static void get_cpu_fpsimd_context(void) local_bh_disable(); else preempt_disable(); - __get_cpu_fpsimd_context(); -} - -static void __put_cpu_fpsimd_context(void) -{ - bool busy = __this_cpu_xchg(fpsimd_context_busy, false); - - WARN_ON(!busy); /* No matching get_cpu_fpsimd_context()? */ } /* @@ -261,18 +240,12 @@ static void __put_cpu_fpsimd_context(void) */ static void put_cpu_fpsimd_context(void) { - __put_cpu_fpsimd_context(); if (!IS_ENABLED(CONFIG_PREEMPT_RT)) local_bh_enable(); else preempt_enable(); } -static bool have_cpu_fpsimd_context(void) -{ - return !preemptible() && __this_cpu_read(fpsimd_context_busy); -} - unsigned int task_get_vl(const struct task_struct *task, enum vec_type type) { return task->thread.vl[type]; @@ -383,7 +356,7 @@ static void task_fpsimd_load(void) bool restore_ffr; WARN_ON(!system_supports_fpsimd()); - WARN_ON(!have_cpu_fpsimd_context()); + WARN_ON(preemptible()); if (system_supports_sve() || system_supports_sme()) { switch (current->thread.fp_type) { @@ -470,7 +443,7 @@ static void fpsimd_save(void) unsigned int vl; WARN_ON(!system_supports_fpsimd()); - WARN_ON(!have_cpu_fpsimd_context()); + WARN_ON(preemptible()); if (test_thread_flag(TIF_FOREIGN_FPSTATE)) return; @@ -1601,7 +1574,7 @@ void fpsimd_thread_switch(struct task_struct *next) if (!system_supports_fpsimd()) return; - __get_cpu_fpsimd_context(); + WARN_ON_ONCE(!irqs_disabled()); /* Save unsaved fpsimd state, if any: */ fpsimd_save(); @@ -1617,8 +1590,6 @@ void fpsimd_thread_switch(struct task_struct *next) update_tsk_thread_flag(next, TIF_FOREIGN_FPSTATE, wrong_task || wrong_cpu); - - __put_cpu_fpsimd_context(); } static void fpsimd_flush_thread_vl(enum vec_type type) @@ -1888,10 +1859,8 @@ void fpsimd_save_and_flush_cpu_state(void) return; WARN_ON(preemptible()); local_irq_save(flags); - __get_cpu_fpsimd_context(); fpsimd_save(); fpsimd_flush_cpu_state(); - __put_cpu_fpsimd_context(); local_irq_restore(flags); } From 47a8ee70378fd0a8de278026a25135009208a827 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 8 Dec 2023 12:32:21 +0100 Subject: [PATCH 30/85] arm64: fpsimd: Preserve/restore kernel mode NEON at context switch commit aefbab8e77eb16b56e18f24b85a09ebf4dc60e93 upstream. Currently, the FPSIMD register file is not preserved and restored along with the general registers on exception entry/exit or context switch. For this reason, we disable preemption when enabling FPSIMD for kernel mode use in task context, and suspend the processing of softirqs so that there are no concurrent uses in the kernel. (Kernel mode FPSIMD may not be used at all in other contexts). Disabling preemption while doing CPU intensive work on inputs of potentially unbounded size is bad for real-time performance, which is why we try and ensure that SIMD crypto code does not operate on more than ~4k at a time, which is an arbitrary limit and requires assembler code to implement efficiently. We can avoid the need for disabling preemption if we can ensure that any in-kernel users of the NEON will not lose the FPSIMD register state across a context switch. And given that disabling softirqs implicitly disables preemption as well, we will also have to ensure that a softirq that runs code using FPSIMD can safely interrupt an in-kernel user. So introduce a thread_info flag TIF_KERNEL_FPSTATE, and modify the context switch hook for FPSIMD to preserve and restore the kernel mode FPSIMD to/from struct thread_struct when it is set. This avoids any scheduling blackouts due to prolonged use of FPSIMD in kernel mode, without the need for manual yielding. In order to support softirq processing while FPSIMD is being used in kernel task context, use the same flag to decide whether the kernel mode FPSIMD state needs to be preserved and restored before allowing FPSIMD to be used in softirq context. Signed-off-by: Ard Biesheuvel Reviewed-by: Mark Brown Reviewed-by: Mark Rutland Link: https://lore.kernel.org/r/20231208113218.3001940-8-ardb@google.com Signed-off-by: Will Deacon [bh: veLinux 6.6 adaptation for fpsimd_save_and_flush_current_state(). ] Signed-off-by: Bin Huang --- arch/arm64/include/asm/processor.h | 2 + arch/arm64/include/asm/thread_info.h | 1 + arch/arm64/kernel/fpsimd.c | 94 ++++++++++++++++++++++------ 3 files changed, 78 insertions(+), 19 deletions(-) diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h index 62d932d05bff..4f122b932cca 100644 --- a/arch/arm64/include/asm/processor.h +++ b/arch/arm64/include/asm/processor.h @@ -169,6 +169,8 @@ struct thread_struct { unsigned long fault_address; /* fault info */ unsigned long fault_code; /* ESR_EL1 value */ struct debug_info debug; /* debugging */ + + struct user_fpsimd_state kernel_fpsimd_state; #ifdef CONFIG_ARM64_PTR_AUTH struct ptrauth_keys_user keys_user; #ifdef CONFIG_ARM64_PTR_AUTH_KERNEL diff --git a/arch/arm64/include/asm/thread_info.h b/arch/arm64/include/asm/thread_info.h index b1266202257f..884dcfe8c8ef 100644 --- a/arch/arm64/include/asm/thread_info.h +++ b/arch/arm64/include/asm/thread_info.h @@ -81,6 +81,7 @@ void arch_setup_new_exec(void); #define TIF_TAGGED_ADDR 26 /* Allow tagged user addresses */ #define TIF_SME 27 /* SME in use */ #define TIF_SME_VL_INHERIT 28 /* Inherit SME vl_onexec across exec */ +#define TIF_KERNEL_FPSTATE 29 /* Task is in a kernel mode FPSIMD section */ #define _TIF_SIGPENDING (1 << TIF_SIGPENDING) #define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED) diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c index 068010995a4c..c064d2322d99 100644 --- a/arch/arm64/kernel/fpsimd.c +++ b/arch/arm64/kernel/fpsimd.c @@ -357,6 +357,7 @@ static void task_fpsimd_load(void) WARN_ON(!system_supports_fpsimd()); WARN_ON(preemptible()); + WARN_ON(test_thread_flag(TIF_KERNEL_FPSTATE)); if (system_supports_sve() || system_supports_sme()) { switch (current->thread.fp_type) { @@ -379,7 +380,7 @@ static void task_fpsimd_load(void) default: /* * This indicates either a bug in - * fpsimd_save() or memory corruption, we + * fpsimd_save_user_state() or memory corruption, we * should always record an explicit format * when we save. We always at least have the * memory allocated for FPSMID registers so @@ -433,7 +434,7 @@ static void task_fpsimd_load(void) * than via current, if we are saving KVM state then it will have * ensured that the type of registers to save is set in last->to_save. */ -static void fpsimd_save(void) +static void fpsimd_save_user_state(void) { struct cpu_fp_state const *last = this_cpu_ptr(&fpsimd_last_state); @@ -878,7 +879,7 @@ int vec_set_vector_length(struct task_struct *task, enum vec_type type, if (task == current) { get_cpu_fpsimd_context(); - fpsimd_save(); + fpsimd_save_user_state(); } fpsimd_flush_task_state(task); @@ -1567,6 +1568,16 @@ void do_fpsimd_exc(unsigned long esr, struct pt_regs *regs) current); } +static void fpsimd_load_kernel_state(struct task_struct *task) +{ + fpsimd_load_state(&task->thread.kernel_fpsimd_state); +} + +static void fpsimd_save_kernel_state(struct task_struct *task) +{ + fpsimd_save_state(&task->thread.kernel_fpsimd_state); +} + void fpsimd_thread_switch(struct task_struct *next) { bool wrong_task, wrong_cpu; @@ -1577,19 +1588,28 @@ void fpsimd_thread_switch(struct task_struct *next) WARN_ON_ONCE(!irqs_disabled()); /* Save unsaved fpsimd state, if any: */ - fpsimd_save(); + if (test_thread_flag(TIF_KERNEL_FPSTATE)) + fpsimd_save_kernel_state(current); + else + fpsimd_save_user_state(); - /* - * Fix up TIF_FOREIGN_FPSTATE to correctly describe next's - * state. For kernel threads, FPSIMD registers are never loaded - * and wrong_task and wrong_cpu will always be true. - */ - wrong_task = __this_cpu_read(fpsimd_last_state.st) != - &next->thread.uw.fpsimd_state; - wrong_cpu = next->thread.fpsimd_cpu != smp_processor_id(); + if (test_tsk_thread_flag(next, TIF_KERNEL_FPSTATE)) { + fpsimd_load_kernel_state(next); + set_tsk_thread_flag(next, TIF_FOREIGN_FPSTATE); + } else { + /* + * Fix up TIF_FOREIGN_FPSTATE to correctly describe next's + * state. For kernel threads, FPSIMD registers are never + * loaded with user mode FPSIMD state and so wrong_task and + * wrong_cpu will always be true. + */ + wrong_task = __this_cpu_read(fpsimd_last_state.st) != + &next->thread.uw.fpsimd_state; + wrong_cpu = next->thread.fpsimd_cpu != smp_processor_id(); - update_tsk_thread_flag(next, TIF_FOREIGN_FPSTATE, - wrong_task || wrong_cpu); + update_tsk_thread_flag(next, TIF_FOREIGN_FPSTATE, + wrong_task || wrong_cpu); + } } static void fpsimd_flush_thread_vl(enum vec_type type) @@ -1682,7 +1702,7 @@ void fpsimd_preserve_current_state(void) return; get_cpu_fpsimd_context(); - fpsimd_save(); + fpsimd_save_user_state(); put_cpu_fpsimd_context(); } @@ -1821,7 +1841,7 @@ void fpsimd_save_and_flush_current_state(void) return; get_cpu_fpsimd_context(); - fpsimd_save(); + fpsimd_save_user_state(); fpsimd_flush_task_state(current); put_cpu_fpsimd_context(); } @@ -1859,7 +1879,7 @@ void fpsimd_save_and_flush_cpu_state(void) return; WARN_ON(preemptible()); local_irq_save(flags); - fpsimd_save(); + fpsimd_save_user_state(); fpsimd_flush_cpu_state(); local_irq_restore(flags); } @@ -1893,10 +1913,37 @@ void kernel_neon_begin(void) get_cpu_fpsimd_context(); /* Save unsaved fpsimd state, if any: */ - fpsimd_save(); + if (test_thread_flag(TIF_KERNEL_FPSTATE)) { + BUG_ON(IS_ENABLED(CONFIG_PREEMPT_RT) || !in_serving_softirq()); + fpsimd_save_kernel_state(current); + } else { + fpsimd_save_user_state(); + + /* + * Set the thread flag so that the kernel mode FPSIMD state + * will be context switched along with the rest of the task + * state. + * + * On non-PREEMPT_RT, softirqs may interrupt task level kernel + * mode FPSIMD, but the task will not be preemptible so setting + * TIF_KERNEL_FPSTATE for those would be both wrong (as it + * would mark the task context FPSIMD state as requiring a + * context switch) and unnecessary. + * + * On PREEMPT_RT, softirqs are serviced from a separate thread, + * which is scheduled as usual, and this guarantees that these + * softirqs are not interrupting use of the FPSIMD in kernel + * mode in task context. So in this case, setting the flag here + * is always appropriate. + */ + if (IS_ENABLED(CONFIG_PREEMPT_RT) || !in_serving_softirq()) + set_thread_flag(TIF_KERNEL_FPSTATE); + } /* Invalidate any task state remaining in the fpsimd regs: */ fpsimd_flush_cpu_state(); + + put_cpu_fpsimd_context(); } EXPORT_SYMBOL_GPL(kernel_neon_begin); @@ -1914,7 +1961,16 @@ void kernel_neon_end(void) if (!system_supports_fpsimd()) return; - put_cpu_fpsimd_context(); + /* + * If we are returning from a nested use of kernel mode FPSIMD, restore + * the task context kernel mode FPSIMD state. This can only happen when + * running in softirq context on non-PREEMPT_RT. + */ + if (!IS_ENABLED(CONFIG_PREEMPT_RT) && in_serving_softirq() && + test_thread_flag(TIF_KERNEL_FPSTATE)) + fpsimd_load_kernel_state(current); + else + clear_thread_flag(TIF_KERNEL_FPSTATE); } EXPORT_SYMBOL_GPL(kernel_neon_end); From 400ce9fde39885c435b483c369363121cf105a54 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 11 Jan 2024 12:24:48 +0100 Subject: [PATCH 31/85] arm64: fpsimd: Bring cond_yield asm macro in line with new rules commit 3931261ecf46151a5e779c96fb3da00677b6dc37 upstream. We no longer disable softirqs or preemption when doing kernel mode SIMD, and so for fully preemptible kernels, there is no longer a need to do any explicit yielding (and for non-preemptible kernels, yielding is not needed either). That leaves voluntary preemption, where only explicit yield calls may result in a reschedule. To retain the existing behavior for such a configuration, we should take the new situation into account, where the preempt count will be zero rather than one, and yielding to pending softirqs is unnecessary. Fixes: aefbab8e77eb ("arm64: fpsimd: Preserve/restore kernel mode NEON at context switch") Signed-off-by: Ard Biesheuvel Reviewed-by: Mark Brown Link: https://lore.kernel.org/r/20240111112447.577640-2-ardb+git@google.com Signed-off-by: Will Deacon Signed-off-by: Bin Huang --- arch/arm64/include/asm/assembler.h | 25 +++++++++---------------- arch/arm64/kernel/asm-offsets.c | 2 -- 2 files changed, 9 insertions(+), 18 deletions(-) diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h index 7e3e6d7e40ac..e92a0fef09b8 100644 --- a/arch/arm64/include/asm/assembler.h +++ b/arch/arm64/include/asm/assembler.h @@ -766,32 +766,25 @@ alternative_endif .endm /* - * Check whether preempt/bh-disabled asm code should yield as soon as - * it is able. This is the case if we are currently running in task - * context, and either a softirq is pending, or the TIF_NEED_RESCHED - * flag is set and re-enabling preemption a single time would result in - * a preempt count of zero. (Note that the TIF_NEED_RESCHED flag is - * stored negated in the top word of the thread_info::preempt_count + * Check whether asm code should yield as soon as it is able. This is + * the case if we are currently running in task context, and the + * TIF_NEED_RESCHED flag is set. (Note that the TIF_NEED_RESCHED flag + * is stored negated in the top word of the thread_info::preempt_count * field) */ - .macro cond_yield, lbl:req, tmp:req, tmp2:req + .macro cond_yield, lbl:req, tmp:req, tmp2 +#ifdef CONFIG_PREEMPT_VOLUNTARY get_current_task \tmp ldr \tmp, [\tmp, #TSK_TI_PREEMPT] /* * If we are serving a softirq, there is no point in yielding: the * softirq will not be preempted no matter what we do, so we should - * run to completion as quickly as we can. + * run to completion as quickly as we can. The preempt_count field will + * have BIT(SOFTIRQ_SHIFT) set in this case, so the zero check will + * catch this case too. */ - tbnz \tmp, #SOFTIRQ_SHIFT, .Lnoyield_\@ -#ifdef CONFIG_PREEMPTION - sub \tmp, \tmp, #PREEMPT_DISABLE_OFFSET cbz \tmp, \lbl #endif - adr_l \tmp, irq_stat + IRQ_CPUSTAT_SOFTIRQ_PENDING - get_this_cpu_offset \tmp2 - ldr w\tmp, [\tmp, \tmp2] - cbnz w\tmp, \lbl // yield on pending softirq in task context -.Lnoyield_\@: .endm /* diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c index ea2d740db81c..37deeca7f4c0 100644 --- a/arch/arm64/kernel/asm-offsets.c +++ b/arch/arm64/kernel/asm-offsets.c @@ -118,8 +118,6 @@ int main(void) DEFINE(DMA_FROM_DEVICE, DMA_FROM_DEVICE); BLANK(); DEFINE(PREEMPT_DISABLE_OFFSET, PREEMPT_DISABLE_OFFSET); - DEFINE(SOFTIRQ_SHIFT, SOFTIRQ_SHIFT); - DEFINE(IRQ_CPUSTAT_SOFTIRQ_PENDING, offsetof(irq_cpustat_t, __softirq_pending)); BLANK(); DEFINE(CPU_BOOT_TASK, offsetof(struct secondary_data, task)); BLANK(); From 951c9023de6b434ec0a08f715d8820c93d79f717 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Wed, 22 May 2024 11:13:36 +0200 Subject: [PATCH 32/85] arm64/fpsimd: Avoid erroneous elide of user state reload commit e92bee9f861b466c676f0200be3e46af7bc4ac6b upstream. TIF_FOREIGN_FPSTATE is a 'convenience' flag that should reflect whether the current CPU holds the most recent user mode FP/SIMD state of the current task. It combines two conditions: - whether the current CPU's FP/SIMD state belongs to the task; - whether that state is the most recent associated with the task (as a task may have executed on other CPUs as well). When a task is scheduled in and TIF_KERNEL_FPSTATE is set, it means the task was in a kernel mode NEON section when it was scheduled out, and so the kernel mode FP/SIMD state is restored. Since this implies that the current CPU is *not* holding the most recent user mode FP/SIMD state of the current task, the TIF_FOREIGN_FPSTATE flag is set too, so that the user mode FP/SIMD state is reloaded from memory when returning to userland. However, the task may be scheduled out after completing the kernel mode NEON section, but before returning to userland. When this happens, the TIF_FOREIGN_FPSTATE flag will not be preserved, but will be set as usual the next time the task is scheduled in, and will be based on the above conditions. This means that, rather than setting TIF_FOREIGN_FPSTATE when scheduling in a task with TIF_KERNEL_FPSTATE set, the underlying state should be updated so that TIF_FOREIGN_FPSTATE will assume the expected value as a result. So instead, call fpsimd_flush_cpu_state(), which takes care of this. Closes: https://lore.kernel.org/all/cb8822182231850108fa43e0446a4c7f@kernel.org Reported-by: Johannes Nixdorf Fixes: aefbab8e77eb ("arm64: fpsimd: Preserve/restore kernel mode NEON at context switch") Cc: Mark Brown Cc: Dave Martin Cc: Janne Grunau Cc: stable@vger.kernel.org Signed-off-by: Ard Biesheuvel Tested-by: Janne Grunau Tested-by: Johannes Nixdorf Reviewed-by: Mark Brown Link: https://lore.kernel.org/r/20240522091335.335346-2-ardb+git@google.com Signed-off-by: Will Deacon [bh: veLinux 6.6.151 applies the state invalidation fix and relocates the existing flush helper unchanged.] Signed-off-by: Bin Huang --- arch/arm64/kernel/fpsimd.c | 44 +++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c index c064d2322d99..cf65b4664fc3 100644 --- a/arch/arm64/kernel/fpsimd.c +++ b/arch/arm64/kernel/fpsimd.c @@ -1578,6 +1578,27 @@ static void fpsimd_save_kernel_state(struct task_struct *task) fpsimd_save_state(&task->thread.kernel_fpsimd_state); } +/* + * Invalidate any task's FPSIMD state that is present on this cpu. + * The FPSIMD context should be acquired with get_cpu_fpsimd_context() + * before calling this function. + */ +static void fpsimd_flush_cpu_state(void) +{ + WARN_ON(!system_supports_fpsimd()); + __this_cpu_write(fpsimd_last_state.st, NULL); + + /* + * Leaving streaming mode enabled will cause issues for any kernel + * NEON and leaving streaming mode or ZA enabled may increase power + * consumption. + */ + if (system_supports_sme()) + sme_smstop(); + + set_thread_flag(TIF_FOREIGN_FPSTATE); +} + void fpsimd_thread_switch(struct task_struct *next) { bool wrong_task, wrong_cpu; @@ -1595,7 +1616,7 @@ void fpsimd_thread_switch(struct task_struct *next) if (test_tsk_thread_flag(next, TIF_KERNEL_FPSTATE)) { fpsimd_load_kernel_state(next); - set_tsk_thread_flag(next, TIF_FOREIGN_FPSTATE); + fpsimd_flush_cpu_state(); } else { /* * Fix up TIF_FOREIGN_FPSTATE to correctly describe next's @@ -1846,27 +1867,6 @@ void fpsimd_save_and_flush_current_state(void) put_cpu_fpsimd_context(); } -/* - * Invalidate any task's FPSIMD state that is present on this cpu. - * The FPSIMD context should be acquired with get_cpu_fpsimd_context() - * before calling this function. - */ -static void fpsimd_flush_cpu_state(void) -{ - WARN_ON(!system_supports_fpsimd()); - __this_cpu_write(fpsimd_last_state.st, NULL); - - /* - * Leaving streaming mode enabled will cause issues for any kernel - * NEON and leaving streaming mode or ZA enabled may increase power - * consumption. - */ - if (system_supports_sme()) - sme_smstop(); - - set_thread_flag(TIF_FOREIGN_FPSTATE); -} - /* * Save the FPSIMD state to memory and invalidate cpu view. * This function must be called with preemption disabled. From dfadad3cde6dba0437c66d931572ac92a60447c2 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 18 Oct 2024 16:53:43 -0700 Subject: [PATCH 33/85] crypto: api - move crypto_simd_disabled_for_test to lib commit 4964a1d91cd186b423666aac6d4ad3a61cf88b54 upstream. Move crypto_simd_disabled_for_test to lib/ so that crypto_simd_usable() can be used by library code. This was discussed previously (https://lore.kernel.org/linux-crypto/20220716062920.210381-4-ebiggers@kernel.org/) but was not done because there was no use case yet. However, this is now needed for the arm64 CRC32 library code. Tested with: export ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- echo CONFIG_CRC32=y > .config echo CONFIG_MODULES=y >> .config echo CONFIG_CRYPTO=m >> .config echo CONFIG_DEBUG_KERNEL=y >> .config echo CONFIG_CRYPTO_MANAGER_DISABLE_TESTS=n >> .config echo CONFIG_CRYPTO_MANAGER_EXTRA_TESTS=y >> .config make olddefconfig make -j$(nproc) Signed-off-by: Eric Biggers Acked-by: Ard Biesheuvel Signed-off-by: Herbert Xu Signed-off-by: Bin Huang --- crypto/algapi.c | 6 ------ lib/crypto/Makefile | 2 ++ lib/crypto/simd.c | 11 +++++++++++ 3 files changed, 13 insertions(+), 6 deletions(-) create mode 100644 lib/crypto/simd.c diff --git a/crypto/algapi.c b/crypto/algapi.c index b3a608604253..57e782ae1371 100644 --- a/crypto/algapi.c +++ b/crypto/algapi.c @@ -6,7 +6,6 @@ */ #include -#include #include #include #include @@ -23,11 +22,6 @@ static LIST_HEAD(crypto_template_list); -#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS -DEFINE_PER_CPU(bool, crypto_simd_disabled_for_test); -EXPORT_PER_CPU_SYMBOL_GPL(crypto_simd_disabled_for_test); -#endif - static inline void crypto_check_module_sig(struct module *mod) { if (fips_enabled && mod && !module_sig_ok(mod)) diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index b1b81998e101..c6497faf2ea2 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -59,3 +59,5 @@ libcurve25519-y += curve25519-selftest.o endif obj-$(CONFIG_MPILIB) += mpi/ + +obj-$(CONFIG_CRYPTO_MANAGER_EXTRA_TESTS) += simd.o diff --git a/lib/crypto/simd.c b/lib/crypto/simd.c new file mode 100644 index 000000000000..9c36cb3bb49c --- /dev/null +++ b/lib/crypto/simd.c @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * SIMD testing utility functions + * + * Copyright 2024 Google LLC + */ + +#include + +DEFINE_PER_CPU(bool, crypto_simd_disabled_for_test); +EXPORT_PER_CPU_SYMBOL_GPL(crypto_simd_disabled_for_test); From aca0ac603b7ad467a5c8da4dd725aadcbe5d7201 Mon Sep 17 00:00:00 2001 From: Xiao Wang Date: Fri, 21 Jun 2024 13:47:07 +0800 Subject: [PATCH 34/85] riscv: Optimize crc32 with Zbc extension commit a43fe27d650375cd9e5ea915c538f6f9eabd185e upstream. As suggested by the B-ext spec, the Zbc (carry-less multiplication) instructions can be used to accelerate CRC calculations. Currently, the crc32 is the most widely used crc function inside kernel, so this patch focuses on the optimization of just the crc32 APIs. Compared with the current table-lookup based optimization, Zbc based optimization can also achieve large stride during CRC calculation loop, meantime, it avoids the memory access latency of the table-lookup based implementation and it reduces memory footprint. If Zbc feature is not supported in a runtime environment, then the table-lookup based implementation would serve as fallback via alternative mechanism. By inspecting the vmlinux built by gcc v12.2.0 with default optimization level (-O2), we can see below instruction count change for each 8-byte stride in the CRC32 loop: rv64: crc32_be (54->31), crc32_le (54->13), __crc32c_le (54->13) rv32: crc32_be (50->32), crc32_le (50->16), __crc32c_le (50->16) The compile target CPU is little endian, extra effort is needed for byte swapping for the crc32_be API, thus, the instruction count change is not as significant as that in the *_le cases. This patch is tested on QEMU VM with the kernel CRC32 selftest for both rv64 and rv32. Running the CRC32 selftest on a real hardware (SpacemiT K1) with Zbc extension shows 65% and 125% performance improvement respectively on crc32_test() and crc32c_test(). Signed-off-by: Xiao Wang Reviewed-by: Charlie Jenkins Link: https://lore.kernel.org/r/20240621054707.1847548-1-xiao.w.wang@intel.com Signed-off-by: Palmer Dabbelt Signed-off-by: Bin Huang --- arch/riscv/Kconfig | 23 ++++ arch/riscv/lib/Makefile | 1 + arch/riscv/lib/crc32.c | 294 ++++++++++++++++++++++++++++++++++++++++ include/linux/crc32.h | 3 + 4 files changed, 321 insertions(+) create mode 100644 arch/riscv/lib/crc32.c diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 1304992232ad..7137722167cd 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -547,6 +547,29 @@ config RISCV_ISA_ZBB If you don't know what to do here, say Y. +config TOOLCHAIN_HAS_ZBC + bool + default y + depends on !64BIT || $(cc-option,-mabi=lp64 -march=rv64ima_zbc) + depends on !32BIT || $(cc-option,-mabi=ilp32 -march=rv32ima_zbc) + depends on LLD_VERSION >= 150000 || LD_VERSION >= 23900 + depends on AS_HAS_OPTION_ARCH + +config RISCV_ISA_ZBC + bool "Zbc extension support for carry-less multiplication instructions" + depends on TOOLCHAIN_HAS_ZBC + depends on MMU + depends on RISCV_ALTERNATIVE + default y + help + Adds support to dynamically detect the presence of the Zbc + extension (carry-less multiplication) and enable its usage. + + The Zbc extension could accelerate CRC (cyclic redundancy check) + calculations. + + If you don't know what to do here, say Y. + config RISCV_ISA_ZICBOM bool "Zicbom extension support for non-coherent DMA operation" depends on MMU diff --git a/arch/riscv/lib/Makefile b/arch/riscv/lib/Makefile index 26cb2502ecf8..183bf2097d57 100644 --- a/arch/riscv/lib/Makefile +++ b/arch/riscv/lib/Makefile @@ -9,5 +9,6 @@ lib-y += strncmp.o lib-$(CONFIG_MMU) += uaccess.o lib-$(CONFIG_64BIT) += tishift.o lib-$(CONFIG_RISCV_ISA_ZICBOZ) += clear_page.o +lib-$(CONFIG_RISCV_ISA_ZBC) += crc32.o obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o diff --git a/arch/riscv/lib/crc32.c b/arch/riscv/lib/crc32.c new file mode 100644 index 000000000000..d7dc599af3ef --- /dev/null +++ b/arch/riscv/lib/crc32.c @@ -0,0 +1,294 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Accelerated CRC32 implementation with Zbc extension. + * + * Copyright (C) 2024 Intel Corporation + */ + +#include +#include +#include + +#include +#include +#include +#include +#include + +/* + * Refer to https://www.corsix.org/content/barrett-reduction-polynomials for + * better understanding of how this math works. + * + * let "+" denotes polynomial add (XOR) + * let "-" denotes polynomial sub (XOR) + * let "*" denotes polynomial multiplication + * let "/" denotes polynomial floor division + * let "S" denotes source data, XLEN bit wide + * let "P" denotes CRC32 polynomial + * let "T" denotes 2^(XLEN+32) + * let "QT" denotes quotient of T/P, with the bit for 2^XLEN being implicit + * + * crc32(S, P) + * => S * (2^32) - S * (2^32) / P * P + * => lowest 32 bits of: S * (2^32) / P * P + * => lowest 32 bits of: S * (2^32) * (T / P) / T * P + * => lowest 32 bits of: S * (2^32) * quotient / T * P + * => lowest 32 bits of: S * quotient / 2^XLEN * P + * => lowest 32 bits of: (clmul_high_part(S, QT) + S) * P + * => clmul_low_part(clmul_high_part(S, QT) + S, P) + * + * In terms of below implementations, the BE case is more intuitive, since the + * higher order bit sits at more significant position. + */ + +#if __riscv_xlen == 64 +/* Slide by XLEN bits per iteration */ +# define STEP_ORDER 3 + +/* Each below polynomial quotient has an implicit bit for 2^XLEN */ + +/* Polynomial quotient of (2^(XLEN+32))/CRC32_POLY, in LE format */ +# define CRC32_POLY_QT_LE 0x5a72d812fb808b20 + +/* Polynomial quotient of (2^(XLEN+32))/CRC32C_POLY, in LE format */ +# define CRC32C_POLY_QT_LE 0xa434f61c6f5389f8 + +/* Polynomial quotient of (2^(XLEN+32))/CRC32_POLY, in BE format, it should be + * the same as the bit-reversed version of CRC32_POLY_QT_LE + */ +# define CRC32_POLY_QT_BE 0x04d101df481b4e5a + +static inline u64 crc32_le_prep(u32 crc, unsigned long const *ptr) +{ + return (u64)crc ^ (__force u64)__cpu_to_le64(*ptr); +} + +static inline u32 crc32_le_zbc(unsigned long s, u32 poly, unsigned long poly_qt) +{ + u32 crc; + + /* We don't have a "clmulrh" insn, so use clmul + slli instead. */ + asm volatile (".option push\n" + ".option arch,+zbc\n" + "clmul %0, %1, %2\n" + "slli %0, %0, 1\n" + "xor %0, %0, %1\n" + "clmulr %0, %0, %3\n" + "srli %0, %0, 32\n" + ".option pop\n" + : "=&r" (crc) + : "r" (s), + "r" (poly_qt), + "r" ((u64)poly << 32) + :); + return crc; +} + +static inline u64 crc32_be_prep(u32 crc, unsigned long const *ptr) +{ + return ((u64)crc << 32) ^ (__force u64)__cpu_to_be64(*ptr); +} + +#elif __riscv_xlen == 32 +# define STEP_ORDER 2 +/* Each quotient should match the upper half of its analog in RV64 */ +# define CRC32_POLY_QT_LE 0xfb808b20 +# define CRC32C_POLY_QT_LE 0x6f5389f8 +# define CRC32_POLY_QT_BE 0x04d101df + +static inline u32 crc32_le_prep(u32 crc, unsigned long const *ptr) +{ + return crc ^ (__force u32)__cpu_to_le32(*ptr); +} + +static inline u32 crc32_le_zbc(unsigned long s, u32 poly, unsigned long poly_qt) +{ + u32 crc; + + /* We don't have a "clmulrh" insn, so use clmul + slli instead. */ + asm volatile (".option push\n" + ".option arch,+zbc\n" + "clmul %0, %1, %2\n" + "slli %0, %0, 1\n" + "xor %0, %0, %1\n" + "clmulr %0, %0, %3\n" + ".option pop\n" + : "=&r" (crc) + : "r" (s), + "r" (poly_qt), + "r" (poly) + :); + return crc; +} + +static inline u32 crc32_be_prep(u32 crc, unsigned long const *ptr) +{ + return crc ^ (__force u32)__cpu_to_be32(*ptr); +} + +#else +# error "Unexpected __riscv_xlen" +#endif + +static inline u32 crc32_be_zbc(unsigned long s) +{ + u32 crc; + + asm volatile (".option push\n" + ".option arch,+zbc\n" + "clmulh %0, %1, %2\n" + "xor %0, %0, %1\n" + "clmul %0, %0, %3\n" + ".option pop\n" + : "=&r" (crc) + : "r" (s), + "r" (CRC32_POLY_QT_BE), + "r" (CRC32_POLY_BE) + :); + return crc; +} + +#define STEP (1 << STEP_ORDER) +#define OFFSET_MASK (STEP - 1) + +typedef u32 (*fallback)(u32 crc, unsigned char const *p, size_t len); + +static inline u32 crc32_le_unaligned(u32 crc, unsigned char const *p, + size_t len, u32 poly, + unsigned long poly_qt) +{ + size_t bits = len * 8; + unsigned long s = 0; + u32 crc_low = 0; + + for (int i = 0; i < len; i++) + s = ((unsigned long)*p++ << (__riscv_xlen - 8)) | (s >> 8); + + s ^= (unsigned long)crc << (__riscv_xlen - bits); + if (__riscv_xlen == 32 || len < sizeof(u32)) + crc_low = crc >> bits; + + crc = crc32_le_zbc(s, poly, poly_qt); + crc ^= crc_low; + + return crc; +} + +static inline u32 __pure crc32_le_generic(u32 crc, unsigned char const *p, + size_t len, u32 poly, + unsigned long poly_qt, + fallback crc_fb) +{ + size_t offset, head_len, tail_len; + unsigned long const *p_ul; + unsigned long s; + + asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0, + RISCV_ISA_EXT_ZBC, 1) + : : : : legacy); + + /* Handle the unaligned head. */ + offset = (unsigned long)p & OFFSET_MASK; + if (offset && len) { + head_len = min(STEP - offset, len); + crc = crc32_le_unaligned(crc, p, head_len, poly, poly_qt); + p += head_len; + len -= head_len; + } + + tail_len = len & OFFSET_MASK; + len = len >> STEP_ORDER; + p_ul = (unsigned long const *)p; + + for (int i = 0; i < len; i++) { + s = crc32_le_prep(crc, p_ul); + crc = crc32_le_zbc(s, poly, poly_qt); + p_ul++; + } + + /* Handle the tail bytes. */ + p = (unsigned char const *)p_ul; + if (tail_len) + crc = crc32_le_unaligned(crc, p, tail_len, poly, poly_qt); + + return crc; + +legacy: + return crc_fb(crc, p, len); +} + +u32 __pure crc32_le(u32 crc, unsigned char const *p, size_t len) +{ + return crc32_le_generic(crc, p, len, CRC32_POLY_LE, CRC32_POLY_QT_LE, + crc32_le_base); +} + +u32 __pure __crc32c_le(u32 crc, unsigned char const *p, size_t len) +{ + return crc32_le_generic(crc, p, len, CRC32C_POLY_LE, + CRC32C_POLY_QT_LE, __crc32c_le_base); +} + +static inline u32 crc32_be_unaligned(u32 crc, unsigned char const *p, + size_t len) +{ + size_t bits = len * 8; + unsigned long s = 0; + u32 crc_low = 0; + + s = 0; + for (int i = 0; i < len; i++) + s = *p++ | (s << 8); + + if (__riscv_xlen == 32 || len < sizeof(u32)) { + s ^= crc >> (32 - bits); + crc_low = crc << bits; + } else { + s ^= (unsigned long)crc << (bits - 32); + } + + crc = crc32_be_zbc(s); + crc ^= crc_low; + + return crc; +} + +u32 __pure crc32_be(u32 crc, unsigned char const *p, size_t len) +{ + size_t offset, head_len, tail_len; + unsigned long const *p_ul; + unsigned long s; + + asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0, + RISCV_ISA_EXT_ZBC, 1) + : : : : legacy); + + /* Handle the unaligned head. */ + offset = (unsigned long)p & OFFSET_MASK; + if (offset && len) { + head_len = min(STEP - offset, len); + crc = crc32_be_unaligned(crc, p, head_len); + p += head_len; + len -= head_len; + } + + tail_len = len & OFFSET_MASK; + len = len >> STEP_ORDER; + p_ul = (unsigned long const *)p; + + for (int i = 0; i < len; i++) { + s = crc32_be_prep(crc, p_ul); + crc = crc32_be_zbc(s); + p_ul++; + } + + /* Handle the tail bytes. */ + p = (unsigned char const *)p_ul; + if (tail_len) + crc = crc32_be_unaligned(crc, p, tail_len); + + return crc; + +legacy: + return crc32_be_base(crc, p, len); +} diff --git a/include/linux/crc32.h b/include/linux/crc32.h index 9e8a032c1788..87f788c0d607 100644 --- a/include/linux/crc32.h +++ b/include/linux/crc32.h @@ -9,7 +9,9 @@ #include u32 __pure crc32_le(u32 crc, unsigned char const *p, size_t len); +u32 __pure crc32_le_base(u32 crc, unsigned char const *p, size_t len); u32 __pure crc32_be(u32 crc, unsigned char const *p, size_t len); +u32 __pure crc32_be_base(u32 crc, unsigned char const *p, size_t len); /** * crc32_le_combine - Combine two crc32 check values into one. For two @@ -37,6 +39,7 @@ static inline u32 crc32_le_combine(u32 crc1, u32 crc2, size_t len2) } u32 __pure __crc32c_le(u32 crc, unsigned char const *p, size_t len); +u32 __pure __crc32c_le_base(u32 crc, unsigned char const *p, size_t len); /** * __crc32c_le_combine - Combine two crc32c check values into one. For two From cdef7da233e12ac847ddb68bfffc82b29506741c Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 18 Oct 2024 09:53:49 +0200 Subject: [PATCH 35/85] arm64/lib: Handle CRC-32 alternative in C code commit fc7454107d1b7c27bb98d3b109e5f44a8a46d7f8 upstream. In preparation for adding another code path for performing CRC-32, move the alternative patching for ARM64_HAS_CRC32 into C code. The logic for deciding whether to use this new code path will be implemented in C too. Reviewed-by: Eric Biggers Signed-off-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20241018075347.2821102-6-ardb+git@google.com Signed-off-by: Catalin Marinas Signed-off-by: Bin Huang --- arch/arm64/lib/Makefile | 2 +- arch/arm64/lib/crc32-glue.c | 34 ++++++++++++++++++++++++++++++++++ arch/arm64/lib/crc32.S | 22 ++++++---------------- 3 files changed, 41 insertions(+), 17 deletions(-) create mode 100644 arch/arm64/lib/crc32-glue.c diff --git a/arch/arm64/lib/Makefile b/arch/arm64/lib/Makefile index 29490be2546b..c03e3b8d1cbe 100644 --- a/arch/arm64/lib/Makefile +++ b/arch/arm64/lib/Makefile @@ -15,7 +15,7 @@ endif lib-$(CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE) += uaccess_flushcache.o -obj-$(CONFIG_CRC32) += crc32.o +obj-$(CONFIG_CRC32) += crc32.o crc32-glue.o obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o diff --git a/arch/arm64/lib/crc32-glue.c b/arch/arm64/lib/crc32-glue.c new file mode 100644 index 000000000000..0b51761d4b75 --- /dev/null +++ b/arch/arm64/lib/crc32-glue.c @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include + +#include + +asmlinkage u32 crc32_le_arm64(u32 crc, unsigned char const *p, size_t len); +asmlinkage u32 crc32c_le_arm64(u32 crc, unsigned char const *p, size_t len); +asmlinkage u32 crc32_be_arm64(u32 crc, unsigned char const *p, size_t len); + +u32 __pure crc32_le(u32 crc, unsigned char const *p, size_t len) +{ + if (!alternative_has_cap_likely(ARM64_HAS_CRC32)) + return crc32_le_base(crc, p, len); + + return crc32_le_arm64(crc, p, len); +} + +u32 __pure __crc32c_le(u32 crc, unsigned char const *p, size_t len) +{ + if (!alternative_has_cap_likely(ARM64_HAS_CRC32)) + return __crc32c_le_base(crc, p, len); + + return crc32c_le_arm64(crc, p, len); +} + +u32 __pure crc32_be(u32 crc, unsigned char const *p, size_t len) +{ + if (!alternative_has_cap_likely(ARM64_HAS_CRC32)) + return crc32_be_base(crc, p, len); + + return crc32_be_arm64(crc, p, len); +} diff --git a/arch/arm64/lib/crc32.S b/arch/arm64/lib/crc32.S index 8340dccff46f..22139691c7ae 100644 --- a/arch/arm64/lib/crc32.S +++ b/arch/arm64/lib/crc32.S @@ -6,7 +6,6 @@ */ #include -#include #include .arch armv8-a+crc @@ -136,25 +135,16 @@ CPU_BE( rev16 \reg, \reg ) .endm .align 5 -SYM_FUNC_START(crc32_le) -alternative_if_not ARM64_HAS_CRC32 - b crc32_le_base -alternative_else_nop_endif +SYM_FUNC_START(crc32_le_arm64) __crc32 -SYM_FUNC_END(crc32_le) +SYM_FUNC_END(crc32_le_arm64) .align 5 -SYM_FUNC_START(__crc32c_le) -alternative_if_not ARM64_HAS_CRC32 - b __crc32c_le_base -alternative_else_nop_endif +SYM_FUNC_START(crc32c_le_arm64) __crc32 c -SYM_FUNC_END(__crc32c_le) +SYM_FUNC_END(crc32c_le_arm64) .align 5 -SYM_FUNC_START(crc32_be) -alternative_if_not ARM64_HAS_CRC32 - b crc32_be_base -alternative_else_nop_endif +SYM_FUNC_START(crc32_be_arm64) __crc32 be=1 -SYM_FUNC_END(crc32_be) +SYM_FUNC_END(crc32_be_arm64) From 782741e6c93fedac1cc2eff75ab669149d253da1 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 18 Oct 2024 09:53:50 +0200 Subject: [PATCH 36/85] arm64/crc32: Reorganize bit/byte ordering macros commit b98b23e19492f4009070761c53b755f623f60e49 upstream. In preparation for a new user, reorganize the bit/byte ordering macros that are used to parameterize the crc32 template code and instantiate CRC-32, CRC-32c and 'big endian' CRC-32. Signed-off-by: Ard Biesheuvel Reviewed-by: Eric Biggers Link: https://lore.kernel.org/r/20241018075347.2821102-7-ardb+git@google.com Signed-off-by: Catalin Marinas Signed-off-by: Bin Huang --- arch/arm64/lib/crc32.S | 91 ++++++++++++++++++------------------------ 1 file changed, 39 insertions(+), 52 deletions(-) diff --git a/arch/arm64/lib/crc32.S b/arch/arm64/lib/crc32.S index 22139691c7ae..f9920492f135 100644 --- a/arch/arm64/lib/crc32.S +++ b/arch/arm64/lib/crc32.S @@ -10,44 +10,48 @@ .arch armv8-a+crc - .macro byteorder, reg, be - .if \be -CPU_LE( rev \reg, \reg ) - .else -CPU_BE( rev \reg, \reg ) - .endif + .macro bitle, reg .endm - .macro byteorder16, reg, be - .if \be -CPU_LE( rev16 \reg, \reg ) - .else -CPU_BE( rev16 \reg, \reg ) - .endif + .macro bitbe, reg + rbit \reg, \reg .endm - .macro bitorder, reg, be - .if \be - rbit \reg, \reg - .endif + .macro bytele, reg .endm - .macro bitorder16, reg, be - .if \be + .macro bytebe, reg rbit \reg, \reg - lsr \reg, \reg, #16 - .endif + lsr \reg, \reg, #24 + .endm + + .macro hwordle, reg +CPU_BE( rev16 \reg, \reg ) .endm - .macro bitorder8, reg, be - .if \be + .macro hwordbe, reg +CPU_LE( rev \reg, \reg ) rbit \reg, \reg - lsr \reg, \reg, #24 - .endif +CPU_BE( lsr \reg, \reg, #16 ) + .endm + + .macro le, regs:vararg + .irp r, \regs +CPU_BE( rev \r, \r ) + .endr + .endm + + .macro be, regs:vararg + .irp r, \regs +CPU_LE( rev \r, \r ) + .endr + .irp r, \regs + rbit \r, \r + .endr .endm - .macro __crc32, c, be=0 - bitorder w0, \be + .macro __crc32, c, order=le + bit\order w0 cmp x2, #16 b.lt 8f // less than 16 bytes @@ -60,14 +64,7 @@ CPU_BE( rev16 \reg, \reg ) add x8, x8, x1 add x1, x1, x7 ldp x5, x6, [x8] - byteorder x3, \be - byteorder x4, \be - byteorder x5, \be - byteorder x6, \be - bitorder x3, \be - bitorder x4, \be - bitorder x5, \be - bitorder x6, \be + \order x3, x4, x5, x6 tst x7, #8 crc32\c\()x w8, w0, x3 @@ -95,42 +92,32 @@ CPU_BE( rev16 \reg, \reg ) 32: ldp x3, x4, [x1], #32 sub x2, x2, #32 ldp x5, x6, [x1, #-16] - byteorder x3, \be - byteorder x4, \be - byteorder x5, \be - byteorder x6, \be - bitorder x3, \be - bitorder x4, \be - bitorder x5, \be - bitorder x6, \be + \order x3, x4, x5, x6 crc32\c\()x w0, w0, x3 crc32\c\()x w0, w0, x4 crc32\c\()x w0, w0, x5 crc32\c\()x w0, w0, x6 cbnz x2, 32b -0: bitorder w0, \be +0: bit\order w0 ret 8: tbz x2, #3, 4f ldr x3, [x1], #8 - byteorder x3, \be - bitorder x3, \be + \order x3 crc32\c\()x w0, w0, x3 4: tbz x2, #2, 2f ldr w3, [x1], #4 - byteorder w3, \be - bitorder w3, \be + \order w3 crc32\c\()w w0, w0, w3 2: tbz x2, #1, 1f ldrh w3, [x1], #2 - byteorder16 w3, \be - bitorder16 w3, \be + hword\order w3 crc32\c\()h w0, w0, w3 1: tbz x2, #0, 0f ldrb w3, [x1] - bitorder8 w3, \be + byte\order w3 crc32\c\()b w0, w0, w3 -0: bitorder w0, \be +0: bit\order w0 ret .endm @@ -146,5 +133,5 @@ SYM_FUNC_END(crc32c_le_arm64) .align 5 SYM_FUNC_START(crc32_be_arm64) - __crc32 be=1 + __crc32 order=be SYM_FUNC_END(crc32_be_arm64) From 907a1e0e1defc576a2d223555ff5ff8acc69a156 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 18 Oct 2024 09:53:51 +0200 Subject: [PATCH 37/85] arm64/crc32: Implement 4-way interleave using PMULL commit a6478d69cf56d5deb4c28a6486376d9c7895abec upstream. Now that kernel mode NEON no longer disables preemption, using FP/SIMD in library code which is not obviously part of the crypto subsystem is no longer problematic, as it will no longer incur unexpected latencies. So accelerate the CRC-32 library code on arm64 to use a 4-way interleave, using PMULL instructions to implement the folding. On Apple M2, this results in a speedup of 2 - 2.8x when using input sizes of 1k - 8k. For smaller sizes, the overhead of preserving and restoring the FP/SIMD register file may not be worth it, so 1k is used as a threshold for choosing this code path. The coefficient tables were generated using code provided by Eric. [0] [0] https://github.com/ebiggers/libdeflate/blob/master/scripts/gen_crc32_multipliers.c Cc: Eric Biggers Signed-off-by: Ard Biesheuvel Reviewed-by: Eric Biggers Link: https://lore.kernel.org/r/20241018075347.2821102-8-ardb+git@google.com Signed-off-by: Catalin Marinas Signed-off-by: Bin Huang --- arch/arm64/lib/crc32-glue.c | 48 ++++++++ arch/arm64/lib/crc32.S | 231 +++++++++++++++++++++++++++++++++++- 2 files changed, 276 insertions(+), 3 deletions(-) diff --git a/arch/arm64/lib/crc32-glue.c b/arch/arm64/lib/crc32-glue.c index 0b51761d4b75..295ae3e6b997 100644 --- a/arch/arm64/lib/crc32-glue.c +++ b/arch/arm64/lib/crc32-glue.c @@ -4,16 +4,40 @@ #include #include +#include +#include +#include + +#include + +// The minimum input length to consider the 4-way interleaved code path +static const size_t min_len = 1024; asmlinkage u32 crc32_le_arm64(u32 crc, unsigned char const *p, size_t len); asmlinkage u32 crc32c_le_arm64(u32 crc, unsigned char const *p, size_t len); asmlinkage u32 crc32_be_arm64(u32 crc, unsigned char const *p, size_t len); +asmlinkage u32 crc32_le_arm64_4way(u32 crc, unsigned char const *p, size_t len); +asmlinkage u32 crc32c_le_arm64_4way(u32 crc, unsigned char const *p, size_t len); +asmlinkage u32 crc32_be_arm64_4way(u32 crc, unsigned char const *p, size_t len); + u32 __pure crc32_le(u32 crc, unsigned char const *p, size_t len) { if (!alternative_has_cap_likely(ARM64_HAS_CRC32)) return crc32_le_base(crc, p, len); + if (len >= min_len && cpu_have_named_feature(PMULL) && crypto_simd_usable()) { + kernel_neon_begin(); + crc = crc32_le_arm64_4way(crc, p, len); + kernel_neon_end(); + + p += round_down(len, 64); + len %= 64; + + if (!len) + return crc; + } + return crc32_le_arm64(crc, p, len); } @@ -22,6 +46,18 @@ u32 __pure __crc32c_le(u32 crc, unsigned char const *p, size_t len) if (!alternative_has_cap_likely(ARM64_HAS_CRC32)) return __crc32c_le_base(crc, p, len); + if (len >= min_len && cpu_have_named_feature(PMULL) && crypto_simd_usable()) { + kernel_neon_begin(); + crc = crc32c_le_arm64_4way(crc, p, len); + kernel_neon_end(); + + p += round_down(len, 64); + len %= 64; + + if (!len) + return crc; + } + return crc32c_le_arm64(crc, p, len); } @@ -30,5 +66,17 @@ u32 __pure crc32_be(u32 crc, unsigned char const *p, size_t len) if (!alternative_has_cap_likely(ARM64_HAS_CRC32)) return crc32_be_base(crc, p, len); + if (len >= min_len && cpu_have_named_feature(PMULL) && crypto_simd_usable()) { + kernel_neon_begin(); + crc = crc32_be_arm64_4way(crc, p, len); + kernel_neon_end(); + + p += round_down(len, 64); + len %= 64; + + if (!len) + return crc; + } + return crc32_be_arm64(crc, p, len); } diff --git a/arch/arm64/lib/crc32.S b/arch/arm64/lib/crc32.S index f9920492f135..68825317460f 100644 --- a/arch/arm64/lib/crc32.S +++ b/arch/arm64/lib/crc32.S @@ -1,14 +1,17 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * Accelerated CRC32(C) using AArch64 CRC instructions + * Accelerated CRC32(C) using AArch64 CRC and PMULL instructions * - * Copyright (C) 2016 - 2018 Linaro Ltd + * Copyright (C) 2016 - 2018 Linaro Ltd. + * Copyright (C) 2024 Google LLC + * + * Author: Ard Biesheuvel */ #include #include - .arch armv8-a+crc + .cpu generic+crc+crypto .macro bitle, reg .endm @@ -135,3 +138,225 @@ SYM_FUNC_END(crc32c_le_arm64) SYM_FUNC_START(crc32_be_arm64) __crc32 order=be SYM_FUNC_END(crc32_be_arm64) + + in .req x1 + len .req x2 + + /* + * w0: input CRC at entry, output CRC at exit + * x1: pointer to input buffer + * x2: length of input in bytes + */ + .macro crc4way, insn, table, order=le + bit\order w0 + lsr len, len, #6 // len := # of 64-byte blocks + + /* Process up to 64 blocks of 64 bytes at a time */ +.La\@: mov x3, #64 + cmp len, #64 + csel x3, x3, len, hi // x3 := min(len, 64) + sub len, len, x3 + + /* Divide the input into 4 contiguous blocks */ + add x4, x3, x3, lsl #1 // x4 := 3 * x3 + add x7, in, x3, lsl #4 // x7 := in + 16 * x3 + add x8, in, x3, lsl #5 // x8 := in + 32 * x3 + add x9, in, x4, lsl #4 // x9 := in + 16 * x4 + + /* Load the folding coefficients from the lookup table */ + adr_l x5, \table - 12 // entry 0 omitted + add x5, x5, x4, lsl #2 // x5 += 12 * x3 + ldp s0, s1, [x5] + ldr s2, [x5, #8] + + /* Zero init partial CRCs for this iteration */ + mov w4, wzr + mov w5, wzr + mov w6, wzr + mov x17, xzr + +.Lb\@: sub x3, x3, #1 + \insn w6, w6, x17 + ldp x10, x11, [in], #16 + ldp x12, x13, [x7], #16 + ldp x14, x15, [x8], #16 + ldp x16, x17, [x9], #16 + + \order x10, x11, x12, x13, x14, x15, x16, x17 + + /* Apply the CRC transform to 4 16-byte blocks in parallel */ + \insn w0, w0, x10 + \insn w4, w4, x12 + \insn w5, w5, x14 + \insn w6, w6, x16 + \insn w0, w0, x11 + \insn w4, w4, x13 + \insn w5, w5, x15 + cbnz x3, .Lb\@ + + /* Combine the 4 partial results into w0 */ + mov v3.d[0], x0 + mov v4.d[0], x4 + mov v5.d[0], x5 + pmull v0.1q, v0.1d, v3.1d + pmull v1.1q, v1.1d, v4.1d + pmull v2.1q, v2.1d, v5.1d + eor v0.8b, v0.8b, v1.8b + eor v0.8b, v0.8b, v2.8b + mov x5, v0.d[0] + eor x5, x5, x17 + \insn w0, w6, x5 + + mov in, x9 + cbnz len, .La\@ + + bit\order w0 + ret + .endm + + .align 5 +SYM_FUNC_START(crc32c_le_arm64_4way) + crc4way crc32cx, .L0 +SYM_FUNC_END(crc32c_le_arm64_4way) + + .align 5 +SYM_FUNC_START(crc32_le_arm64_4way) + crc4way crc32x, .L1 +SYM_FUNC_END(crc32_le_arm64_4way) + + .align 5 +SYM_FUNC_START(crc32_be_arm64_4way) + crc4way crc32x, .L1, be +SYM_FUNC_END(crc32_be_arm64_4way) + + .section .rodata, "a", %progbits + .align 6 +.L0: .long 0xddc0152b, 0xba4fc28e, 0x493c7d27 + .long 0x0715ce53, 0x9e4addf8, 0xba4fc28e + .long 0xc96cfdc0, 0x0715ce53, 0xddc0152b + .long 0xab7aff2a, 0x0d3b6092, 0x9e4addf8 + .long 0x299847d5, 0x878a92a7, 0x39d3b296 + .long 0xb6dd949b, 0xab7aff2a, 0x0715ce53 + .long 0xa60ce07b, 0x83348832, 0x47db8317 + .long 0xd270f1a2, 0xb9e02b86, 0x0d3b6092 + .long 0x65863b64, 0xb6dd949b, 0xc96cfdc0 + .long 0xb3e32c28, 0xbac2fd7b, 0x878a92a7 + .long 0xf285651c, 0xce7f39f4, 0xdaece73e + .long 0x271d9844, 0xd270f1a2, 0xab7aff2a + .long 0x6cb08e5c, 0x2b3cac5d, 0x2162d385 + .long 0xcec3662e, 0x1b03397f, 0x83348832 + .long 0x8227bb8a, 0xb3e32c28, 0x299847d5 + .long 0xd7a4825c, 0xdd7e3b0c, 0xb9e02b86 + .long 0xf6076544, 0x10746f3c, 0x18b33a4e + .long 0x98d8d9cb, 0x271d9844, 0xb6dd949b + .long 0x57a3d037, 0x93a5f730, 0x78d9ccb7 + .long 0x3771e98f, 0x6b749fb2, 0xbac2fd7b + .long 0xe0ac139e, 0xcec3662e, 0xa60ce07b + .long 0x6f345e45, 0xe6fc4e6a, 0xce7f39f4 + .long 0xa2b73df1, 0xb0cd4768, 0x61d82e56 + .long 0x86d8e4d2, 0xd7a4825c, 0xd270f1a2 + .long 0xa90fd27a, 0x0167d312, 0xc619809d + .long 0xca6ef3ac, 0x26f6a60a, 0x2b3cac5d + .long 0x4597456a, 0x98d8d9cb, 0x65863b64 + .long 0xc9c8b782, 0x68bce87a, 0x1b03397f + .long 0x62ec6c6d, 0x6956fc3b, 0xebb883bd + .long 0x2342001e, 0x3771e98f, 0xb3e32c28 + .long 0xe8b6368b, 0x2178513a, 0x064f7f26 + .long 0x9ef68d35, 0x170076fa, 0xdd7e3b0c + .long 0x0b0bf8ca, 0x6f345e45, 0xf285651c + .long 0x02ee03b2, 0xff0dba97, 0x10746f3c + .long 0x135c83fd, 0xf872e54c, 0xc7a68855 + .long 0x00bcf5f6, 0x86d8e4d2, 0x271d9844 + .long 0x58ca5f00, 0x5bb8f1bc, 0x8e766a0c + .long 0xded288f8, 0xb3af077a, 0x93a5f730 + .long 0x37170390, 0xca6ef3ac, 0x6cb08e5c + .long 0xf48642e9, 0xdd66cbbb, 0x6b749fb2 + .long 0xb25b29f2, 0xe9e28eb4, 0x1393e203 + .long 0x45cddf4e, 0xc9c8b782, 0xcec3662e + .long 0xdfd94fb2, 0x93e106a4, 0x96c515bb + .long 0x021ac5ef, 0xd813b325, 0xe6fc4e6a + .long 0x8e1450f7, 0x2342001e, 0x8227bb8a + .long 0xe0cdcf86, 0x6d9a4957, 0xb0cd4768 + .long 0x613eee91, 0xd2c3ed1a, 0x39c7ff35 + .long 0xbedc6ba1, 0x9ef68d35, 0xd7a4825c + .long 0x0cd1526a, 0xf2271e60, 0x0ab3844b + .long 0xd6c3a807, 0x2664fd8b, 0x0167d312 + .long 0x1d31175f, 0x02ee03b2, 0xf6076544 + .long 0x4be7fd90, 0x363bd6b3, 0x26f6a60a + .long 0x6eeed1c9, 0x5fabe670, 0xa741c1bf + .long 0xb3a6da94, 0x00bcf5f6, 0x98d8d9cb + .long 0x2e7d11a7, 0x17f27698, 0x49c3cc9c + .long 0x889774e1, 0xaa7c7ad5, 0x68bce87a + .long 0x8a074012, 0xded288f8, 0x57a3d037 + .long 0xbd0bb25f, 0x6d390dec, 0x6956fc3b + .long 0x3be3c09b, 0x6353c1cc, 0x42d98888 + .long 0x465a4eee, 0xf48642e9, 0x3771e98f + .long 0x2e5f3c8c, 0xdd35bc8d, 0xb42ae3d9 + .long 0xa52f58ec, 0x9a5ede41, 0x2178513a + .long 0x47972100, 0x45cddf4e, 0xe0ac139e + .long 0x359674f7, 0xa51b6135, 0x170076fa + +.L1: .long 0xaf449247, 0x81256527, 0xccaa009e + .long 0x57c54819, 0x1d9513d7, 0x81256527 + .long 0x3f41287a, 0x57c54819, 0xaf449247 + .long 0xf5e48c85, 0x910eeec1, 0x1d9513d7 + .long 0x1f0c2cdd, 0x9026d5b1, 0xae0b5394 + .long 0x71d54a59, 0xf5e48c85, 0x57c54819 + .long 0x1c63267b, 0xfe807bbd, 0x0cbec0ed + .long 0xd31343ea, 0xe95c1271, 0x910eeec1 + .long 0xf9d9c7ee, 0x71d54a59, 0x3f41287a + .long 0x9ee62949, 0xcec97417, 0x9026d5b1 + .long 0xa55d1514, 0xf183c71b, 0xd1df2327 + .long 0x21aa2b26, 0xd31343ea, 0xf5e48c85 + .long 0x9d842b80, 0xeea395c4, 0x3c656ced + .long 0xd8110ff1, 0xcd669a40, 0xfe807bbd + .long 0x3f9e9356, 0x9ee62949, 0x1f0c2cdd + .long 0x1d6708a0, 0x0c30f51d, 0xe95c1271 + .long 0xef82aa68, 0xdb3935ea, 0xb918a347 + .long 0xd14bcc9b, 0x21aa2b26, 0x71d54a59 + .long 0x99cce860, 0x356d209f, 0xff6f2fc2 + .long 0xd8af8e46, 0xc352f6de, 0xcec97417 + .long 0xf1996890, 0xd8110ff1, 0x1c63267b + .long 0x631bc508, 0xe95c7216, 0xf183c71b + .long 0x8511c306, 0x8e031a19, 0x9b9bdbd0 + .long 0xdb3839f3, 0x1d6708a0, 0xd31343ea + .long 0x7a92fffb, 0xf7003835, 0x4470ac44 + .long 0x6ce68f2a, 0x00eba0c8, 0xeea395c4 + .long 0x4caaa263, 0xd14bcc9b, 0xf9d9c7ee + .long 0xb46f7cff, 0x9a1b53c8, 0xcd669a40 + .long 0x60290934, 0x81b6f443, 0x6d40f445 + .long 0x8e976a7d, 0xd8af8e46, 0x9ee62949 + .long 0xdcf5088a, 0x9dbdc100, 0x145575d5 + .long 0x1753ab84, 0xbbf2f6d6, 0x0c30f51d + .long 0x255b139e, 0x631bc508, 0xa55d1514 + .long 0xd784eaa8, 0xce26786c, 0xdb3935ea + .long 0x6d2c864a, 0x8068c345, 0x2586d334 + .long 0x02072e24, 0xdb3839f3, 0x21aa2b26 + .long 0x06689b0a, 0x5efd72f5, 0xe0575528 + .long 0x1e52f5ea, 0x4117915b, 0x356d209f + .long 0x1d3d1db6, 0x6ce68f2a, 0x9d842b80 + .long 0x3796455c, 0xb8e0e4a8, 0xc352f6de + .long 0xdf3a4eb3, 0xc55a2330, 0xb84ffa9c + .long 0x28ae0976, 0xb46f7cff, 0xd8110ff1 + .long 0x9764bc8d, 0xd7e7a22c, 0x712510f0 + .long 0x13a13e18, 0x3e9a43cd, 0xe95c7216 + .long 0xb8ee242e, 0x8e976a7d, 0x3f9e9356 + .long 0x0c540e7b, 0x753c81ff, 0x8e031a19 + .long 0x9924c781, 0xb9220208, 0x3edcde65 + .long 0x3954de39, 0x1753ab84, 0x1d6708a0 + .long 0xf32238b5, 0xbec81497, 0x9e70b943 + .long 0xbbd2cd2c, 0x0925d861, 0xf7003835 + .long 0xcc401304, 0xd784eaa8, 0xef82aa68 + .long 0x4987e684, 0x6044fbb0, 0x00eba0c8 + .long 0x3aa11427, 0x18fe3b4a, 0x87441142 + .long 0x297aad60, 0x02072e24, 0xd14bcc9b + .long 0xf60c5e51, 0x6ef6f487, 0x5b7fdd0a + .long 0x632d78c5, 0x3fc33de4, 0x9a1b53c8 + .long 0x25b8822a, 0x1e52f5ea, 0x99cce860 + .long 0xd4fc84bc, 0x1af62fb8, 0x81b6f443 + .long 0x5690aa32, 0xa91fdefb, 0x688a110e + .long 0x1357a093, 0x3796455c, 0xd8af8e46 + .long 0x798fdd33, 0xaaa18a37, 0x357b9517 + .long 0xc2815395, 0x54d42691, 0x9dbdc100 + .long 0x21cfc0f7, 0x28ae0976, 0xf1996890 + .long 0xa0decef3, 0x7b4aa8b7, 0xbbf2f6d6 From 2858d16353ff16e140f43c7008deb26e968a379b Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Wed, 9 Apr 2025 17:40:04 +0100 Subject: [PATCH 38/85] arm64/fpsimd: Avoid clobbering kernel FPSIMD state with SMSTOP commit 01098d893fa8a6edb2b56e178b798e3e6b674f02 upstream. On system with SME, a thread's kernel FPSIMD state may be erroneously clobbered during a context switch immediately after that state is restored. Systems without SME are unaffected. If the CPU happens to be in streaming SVE mode before a context switch to a thread with kernel FPSIMD state, fpsimd_thread_switch() will restore the kernel FPSIMD state using fpsimd_load_kernel_state() while the CPU is still in streaming SVE mode. When fpsimd_thread_switch() subsequently calls fpsimd_flush_cpu_state(), this will execute an SMSTOP, causing an exit from streaming SVE mode. The exit from streaming SVE mode will cause the hardware to reset a number of FPSIMD/SVE/SME registers, clobbering the FPSIMD state. Fix this by calling fpsimd_flush_cpu_state() before restoring the kernel FPSIMD state. Fixes: e92bee9f861b ("arm64/fpsimd: Avoid erroneous elide of user state reload") Signed-off-by: Mark Rutland Cc: Ard Biesheuvel Cc: Mark Brown Cc: Will Deacon Reviewed-by: Mark Brown Link: https://lore.kernel.org/r/20250409164010.3480271-8-mark.rutland@arm.com Signed-off-by: Catalin Marinas Signed-off-by: Bin Huang --- arch/arm64/kernel/fpsimd.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c index cf65b4664fc3..2d2e48ebe189 100644 --- a/arch/arm64/kernel/fpsimd.c +++ b/arch/arm64/kernel/fpsimd.c @@ -1615,8 +1615,8 @@ void fpsimd_thread_switch(struct task_struct *next) fpsimd_save_user_state(); if (test_tsk_thread_flag(next, TIF_KERNEL_FPSTATE)) { - fpsimd_load_kernel_state(next); fpsimd_flush_cpu_state(); + fpsimd_load_kernel_state(next); } else { /* * Fix up TIF_FOREIGN_FPSTATE to correctly describe next's From 6a8b470b6361e0bfa940b03e415ded003459ae19 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Wed, 11 Dec 2024 01:02:46 +0000 Subject: [PATCH 39/85] arm64/sysreg: Update ID_AA64PFR2_EL1 to DDI0601 2024-09 commit 1ad9a56442a06298fc8e5a982d1ca6d04f84fe44 upstream. DDI0601 2024-09 defines a new feature flags in ID_AA64PFR2_EL1 describing support for injecting UNDEF exceptions, update sysreg to include this. Signed-off-by: Mark Brown Link: https://lore.kernel.org/r/20241211-arm64-2024-dpisa-v4-1-0fd403876df2@kernel.org Signed-off-by: Will Deacon Signed-off-by: Bin Huang --- arch/arm64/tools/sysreg | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index 03de36189a0d..5fc2c6cf5176 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -1008,7 +1008,12 @@ UnsignedEnum 35:32 FPMR 0b0000 NI 0b0001 IMP EndEnum -Res0 31:12 +Res0 31:20 +UnsignedEnum 19:16 UINJ + 0b0000 NI + 0b0001 IMP +EndEnum +Res0 15:12 UnsignedEnum 11:8 MTEFAR 0b0000 NI 0b0001 IMP From 93ea17c638e73d622c9f622d04e03c94e8da2486 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Wed, 11 Dec 2024 01:02:47 +0000 Subject: [PATCH 40/85] arm64/sysreg: Update ID_AA64ISAR3_EL1 to DDI0601 2024-09 commit 054339beae58ad7a401ba99095991d7434f01ad6 upstream. DDI0601 2024-09 defines several new feature flags in ID_AA64ISAR3_EL1, update our description in sysreg to reflect these. Signed-off-by: Mark Brown Link: https://lore.kernel.org/r/20241211-arm64-2024-dpisa-v4-2-0fd403876df2@kernel.org Signed-off-by: Will Deacon [ bh: backport to v6.6 - preserve existing PACM FULL_IMP, TLBIW, FAMINMAX, and CPA definitions. ] Signed-off-by: Bin Huang --- arch/arm64/tools/sysreg | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index 5fc2c6cf5176..efc2881f5531 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -1490,7 +1490,28 @@ EndEnum EndSysreg Sysreg ID_AA64ISAR3_EL1 3 0 0 6 3 -Res0 63:12 +Res0 63:32 +UnsignedEnum 31:28 FPRCVT + 0b0000 NI + 0b0001 IMP +EndEnum +UnsignedEnum 27:24 LSUI + 0b0000 NI + 0b0001 IMP +EndEnum +UnsignedEnum 23:20 OCCMO + 0b0000 NI + 0b0001 IMP +EndEnum +UnsignedEnum 19:16 LSFE + 0b0000 NI + 0b0001 IMP +EndEnum +UnsignedEnum 15:12 PACM + 0b0000 NI + 0b0001 TRIVIAL_IMP + 0b0010 FULL_IMP +EndEnum UnsignedEnum 11:8 TLBIW 0b0000 NI 0b0001 IMP From 1f34fa964e410f4cc7fffa10c883b90c4a538f37 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Wed, 11 Dec 2024 01:02:48 +0000 Subject: [PATCH 41/85] arm64/sysreg: Update ID_AA64FPFR0_EL1 to DDI0601 2024-09 commit 12b5ff517a19b77abb3f1cb082a0e0124307fb53 upstream. DDI0601 2024-09 defines two new feature flags in ID_AA64FPFR0_EL1 describing new FP8 operations, describe them in sysreg. Signed-off-by: Mark Brown Link: https://lore.kernel.org/r/20241211-arm64-2024-dpisa-v4-3-0fd403876df2@kernel.org Signed-off-by: Will Deacon Signed-off-by: Bin Huang --- arch/arm64/tools/sysreg | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index efc2881f5531..c3d3222e57e4 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -1178,7 +1178,15 @@ UnsignedEnum 28 F8DP2 0b0 NI 0b1 IMP EndEnum -Res0 27:2 +UnsignedEnum 27 F8MM8 + 0b0 NI + 0b1 IMP +EndEnum +UnsignedEnum 26 F8MM4 + 0b0 NI + 0b1 IMP +EndEnum +Res0 25:2 UnsignedEnum 1 F8E4M3 0b0 NI 0b1 IMP From 6771ead5bf914b92b28ccb2e079fb552037a40c7 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Wed, 11 Dec 2024 01:02:49 +0000 Subject: [PATCH 42/85] arm64/sysreg: Update ID_AA64ZFR0_EL1 to DDI0601 2024-09 commit 9a43ee86434970fcb259eeea766820b607f868be upstream. DDI0601 2024-09 introduces SVE 2.2 as well as a few new optional features, update sysreg to reflect the changes in ID_AA64ZFR0_EL1 enumerating them. Signed-off-by: Mark Brown Link: https://lore.kernel.org/r/20241211-arm64-2024-dpisa-v4-4-0fd403876df2@kernel.org Signed-off-by: Will Deacon Signed-off-by: Bin Huang --- arch/arm64/tools/sysreg | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index c3d3222e57e4..257750a80acd 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -1038,7 +1038,10 @@ UnsignedEnum 55:52 F32MM 0b0000 NI 0b0001 IMP EndEnum -Res0 51:48 +UnsignedEnum 51:48 F16MM + 0b0000 NI + 0b0001 IMP +EndEnum UnsignedEnum 47:44 I8MM 0b0000 NI 0b0001 IMP @@ -1056,6 +1059,7 @@ Res0 31:28 UnsignedEnum 27:24 B16B16 0b0000 NI 0b0001 IMP + 0b0010 BFSCALE EndEnum UnsignedEnum 23:20 BF16 0b0000 NI @@ -1066,16 +1070,22 @@ UnsignedEnum 19:16 BitPerm 0b0000 NI 0b0001 IMP EndEnum -Res0 15:8 +UnsignedEnum 15:12 EltPerm + 0b0000 NI + 0b0001 IMP +EndEnum +Res0 11:8 UnsignedEnum 7:4 AES 0b0000 NI 0b0001 IMP 0b0010 PMULL128 + 0b0011 AES2 EndEnum UnsignedEnum 3:0 SVEver 0b0000 IMP 0b0001 SVE2 0b0010 SVE2p1 + 0b0011 SVE2p2 EndEnum EndSysreg From ae45bcd8d3f3dfd96ceb029a42de556a2ddaca09 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Wed, 11 Dec 2024 01:02:51 +0000 Subject: [PATCH 43/85] arm64/sysreg: Update ID_AA64ISAR2_EL1 to DDI0601 2024-09 commit d66e21d59ed0e043e68ef8c6541c1e9f1a962614 upstream. DDI0601 2024-09 introduces new features which are enumerated via ID_AA64ISAR2_EL1, update the sysreg file to reflect these updates. Signed-off-by: Mark Brown Link: https://lore.kernel.org/r/20241211-arm64-2024-dpisa-v4-6-0fd403876df2@kernel.org Signed-off-by: Will Deacon Signed-off-by: Bin Huang --- arch/arm64/tools/sysreg | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index 257750a80acd..30e7db54f409 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -1450,12 +1450,16 @@ EndEnum UnsignedEnum 55:52 CSSC 0b0000 NI 0b0001 IMP + 0b0010 CMPBR EndEnum UnsignedEnum 51:48 RPRFM 0b0000 NI 0b0001 IMP EndEnum -Res0 47:44 +UnsignedEnum 47:44 PCDPHINT + 0b0000 NI + 0b0001 IMP +EndEnum UnsignedEnum 43:40 PRFMSLC 0b0000 NI 0b0001 IMP From 7197e40e5ba9173d331e966168ca78d5d34ef815 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Tue, 7 Jan 2025 22:59:42 +0000 Subject: [PATCH 44/85] arm64/sysreg: Update ID_AA64SMFR0_EL1 to DDI0601 2024-12 commit 47e4717eb6fc5fc7d5437bd096885e0e85d3cea3 upstream. DDI0601 2024-12 introduces SME 2.2 as well as a few new optional features, update sysreg to reflect the changes in ID_AA64SMFR0_EL1 enumerating them. Signed-off-by: Mark Brown Link: https://lore.kernel.org/r/20250107-arm64-2024-dpisa-v5-2-7578da51fc3d@kernel.org Signed-off-by: Will Deacon Signed-off-by: Bin Huang --- arch/arm64/tools/sysreg | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index 30e7db54f409..936c3386185d 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -1103,7 +1103,7 @@ UnsignedEnum 59:56 SMEver 0b0000 SME 0b0001 SME2 0b0010 SME2p1 - 0b0000 IMP + 0b0011 SME2p2 EndEnum UnsignedEnum 55:52 I16I64 0b0000 NI @@ -1167,7 +1167,29 @@ UnsignedEnum 28 SF8DP2 0b0 NI 0b1 IMP EndEnum -Res0 27:0 +Res0 27:26 +UnsignedEnum 25 SBitPerm + 0b0 NI + 0b1 IMP +EndEnum +UnsignedEnum 24 AES + 0b0 NI + 0b1 IMP +EndEnum +UnsignedEnum 23 SFEXPA + 0b0 NI + 0b1 IMP +EndEnum +Res0 22:17 +UnsignedEnum 16 STMOP + 0b0 NI + 0b1 IMP +EndEnum +Res0 15:1 +UnsignedEnum 0 SMOP4 + 0b0 NI + 0b1 IMP +EndEnum EndSysreg Sysreg ID_AA64FPFR0_EL1 3 0 0 4 7 From 172f195c87500560cd67cf08a86d09d189efc4db Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Tue, 7 Jan 2025 22:59:43 +0000 Subject: [PATCH 45/85] arm64/hwcap: Describe 2024 dpISA extensions to userspace commit 819935464cb2f72fff8dfbbf95cf2726d4a66388 upstream. The 2024 dpISA introduces a number of architecture features all of which only add new instructions so only require the addition of hwcaps and ID register visibility. Signed-off-by: Mark Brown Link: https://lore.kernel.org/r/20250107-arm64-2024-dpisa-v5-3-7578da51fc3d@kernel.org Signed-off-by: Will Deacon [ bh: backport to v6.6 - re-anchor HWCAP tables and retain target SVE/SME feature gating. ] Signed-off-by: Bin Huang --- Documentation/arch/arm64/elf_hwcaps.rst | 50 +++++++++++++++++++++++++ arch/arm64/include/asm/hwcap.h | 15 ++++++++ arch/arm64/include/uapi/asm/hwcap.h | 15 ++++++++ arch/arm64/kernel/cpufeature.c | 31 +++++++++++++++ arch/arm64/kernel/cpuinfo.c | 15 ++++++++ 5 files changed, 126 insertions(+) diff --git a/Documentation/arch/arm64/elf_hwcaps.rst b/Documentation/arch/arm64/elf_hwcaps.rst index 79802e7707b2..6753dde125c1 100644 --- a/Documentation/arch/arm64/elf_hwcaps.rst +++ b/Documentation/arch/arm64/elf_hwcaps.rst @@ -170,6 +170,56 @@ HWCAP_PACG ID_AA64ISAR1_EL1.GPI == 0b0001, as described by Documentation/arch/arm64/pointer-authentication.rst. +HWCAP_CMPBR + Functionality implied by ID_AA64ISAR2_EL1.CSSC == 0b0010. + +HWCAP_FPRCVT + Functionality implied by ID_AA64ISAR3_EL1.FPRCVT == 0b0001. + +HWCAP_F8MM8 + Functionality implied by ID_AA64FPFR0_EL1.F8MM8 == 0b0001. + +HWCAP_F8MM4 + Functionality implied by ID_AA64FPFR0_EL1.F8MM4 == 0b0001. + +HWCAP_SVE_F16MM + Functionality implied by ID_AA64PFR0_EL1.SVE == 0b0001 and + ID_AA64ZFR0_EL1.F16MM == 0b0001. + +HWCAP_SVE_ELTPERM + Functionality implied by ID_AA64PFR0_EL1.SVE == 0b0001 and + ID_AA64ZFR0_EL1.ELTPERM == 0b0001. + +HWCAP_SVE_AES2 + Functionality implied by ID_AA64PFR0_EL1.SVE == 0b0001 and + ID_AA64ZFR0_EL1.AES == 0b0011. + +HWCAP_SVE_BFSCALE + Functionality implied by ID_AA64PFR0_EL1.SVE == 0b0001 and + ID_AA64ZFR0_EL1.B16B16 == 0b0010. + +HWCAP_SVE2P2 + Functionality implied by ID_AA64PFR0_EL1.SVE == 0b0001 and + ID_AA64ZFR0_EL1.SVEver == 0b0011. + +HWCAP_SME2P2 + Functionality implied by ID_AA64SMFR0_EL1.SMEver == 0b0011. + +HWCAP_SME_SBITPERM + Functionality implied by ID_AA64SMFR0_EL1.SBitPerm == 0b1. + +HWCAP_SME_AES + Functionality implied by ID_AA64SMFR0_EL1.AES == 0b1. + +HWCAP_SME_SFEXPA + Functionality implied by ID_AA64SMFR0_EL1.SFEXPA == 0b1. + +HWCAP_SME_STMOP + Functionality implied by ID_AA64SMFR0_EL1.STMOP == 0b1. + +HWCAP_SME_SMOP4 + Functionality implied by ID_AA64SMFR0_EL1.SMOP4 == 0b1. + HWCAP2_DCPODP Functionality implied by ID_AA64ISAR1_EL1.DPB == 0b0010. diff --git a/arch/arm64/include/asm/hwcap.h b/arch/arm64/include/asm/hwcap.h index 92e6cd25ccfb..8a9afecc9c5a 100644 --- a/arch/arm64/include/asm/hwcap.h +++ b/arch/arm64/include/asm/hwcap.h @@ -92,6 +92,21 @@ #define KERNEL_HWCAP_SB __khwcap_feature(SB) #define KERNEL_HWCAP_PACA __khwcap_feature(PACA) #define KERNEL_HWCAP_PACG __khwcap_feature(PACG) +#define KERNEL_HWCAP_CMPBR __khwcap_feature(CMPBR) +#define KERNEL_HWCAP_FPRCVT __khwcap_feature(FPRCVT) +#define KERNEL_HWCAP_F8MM8 __khwcap_feature(F8MM8) +#define KERNEL_HWCAP_F8MM4 __khwcap_feature(F8MM4) +#define KERNEL_HWCAP_SVE_F16MM __khwcap_feature(SVE_F16MM) +#define KERNEL_HWCAP_SVE_ELTPERM __khwcap_feature(SVE_ELTPERM) +#define KERNEL_HWCAP_SVE_AES2 __khwcap_feature(SVE_AES2) +#define KERNEL_HWCAP_SVE_BFSCALE __khwcap_feature(SVE_BFSCALE) +#define KERNEL_HWCAP_SVE2P2 __khwcap_feature(SVE2P2) +#define KERNEL_HWCAP_SME2P2 __khwcap_feature(SME2P2) +#define KERNEL_HWCAP_SME_SBITPERM __khwcap_feature(SME_SBITPERM) +#define KERNEL_HWCAP_SME_AES __khwcap_feature(SME_AES) +#define KERNEL_HWCAP_SME_SFEXPA __khwcap_feature(SME_SFEXPA) +#define KERNEL_HWCAP_SME_STMOP __khwcap_feature(SME_STMOP) +#define KERNEL_HWCAP_SME_SMOP4 __khwcap_feature(SME_SMOP4) #define __khwcap2_feature(x) (const_ilog2(HWCAP2_ ## x) + 64) #define KERNEL_HWCAP_DCPODP __khwcap2_feature(DCPODP) diff --git a/arch/arm64/include/uapi/asm/hwcap.h b/arch/arm64/include/uapi/asm/hwcap.h index 49fb2d8b5375..96d246fbee0b 100644 --- a/arch/arm64/include/uapi/asm/hwcap.h +++ b/arch/arm64/include/uapi/asm/hwcap.h @@ -55,6 +55,21 @@ #define HWCAP_SB (1 << 29) #define HWCAP_PACA (1 << 30) #define HWCAP_PACG (1UL << 31) +#define HWCAP_CMPBR (1UL << 33) +#define HWCAP_FPRCVT (1UL << 34) +#define HWCAP_F8MM8 (1UL << 35) +#define HWCAP_F8MM4 (1UL << 36) +#define HWCAP_SVE_F16MM (1UL << 37) +#define HWCAP_SVE_ELTPERM (1UL << 38) +#define HWCAP_SVE_AES2 (1UL << 39) +#define HWCAP_SVE_BFSCALE (1UL << 40) +#define HWCAP_SVE2P2 (1UL << 41) +#define HWCAP_SME2P2 (1UL << 42) +#define HWCAP_SME_SBITPERM (1UL << 43) +#define HWCAP_SME_AES (1UL << 44) +#define HWCAP_SME_SFEXPA (1UL << 45) +#define HWCAP_SME_STMOP (1UL << 46) +#define HWCAP_SME_SMOP4 (1UL << 47) /* * HWCAP2 flags - for AT_HWCAP2 diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index fa32aebdd45e..27bef8db827e 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -238,6 +238,7 @@ static const struct arm64_ftr_bits ftr_id_aa64isar2[] = { }; static const struct arm64_ftr_bits ftr_id_aa64isar3[] = { + ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR3_EL1_FPRCVT_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR3_EL1_FAMINMAX_SHIFT, 4, 0), ARM64_FTR_END, }; @@ -285,6 +286,8 @@ static const struct arm64_ftr_bits ftr_id_aa64zfr0[] = { FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_EL1_F64MM_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE), FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_EL1_F32MM_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE), + FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_EL1_F16MM_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE), FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_EL1_I8MM_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE), @@ -297,6 +300,8 @@ static const struct arm64_ftr_bits ftr_id_aa64zfr0[] = { FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_EL1_BF16_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE), FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_EL1_BitPerm_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE), + FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_EL1_EltPerm_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE), FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_EL1_AES_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE), @@ -341,6 +346,16 @@ static const struct arm64_ftr_bits ftr_id_aa64smfr0[] = { FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_SF8DP4_SHIFT, 1, 0), ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_SF8DP2_SHIFT, 1, 0), + ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), + FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_SBitPerm_SHIFT, 1, 0), + ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), + FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_AES_SHIFT, 1, 0), + ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), + FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_SFEXPA_SHIFT, 1, 0), + ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), + FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_STMOP_SHIFT, 1, 0), + ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), + FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_SMOP4_SHIFT, 1, 0), ARM64_FTR_END, }; @@ -349,6 +364,8 @@ static const struct arm64_ftr_bits ftr_id_aa64fpfr0[] = { ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8FMA_SHIFT, 1, 0), ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8DP4_SHIFT, 1, 0), ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8DP2_SHIFT, 1, 0), + ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8MM8_SHIFT, 1, 0), + ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8MM4_SHIFT, 1, 0), ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8E4M3_SHIFT, 1, 0), ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8E5M2_SHIFT, 1, 0), ARM64_FTR_END, @@ -3051,12 +3068,15 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { HWCAP_CAP(ID_AA64MMFR2_EL1, AT, IMP, CAP_HWCAP, KERNEL_HWCAP_USCAT), #ifdef CONFIG_ARM64_SVE HWCAP_CAP(ID_AA64PFR0_EL1, SVE, IMP, CAP_HWCAP, KERNEL_HWCAP_SVE), + HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, SVEver, SVE2p2, CAP_HWCAP, KERNEL_HWCAP_SVE2P2), HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, SVEver, SVE2p1, CAP_HWCAP, KERNEL_HWCAP_SVE2P1), HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, SVEver, SVE2, CAP_HWCAP, KERNEL_HWCAP_SVE2), HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, AES, IMP, CAP_HWCAP, KERNEL_HWCAP_SVEAES), HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, AES, PMULL128, CAP_HWCAP, KERNEL_HWCAP_SVEPMULL), + HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, AES, AES2, CAP_HWCAP, KERNEL_HWCAP_SVE_AES2), HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, BitPerm, IMP, CAP_HWCAP, KERNEL_HWCAP_SVEBITPERM), HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, B16B16, IMP, CAP_HWCAP, KERNEL_HWCAP_SVE_B16B16), + HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, B16B16, BFSCALE, CAP_HWCAP, KERNEL_HWCAP_SVE_BFSCALE), HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, BF16, IMP, CAP_HWCAP, KERNEL_HWCAP_SVEBF16), HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, BF16, EBF16, CAP_HWCAP, KERNEL_HWCAP_SVE_EBF16), HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, SHA3, IMP, CAP_HWCAP, KERNEL_HWCAP_SVESHA3), @@ -3064,6 +3084,8 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, I8MM, IMP, CAP_HWCAP, KERNEL_HWCAP_SVEI8MM), HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, F32MM, IMP, CAP_HWCAP, KERNEL_HWCAP_SVEF32MM), HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, F64MM, IMP, CAP_HWCAP, KERNEL_HWCAP_SVEF64MM), + HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, F16MM, IMP, CAP_HWCAP, KERNEL_HWCAP_SVE_F16MM), + HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, EltPerm, IMP, CAP_HWCAP, KERNEL_HWCAP_SVE_ELTPERM), #endif HWCAP_CAP(ID_AA64PFR1_EL1, SSBS, SSBS2, CAP_HWCAP, KERNEL_HWCAP_SSBS), #ifdef CONFIG_ARM64_BTI @@ -3080,6 +3102,7 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { HWCAP_CAP(ID_AA64MMFR0_EL1, ECV, IMP, CAP_HWCAP, KERNEL_HWCAP_ECV), HWCAP_CAP(ID_AA64MMFR1_EL1, AFP, IMP, CAP_HWCAP, KERNEL_HWCAP_AFP), HWCAP_CAP(ID_AA64ISAR2_EL1, CSSC, IMP, CAP_HWCAP, KERNEL_HWCAP_CSSC), + HWCAP_CAP(ID_AA64ISAR2_EL1, CSSC, CMPBR, CAP_HWCAP, KERNEL_HWCAP_CMPBR), HWCAP_CAP(ID_AA64ISAR2_EL1, RPRFM, IMP, CAP_HWCAP, KERNEL_HWCAP_RPRFM), HWCAP_CAP(ID_AA64ISAR2_EL1, RPRES, IMP, CAP_HWCAP, KERNEL_HWCAP_RPRES), HWCAP_CAP(ID_AA64ISAR2_EL1, WFxT, IMP, CAP_HWCAP, KERNEL_HWCAP_WFXT), @@ -3089,6 +3112,7 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64PFR1_EL1, SME, IMP, CAP_HWCAP, KERNEL_HWCAP_SME), HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, FA64, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_FA64), HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, LUTv2, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_LUTV2), + HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SMEver, SME2p2, CAP_HWCAP, KERNEL_HWCAP_SME2P2), HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SMEver, SME2p1, CAP_HWCAP, KERNEL_HWCAP_SME2P1), HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SMEver, SME2, CAP_HWCAP, KERNEL_HWCAP_SME2), HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, I16I64, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_I16I64), @@ -3106,6 +3130,13 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SF8FMA, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_SF8FMA), HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SF8DP4, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_SF8DP4), HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SF8DP2, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_SF8DP2), + HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SF8MM8, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_SF8MM8), + HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SF8MM4, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_SF8MM4), + HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SBitPerm, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_SBITPERM), + HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, AES, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_AES), + HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SFEXPA, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_SFEXPA), + HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, STMOP, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_STMOP), + HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SMOP4, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_SMOP4), #endif /* CONFIG_ARM64_SME */ HWCAP_CAP(ID_AA64FPFR0_EL1, F8CVT, IMP, CAP_HWCAP, KERNEL_HWCAP_F8CVT), HWCAP_CAP(ID_AA64FPFR0_EL1, F8FMA, IMP, CAP_HWCAP, KERNEL_HWCAP_F8FMA), diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c index e7910b3e7fb4..1f003f6a9c81 100644 --- a/arch/arm64/kernel/cpuinfo.c +++ b/arch/arm64/kernel/cpuinfo.c @@ -145,6 +145,21 @@ static const char *const hwcap_str[] = { [KERNEL_HWCAP_SME_SF8FMA] = "smesf8fma", [KERNEL_HWCAP_SME_SF8DP4] = "smesf8dp4", [KERNEL_HWCAP_SME_SF8DP2] = "smesf8dp2", + [KERNEL_HWCAP_CMPBR] = "cmpbr", + [KERNEL_HWCAP_FPRCVT] = "fprcvt", + [KERNEL_HWCAP_F8MM8] = "f8mm8", + [KERNEL_HWCAP_F8MM4] = "f8mm4", + [KERNEL_HWCAP_SVE_F16MM] = "svef16mm", + [KERNEL_HWCAP_SVE_ELTPERM] = "sveeltperm", + [KERNEL_HWCAP_SVE_AES2] = "sveaes2", + [KERNEL_HWCAP_SVE_BFSCALE] = "svebfscale", + [KERNEL_HWCAP_SVE2P2] = "sve2p2", + [KERNEL_HWCAP_SME2P2] = "sme2p2", + [KERNEL_HWCAP_SME_SBITPERM] = "smesbitperm", + [KERNEL_HWCAP_SME_AES] = "smeaes", + [KERNEL_HWCAP_SME_SFEXPA] = "smesfexpa", + [KERNEL_HWCAP_SME_STMOP] = "smestmop", + [KERNEL_HWCAP_SME_SMOP4] = "smesmop4", }; #ifdef CONFIG_COMPAT From 1a291ff893fab1c687c311d45d68634ca98a981a Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Tue, 7 Jan 2025 22:59:44 +0000 Subject: [PATCH 46/85] KVM: arm64: Allow control of dpISA extensions in ID_AA64ISAR3_EL1 commit fd22af17a458d98c14cebd00091cebf69b954b40 upstream. ID_AA64ISAR3_EL1 is currently marked as unallocated in KVM but does have a number of bitfields defined in it. Expose FPRCVT and FAMINMAX, two simple instruction only extensions to guests. Reviewed-by: Marc Zyngier Signed-off-by: Mark Brown Link: https://lore.kernel.org/r/20250107-arm64-2024-dpisa-v5-4-7578da51fc3d@kernel.org Signed-off-by: Will Deacon [ bh: backport to v6.6 - add the target-native ID_WRITABLE() initializer using SYS_DESC(SYS_##name). ] Signed-off-by: Bin Huang --- arch/arm64/kvm/sys_regs.c | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index f844dba536b3..ebb3808439ed 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -1350,6 +1350,9 @@ static u64 __kvm_read_sanitised_id_reg(const struct kvm_vcpu *vcpu, val &= ~ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_WFxT); val &= ~ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_MOPS); break; + case SYS_ID_AA64ISAR3_EL1: + val &= ID_AA64ISAR3_EL1_FPRCVT | ID_AA64ISAR3_EL1_FAMINMAX; + break; case SYS_ID_AA64MMFR2_EL1: val &= ~ID_AA64MMFR2_EL1_CCIDX_MASK; break; @@ -1881,6 +1884,17 @@ static unsigned int elx2_visibility(const struct kvm_vcpu *vcpu, .val = 0, \ } +/* sys_reg_desc initialiser for writable ID registers */ +#define ID_WRITABLE(name, mask) { \ + SYS_DESC(SYS_##name), \ + .access = access_id_reg, \ + .get_user = get_id_reg, \ + .set_user = set_id_reg, \ + .visibility = id_visibility, \ + .reset = kvm_read_sanitised_id_reg, \ + .val = mask, \ +} + /* sys_reg_desc initialiser for known cpufeature ID registers */ #define AA32_ID_SANITISED(name) { \ SYS_DESC(SYS_##name), \ @@ -2099,7 +2113,8 @@ static const struct sys_reg_desc sys_reg_descs[] = { ID_SANITISED(ID_AA64ISAR0_EL1), ID_SANITISED(ID_AA64ISAR1_EL1), ID_SANITISED(ID_AA64ISAR2_EL1), - ID_UNALLOCATED(6,3), + ID_WRITABLE(ID_AA64ISAR3_EL1, (ID_AA64ISAR3_EL1_FPRCVT | + ID_AA64ISAR3_EL1_FAMINMAX)), ID_UNALLOCATED(6,4), ID_UNALLOCATED(6,5), ID_UNALLOCATED(6,6), From 23b080a676fd7b2144b846d59612c673f739beb9 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Tue, 7 Jan 2025 22:59:45 +0000 Subject: [PATCH 47/85] kselftest/arm64: Add 2024 dpISA extensions to hwcap test commit 8600640d21cf90f3c5c4f06a5b214fbe4be9a74a upstream. Add coverage of the hwcaps for the 2024 dpISA extensions to the hwcap test. We don't actually test SIGILL generation for CMPBR since the need to branch makes it a pain to generate and the SIGILL detection would be unreliable anyway. Since this should be very unusual we provide a stub function rather than supporting a missing test. The sigill functions aren't well sorted in the file so the ordering is a bit random. Signed-off-by: Mark Brown Link: https://lore.kernel.org/r/20250107-arm64-2024-dpisa-v5-5-7578da51fc3d@kernel.org Signed-off-by: Will Deacon [ bh: backport to v6.6 - re-anchor the arm64 HWCAP selftest to the target test layout. ] Signed-off-by: Bin Huang --- tools/testing/selftests/arm64/abi/hwcap.c | 235 +++++++++++++++++++++- 1 file changed, 233 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/arm64/abi/hwcap.c b/tools/testing/selftests/arm64/abi/hwcap.c index dd821046ebd5..56b7fcd7b21d 100644 --- a/tools/testing/selftests/arm64/abi/hwcap.c +++ b/tools/testing/selftests/arm64/abi/hwcap.c @@ -46,6 +46,12 @@ static void atomics_sigill(void) asm volatile(".inst 0xb82003ff" : : : ); } +static void cmpbr_sigill(void) +{ + /* Not implemented, too complicated and unreliable anyway */ +} + + static void crc32_sigill(void) { /* CRC32W W0, W0, W1 */ @@ -82,6 +88,18 @@ static void f8fma_sigill(void) asm volatile(".inst 0xec0fc00"); } +static void f8mm4_sigill(void) +{ + /* FMMLA V0.4SH, V0.16B, V0.16B */ + asm volatile(".inst 0x6e00ec00"); +} + +static void f8mm8_sigill(void) +{ + /* FMMLA V0.4S, V0.16B, V0.16B */ + asm volatile(".inst 0x6e80ec00"); +} + static void faminmax_sigill(void) { /* FAMIN V0.4H, V0.4H, V0.4H */ @@ -98,6 +116,12 @@ static void fpmr_sigill(void) asm volatile("mrs x0, S3_3_C4_C4_2" : : : "x0"); } +static void fprcvt_sigill(void) +{ + /* FCVTAS S0, H0 */ + asm volatile(".inst 0x1efa0000"); +} + static void ilrcpc_sigill(void) { /* LDAPUR W0, [SP, #8] */ @@ -195,6 +219,42 @@ static void sme2p1_sigill(void) asm volatile("msr S0_3_C4_C6_3, xzr" : : : ); } +static void sme2p2_sigill(void) +{ + /* SMSTART SM */ + asm volatile("msr S0_3_C4_C3_3, xzr" : : : ); + + /* UXTB Z0.D, P0/Z, Z0.D */ + asm volatile(".inst 0x4c1a000" : : : ); + + /* SMSTOP */ + asm volatile("msr S0_3_C4_C6_3, xzr" : : : ); +} + +static void sme_aes_sigill(void) +{ + /* SMSTART SM */ + asm volatile("msr S0_3_C4_C3_3, xzr" : : : ); + + /* AESD z0.b, z0.b, z0.b */ + asm volatile(".inst 0x4522e400" : : : "z0"); + + /* SMSTOP */ + asm volatile("msr S0_3_C4_C6_3, xzr" : : : ); +} + +static void sme_sbitperm_sigill(void) +{ + /* SMSTART SM */ + asm volatile("msr S0_3_C4_C3_3, xzr" : : : ); + + /* BDEP Z0.B, Z0.B, Z0.B */ + asm volatile(".inst 0x4500b400" : : : "z0"); + + /* SMSTOP */ + asm volatile("msr S0_3_C4_C6_3, xzr" : : : ); +} + static void smei16i32_sigill(void) { /* SMSTART */ @@ -308,8 +368,44 @@ static void smesf8fma_sigill(void) /* SMSTART */ asm volatile("msr S0_3_C4_C7_3, xzr" : : : ); - /* FMLALB V0.8H, V0.16B, V0.16B */ - asm volatile(".inst 0xec0fc00"); + /* FMLALB Z0.8H, Z0.B, Z0.B */ + asm volatile(".inst 0x64205000"); + + /* SMSTOP */ + asm volatile("msr S0_3_C4_C6_3, xzr" : : : ); +} + +static void smesfexpa_sigill(void) +{ + /* SMSTART */ + asm volatile("msr S0_3_C4_C7_3, xzr" : : : ); + + /* FEXPA Z0.D, Z0.D */ + asm volatile(".inst 0x04e0b800"); + + /* SMSTOP */ + asm volatile("msr S0_3_C4_C6_3, xzr" : : : ); +} + +static void smesmop4_sigill(void) +{ + /* SMSTART */ + asm volatile("msr S0_3_C4_C7_3, xzr" : : : ); + + /* SMOP4A ZA0.S, Z0.B, { Z0.B - Z1.B } */ + asm volatile(".inst 0x80108000"); + + /* SMSTOP */ + asm volatile("msr S0_3_C4_C6_3, xzr" : : : ); +} + +static void smestmop_sigill(void) +{ + /* SMSTART */ + asm volatile("msr S0_3_C4_C7_3, xzr" : : : ); + + /* STMOPA ZA0.S, { Z0.H - Z1.H }, Z0.H, Z20[0] */ + asm volatile(".inst 0x80408008"); /* SMSTOP */ asm volatile("msr S0_3_C4_C6_3, xzr" : : : ); @@ -333,18 +429,42 @@ static void sve2p1_sigill(void) asm volatile(".inst 0xC400A000" : : : "z0"); } +static void sve2p2_sigill(void) +{ + /* NOT Z0.D, P0/Z, Z0.D */ + asm volatile(".inst 0x4cea000" : : : "z0"); +} + static void sveaes_sigill(void) { /* AESD z0.b, z0.b, z0.b */ asm volatile(".inst 0x4522e400" : : : "z0"); } +static void sveaes2_sigill(void) +{ + /* AESD {Z0.B - Z1.B }, { Z0.B - Z1.B }, Z0.Q */ + asm volatile(".inst 0x4522ec00" : : : "z0"); +} + static void sveb16b16_sigill(void) { /* BFADD Z0.H, Z0.H, Z0.H */ asm volatile(".inst 0x65000000" : : : ); } +static void svebfscale_sigill(void) +{ + /* BFSCALE Z0.H, P0/M, Z0.H, Z0.H */ + asm volatile(".inst 0x65098000" : : : "z0"); +} + +static void svef16mm_sigill(void) +{ + /* FMMLA Z0.S, Z0.H, Z0.H */ + asm volatile(".inst 0x6420e400"); +} + static void svepmull_sigill(void) { /* PMULLB Z0.Q, Z0.D, Z0.D */ @@ -363,6 +483,12 @@ static void svesha3_sigill(void) asm volatile(".inst 0x4203800" : : : "z0"); } +static void sveeltperm_sigill(void) +{ + /* COMPACT Z0.B, P0, Z0.B */ + asm volatile(".inst 0x5218000" : : : "x0"); +} + static void svesm4_sigill(void) { /* SM4E Z0.S, Z0.S, Z0.S */ @@ -425,6 +551,13 @@ static const struct hwcap_data { .cpuinfo = "aes", .sigill_fn = aes_sigill, }, + { + .name = "CMPBR", + .at_hwcap = AT_HWCAP, + .hwcap_bit = HWCAP_CMPBR, + .cpuinfo = "cmpbr", + .sigill_fn = cmpbr_sigill, + }, { .name = "CRC32", .at_hwcap = AT_HWCAP, @@ -479,6 +612,20 @@ static const struct hwcap_data { .cpuinfo = "f8fma", .sigill_fn = f8fma_sigill, }, + { + .name = "F8MM8", + .at_hwcap = AT_HWCAP, + .hwcap_bit = HWCAP_F8MM8, + .cpuinfo = "f8mm8", + .sigill_fn = f8mm8_sigill, + }, + { + .name = "F8MM4", + .at_hwcap = AT_HWCAP, + .hwcap_bit = HWCAP_F8MM4, + .cpuinfo = "f8mm4", + .sigill_fn = f8mm4_sigill, + }, { .name = "FAMINMAX", .at_hwcap = AT_HWCAP2, @@ -501,6 +648,13 @@ static const struct hwcap_data { .sigill_fn = fpmr_sigill, .sigill_reliable = true, }, + { + .name = "FPRCVT", + .at_hwcap = AT_HWCAP, + .hwcap_bit = HWCAP_FPRCVT, + .cpuinfo = "fprcvt", + .sigill_fn = fprcvt_sigill, + }, { .name = "JSCVT", .at_hwcap = AT_HWCAP, @@ -617,6 +771,20 @@ static const struct hwcap_data { .cpuinfo = "sme2p1", .sigill_fn = sme2p1_sigill, }, + { + .name = "SME 2.2", + .at_hwcap = AT_HWCAP, + .hwcap_bit = HWCAP_SME2P2, + .cpuinfo = "sme2p2", + .sigill_fn = sme2p2_sigill, + }, + { + .name = "SME AES", + .at_hwcap = AT_HWCAP, + .hwcap_bit = HWCAP_SME_AES, + .cpuinfo = "smeaes", + .sigill_fn = sme_aes_sigill, + }, { .name = "SME I16I32", .at_hwcap = AT_HWCAP2, @@ -666,6 +834,13 @@ static const struct hwcap_data { .cpuinfo = "smelutv2", .sigill_fn = smelutv2_sigill, }, + { + .name = "SME SBITPERM", + .at_hwcap = AT_HWCAP, + .hwcap_bit = HWCAP_SME_SBITPERM, + .cpuinfo = "smesbitperm", + .sigill_fn = sme_sbitperm_sigill, + }, { .name = "SME SF8FMA", .at_hwcap = AT_HWCAP2, @@ -687,6 +862,27 @@ static const struct hwcap_data { .cpuinfo = "smesf8dp4", .sigill_fn = smesf8dp4_sigill, }, + { + .name = "SME SFEXPA", + .at_hwcap = AT_HWCAP, + .hwcap_bit = HWCAP_SME_SFEXPA, + .cpuinfo = "smesfexpa", + .sigill_fn = smesfexpa_sigill, + }, + { + .name = "SME SMOP4", + .at_hwcap = AT_HWCAP, + .hwcap_bit = HWCAP_SME_SMOP4, + .cpuinfo = "smesmop4", + .sigill_fn = smesmop4_sigill, + }, + { + .name = "SME STMOP", + .at_hwcap = AT_HWCAP, + .hwcap_bit = HWCAP_SME_STMOP, + .cpuinfo = "smestmop", + .sigill_fn = smestmop_sigill, + }, { .name = "SVE", .at_hwcap = AT_HWCAP, @@ -709,6 +905,13 @@ static const struct hwcap_data { .cpuinfo = "sve2p1", .sigill_fn = sve2p1_sigill, }, + { + .name = "SVE 2.2", + .at_hwcap = AT_HWCAP, + .hwcap_bit = HWCAP_SVE2P2, + .cpuinfo = "sve2p2", + .sigill_fn = sve2p2_sigill, + }, { .name = "SVE AES", .at_hwcap = AT_HWCAP2, @@ -716,6 +919,34 @@ static const struct hwcap_data { .cpuinfo = "sveaes", .sigill_fn = sveaes_sigill, }, + { + .name = "SVE AES2", + .at_hwcap = AT_HWCAP, + .hwcap_bit = HWCAP_SVE_AES2, + .cpuinfo = "sveaes2", + .sigill_fn = sveaes2_sigill, + }, + { + .name = "SVE BFSCALE", + .at_hwcap = AT_HWCAP, + .hwcap_bit = HWCAP_SVE_BFSCALE, + .cpuinfo = "svebfscale", + .sigill_fn = svebfscale_sigill, + }, + { + .name = "SVE ELTPERM", + .at_hwcap = AT_HWCAP, + .hwcap_bit = HWCAP_SVE_ELTPERM, + .cpuinfo = "sveeltperm", + .sigill_fn = sveeltperm_sigill, + }, + { + .name = "SVE F16MM", + .at_hwcap = AT_HWCAP, + .hwcap_bit = HWCAP_SVE_F16MM, + .cpuinfo = "svef16mm", + .sigill_fn = svef16mm_sigill, + }, { .name = "SVE2 B16B16", .at_hwcap = AT_HWCAP2, From 0f82983e4f86d0b4ee451537d07c814dcd6b4fab Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Mon, 3 Feb 2025 20:11:04 +0000 Subject: [PATCH 48/85] arm64/hwcap: Remove stray references to SF8MMx commit 21fed7c223e20e694b91dbf25936d922a50c8b19 upstream. Due to SME currently being disabled when removing the SF8MMx support it wasn't noticed that there were some stray references in the hwcap table, delete them. Fixes: 819935464cb2 ("arm64/hwcap: Describe 2024 dpISA extensions to userspace") Signed-off-by: Mark Brown Link: https://lore.kernel.org/r/20250203-arm64-remove-sf8mmx-v1-1-6f1da3dbff82@kernel.org Signed-off-by: Will Deacon [ bh: backport to v6.6 - retain target SME feature gating while removing the two upstream stray registrations. ] Signed-off-by: Bin Huang --- arch/arm64/kernel/cpufeature.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 27bef8db827e..de51b58acb68 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -3130,8 +3130,6 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SF8FMA, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_SF8FMA), HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SF8DP4, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_SF8DP4), HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SF8DP2, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_SF8DP2), - HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SF8MM8, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_SF8MM8), - HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SF8MM4, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_SF8MM4), HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SBitPerm, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_SBITPERM), HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, AES, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_AES), HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SFEXPA, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_SFEXPA), From 1de050243a760447cf338d3e004530a90d49f3d2 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Wed, 12 Feb 2025 00:30:42 +0000 Subject: [PATCH 49/85] arm64: Add missing registrations of hwcaps commit a4cc8494f1d853a0945d2a655b4891935d717355 upstream. Commit 819935464cb2 ("arm64/hwcap: Describe 2024 dpISA extensions to userspace") added definitions for HWCAP_FPRCVT, HWCAP_F8MM8 and HWCAP_F8MM4 but did not include the crucial registration in arm64_elf_hwcaps. Add it. Fixes: 819935464cb2 ("arm64/hwcap: Describe 2024 dpISA extensions to userspace") Reported-by: Mark Rutland Signed-off-by: Mark Brown Acked-by: Mark Rutland Link: https://lore.kernel.org/r/20250212-arm64-fix-2024-dpisa-v2-1-67a1c11d6001@kernel.org Signed-off-by: Will Deacon [ bh: backport to v6.6 - re-anchor registrations in the target HWCAP capability table. ] Signed-off-by: Bin Huang --- arch/arm64/kernel/cpufeature.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index de51b58acb68..e7dab7757acb 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -3065,6 +3065,7 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { HWCAP_CAP(ID_AA64ISAR1_EL1, LS64, LS64_V, CAP_HWCAP, KERNEL_HWCAP_LS64_V), HWCAP_CAP(ID_AA64ISAR2_EL1, LUT, IMP, CAP_HWCAP, KERNEL_HWCAP_LUT), HWCAP_CAP(ID_AA64ISAR3_EL1, FAMINMAX, IMP, CAP_HWCAP, KERNEL_HWCAP_FAMINMAX), + HWCAP_CAP(ID_AA64ISAR3_EL1, FPRCVT, IMP, CAP_HWCAP, KERNEL_HWCAP_FPRCVT), HWCAP_CAP(ID_AA64MMFR2_EL1, AT, IMP, CAP_HWCAP, KERNEL_HWCAP_USCAT), #ifdef CONFIG_ARM64_SVE HWCAP_CAP(ID_AA64PFR0_EL1, SVE, IMP, CAP_HWCAP, KERNEL_HWCAP_SVE), @@ -3140,6 +3141,8 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { HWCAP_CAP(ID_AA64FPFR0_EL1, F8FMA, IMP, CAP_HWCAP, KERNEL_HWCAP_F8FMA), HWCAP_CAP(ID_AA64FPFR0_EL1, F8DP4, IMP, CAP_HWCAP, KERNEL_HWCAP_F8DP4), HWCAP_CAP(ID_AA64FPFR0_EL1, F8DP2, IMP, CAP_HWCAP, KERNEL_HWCAP_F8DP2), + HWCAP_CAP(ID_AA64FPFR0_EL1, F8MM8, IMP, CAP_HWCAP, KERNEL_HWCAP_F8MM8), + HWCAP_CAP(ID_AA64FPFR0_EL1, F8MM4, IMP, CAP_HWCAP, KERNEL_HWCAP_F8MM4), HWCAP_CAP(ID_AA64FPFR0_EL1, F8E4M3, IMP, CAP_HWCAP, KERNEL_HWCAP_F8E4M3), HWCAP_CAP(ID_AA64FPFR0_EL1, F8E5M2, IMP, CAP_HWCAP, KERNEL_HWCAP_F8E5M2), {}, From bebb33b76f473f0117fda6d5e567b68582996e0c Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Fri, 20 Jun 2025 12:28:48 +0100 Subject: [PATCH 50/85] arm64: Filter out SME hwcaps when FEAT_SME isn't implemented commit a75ad2fc76a2ab70817c7eed3163b66ea84ca6ac upstream. We have a number of hwcaps for various SME subfeatures enumerated via ID_AA64SMFR0_EL1. Currently we advertise these without cross checking against the main SME feature, advertised in ID_AA64PFR1_EL1.SME which means that if the two are out of sync userspace can see a confusing situation where SME subfeatures are advertised without the base SME hwcap. This can be readily triggered by using the arm64.nosme override which only masks out ID_AA64PFR1_EL1.SME, and there have also been reports of VMMs which do the same thing. Fix this as we did previously for SVE in 064737920bdb ("arm64: Filter out SVE hwcaps when FEAT_SVE isn't implemented") by filtering out the SME subfeature hwcaps when FEAT_SME is not present. Fixes: 5e64b862c482 ("arm64/sme: Basic enumeration support") Reported-by: Yury Khrustalev Signed-off-by: Mark Brown Cc: stable@vger.kernel.org Link: https://lore.kernel.org/r/20250620-arm64-sme-filter-hwcaps-v1-1-02b9d3c2d8ef@kernel.org Signed-off-by: Will Deacon [ bh: Retain the SME filtering already present and correct its malformed I8I32 capability entry. ] Signed-off-by: Bin Huang --- arch/arm64/kernel/cpufeature.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index e7dab7757acb..6a119d5d1f35 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -3123,7 +3123,7 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, F16F16, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_F16F16), HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, F8F16, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_F8F16), HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, F8F32, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_F8F32), - HWCAP_CAP(ID_MATCH_ID(has_sme_feature, AA64SMFR0_EL1, I8I32, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_I8I32), + HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, I8I32, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_I8I32), HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, F16F32, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_F16F32), HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, B16F32, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_B16F32), HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, BI32I32, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_BI32I32), From ab49b2b5a93e17ada0ccc253ad90967bddf30462 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Thu, 17 Apr 2025 20:01:13 +0100 Subject: [PATCH 51/85] arm64/fpsimd: signal: Clear TPIDR2 when delivering signals commit b376108e1f88df9fbbb3867634150a3dd71e3acb upstream. Linux is intended to be compatible with userspace written to Arm's AAPCS64 procedure call standard [1,2]. For the Scalable Matrix Extension (SME), AAPCS64 was extended with a "ZA lazy saving scheme", where SME's ZA tile is lazily callee-saved and caller-restored. In this scheme, TPIDR2_EL0 indicates whether the ZA tile is live or has been saved by pointing to a "TPIDR2 block" in memory, which has a "za_save_buffer" pointer. This scheme has been implemented in GCC and LLVM, with necessary runtime support implemented in glibc. AAPCS64 does not specify how the ZA lazy saving scheme is expected to interact with signal handling, and the behaviour that AAPCS64 currently recommends for (sig)setjmp() and (sig)longjmp() does not always compose safely with signal handling, as explained below. When Linux delivers a signal, it creates signal frames which contain the original values of PSTATE.ZA, the ZA tile, and TPIDR_EL2. Between saving the original state and entering the signal handler, Linux clears PSTATE.ZA, but leaves TPIDR2_EL0 unchanged. Consequently a signal handler can be entered with PSTATE.ZA=0 (meaning accesses to ZA will trap), while TPIDR_EL0 is non-null (which may indicate that ZA needs to be lazily saved, depending on the contents of the TPIDR2 block). While in this state, libc and/or compiler runtime code, such as longjmp(), may attempt to save ZA. As PSTATE.ZA=0, these accesses will trap, causing the kernel to inject a SIGILL. Note that by virtue of lazy saving occurring in libc and/or C runtime code, this can be triggered by application/library code which is unaware of SME. To avoid the problem above, the kernel must ensure that signal handlers are entered with PSTATE.ZA and TPIDR2_EL0 configured in a manner which complies with the ZA lazy saving scheme. Practically speaking, the only choice is to enter signal handlers with PSTATE.ZA=0 and TPIDR2_EL0=NULL. This change should not impact SME code which does not follow the ZA lazy saving scheme (and hence does not use TPIDR2_EL0). An alternative approach that was considered is to have the signal handler inherit the original values of both PSTATE.ZA and TPIDR2_EL0, relying on lazy save/restore sequences being idempotent and capable of racing safely. This is not safe as signal handlers must be assumed to have a "private ZA" interface, and therefore cannot be entered with PSTATE.ZA=1 and TPIDR2_EL0=NULL, but it is legitimate for signals to be taken from this state. With the kernel fixed to clear TPIDR2_EL0, there are a couple of remaining issues (largely masked by the first issue) that must be fixed in userspace: (1) When a (sig)setjmp() + (sig)longjmp() pair cross a signal boundary, ZA state may be discarded when it needs to be preserved. Currently, the ZA lazy saving scheme recommends that setjmp() does not save ZA, and recommends that longjmp() is responsible for saving ZA. A call to longjmp() in a signal handler will not have visibility of ZA state that existed prior to entry to the signal, and when a longjmp() is used to bypass a usual signal return, unsaved ZA state will be discarded erroneously. To fix this, it is necessary for setjmp() to eagerly save ZA state, and for longjmp() to configure PSTATE.ZA=0 and TPIDR2_EL0=NULL. This works regardless of whether a signal boundary is crossed. (2) When a C++ exception is thrown and crosses a signal boundary before it is caught, ZA state may be discarded when it needs to be preserved. AAPCS64 requires that exception handlers are entered with PSTATE.{SM,ZA}={0,0} and TPIDR2_EL0=NULL, with exception unwind code expected to perform any necessary save of ZA state. Where it is necessary to perform an exception unwind across an exception boundary, the unwind code must recover any necessary ZA state (along with TPIDR2) from signal frames. Fix the kernel as described above, with setup_return() clearing TPIDR2_EL0 when delivering a signal. Folk on CC are working on fixes for the remaining userspace issues, including updates/fixes to the AAPCS64 specification and glibc. [1] https://github.com/ARM-software/abi-aa/releases/download/2025Q1/aapcs64.pdf [2] https://github.com/ARM-software/abi-aa/blob/c51addc3dc03e73a016a1e4edf25440bcac76431/aapcs64/aapcs64.rst Fixes: 39782210eb7e ("arm64/sme: Implement ZA signal handling") Fixes: 39e54499280f ("arm64/signal: Include TPIDR2 in the signal context") Signed-off-by: Mark Rutland Cc: Daniel Kiss Cc: Marc Zyngier Cc: Mark Brown Cc: Richard Sandiford Cc: Sander De Smalen Cc: Tamas Petz Cc: Will Deacon Cc: Yury Khrustalev Link: https://lore.kernel.org/r/20250417190113.3778111-1-mark.rutland@arm.com Signed-off-by: Catalin Marinas [ bh: Adjust setup_return context to the existing signal delivery implementation. ] Signed-off-by: Bin Huang --- Documentation/arch/arm64/sme.rst | 2 +- arch/arm64/kernel/signal.c | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Documentation/arch/arm64/sme.rst b/Documentation/arch/arm64/sme.rst index 3d0e53ecac4f..316c7b4e6fb7 100644 --- a/Documentation/arch/arm64/sme.rst +++ b/Documentation/arch/arm64/sme.rst @@ -115,7 +115,7 @@ be zeroed. 5. Signal handling ------------------- -* Signal handlers are invoked with streaming mode and ZA disabled. +* Signal handlers are invoked with PSTATE.SM=0, PSTATE.ZA=0, and TPIDR2_EL0=0. * A new signal frame record TPIDR2_MAGIC is added formatted as a struct tpidr2_context to allow access to TPIDR2_EL0 from signal handlers. diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c index 14142c234918..452e0a9a4056 100644 --- a/arch/arm64/kernel/signal.c +++ b/arch/arm64/kernel/signal.c @@ -1161,6 +1161,7 @@ static void setup_return(struct pt_regs *regs, struct k_sigaction *ka, } current->thread.svcr &= ~(SVCR_ZA_MASK | SVCR_SM_MASK); + write_sysreg_s(0, SYS_TPIDR2_EL0); } if (ka->sa.sa_flags & SA_RESTORER) From 7e68a34c319fe760083a3a0c2ad307e4c93d7d80 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Thu, 8 May 2025 14:26:22 +0100 Subject: [PATCH 52/85] arm64/fpsimd: signal: Clear PSTATE.SM when restoring FPSIMD frame only commit 1bf663a86a4505a97f08d06c373704de9441c891 upstream. On systems with SVE and/or SME, the kernel will always create SVE and FPSIMD signal frames when delivering a signal, but a user can manipulate signal frames such that a signal return only observes an FPSIMD signal frame. When this happens, restore_fpsimd_context() will restore state such that fp_type==FP_STATE_FPSIMD, but will leave PSTATE.SM as-is. It is possible for a user to set PSTATE.SM between syscall entry and execution of the sigreturn logic (e.g. via ptrace), and consequently the sigreturn may result in the task having PSTATE.SM==1 and fp_type==FP_STATE_FPSIMD. For various reasons it is not legitimate for a task to be in a state where PSTATE.SM==1 and fp_type==FP_STATE_FPSIMD. Portions of the user ABI are written with the requirement that streaming SVE state is always presented in SVE format rather than FPSIMD format, and as there is no mechanism to permit access to only the FPSIMD subset of streaming SVE state, streaming SVE state must always be saved and restored in SVE format. Fix restore_fpsimd_context() to clear PSTATE.SM when restoring an FPSIMD signal frame without an SVE signal frame. This matches the current behaviour when an SVE signal frame is present, but the SVE signal frame has no register payload (e.g. as is the case on SME-only systems which lack SVE). This change should have no effect for applications which do not alter signal frames (i.e. almost all applications). I do not expect non-{malicious,buggy} applications to hide the SVE signal frame, but I've chosen to clear PSTATE.SM rather than mandating the presence of an SVE signal frame in case there is some legacy (non-SME) usage that I am not currently aware of. For context, the SME handling was originally introduced in commit: 85ed24dad290 ("arm64/sme: Implement streaming SVE signal handling") ... and subsequently updated/fixed to handle SME-only systems in commits: 7dde62f0687c ("arm64/signal: Always accept SVE signal frames on SME only systems") f26cd7372160 ("arm64/signal: Always allocate SVE signal frames on SME only systems") Fixes: 85ed24dad290 ("arm64/sme: Implement streaming SVE signal handling") Signed-off-by: Mark Rutland Cc: Catalin Marinas Cc: Marc Zyngier Cc: Mark Brown Cc: Will Deacon Link: https://lore.kernel.org/r/20250508132644.1395904-3-mark.rutland@arm.com Signed-off-by: Will Deacon [ bh: Adjust context for the FPSIMD context reader already present in the target kernel. ] Signed-off-by: Bin Huang --- arch/arm64/kernel/signal.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c index 452e0a9a4056..1dea0d522ac6 100644 --- a/arch/arm64/kernel/signal.c +++ b/arch/arm64/kernel/signal.c @@ -236,6 +236,7 @@ static int restore_fpsimd_context(struct user_ctxs *user) return err; clear_thread_flag(TIF_SVE); + current->thread.svcr &= ~SVCR_SM_MASK; current->thread.fp_type = FP_STATE_FPSIMD; /* load the hardware registers from the fpsimd_state structure */ From a5d04bed14cbd15fde4f608a0a10bf49c2ea6aaf Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Thu, 8 May 2025 14:26:25 +0100 Subject: [PATCH 53/85] arm64/fpsimd: ptrace: Consistently handle partial writes to NT_ARM_(S)SVE commit 316283f276ebd5596d2015e4653198bdc2e138d4 upstream. Partial writes to the NT_ARM_SVE and NT_ARM_SSVE regsets using an payload are handled inconsistently and non-deterministically. A comment within sve_set_common() indicates that we intended that a partial write would preserve any effective FPSIMD/SVE state which was not overwritten, but this has never worked consistently, and during syscalls the FPSIMD vector state may be non-deterministically preserved and may be erroneously migrated between streaming and non-streaming SVE modes. The simplest fix is to handle a partial write by consistently zeroing the remaining state. As detailed below I do not believe this will adversely affect any real usage. Neither GDB nor LLDB attempt partial writes to these regsets, and the documentation (in Documentation/arch/arm64/sve.rst) has always indicated that state preservation was not guaranteed, as is says: | The effect of writing a partial, incomplete payload is unspecified. When the logic was originally introduced in commit: 43d4da2c45b2 ("arm64/sve: ptrace and ELF coredump support") ... there were two potential behaviours, depending on TIF_SVE: * When TIF_SVE was clear, all SVE state would be zeroed, excluding the low 128 bits of vectors shared with FPSIMD, FPSR, and FPCR. * When TIF_SVE was set, all SVE state would be zeroed, including the low 128 bits of vectors shared with FPSIMD, but excluding FPSR and FPCR. Note that as writing to NT_ARM_SVE would set TIF_SVE, partial writes to NT_ARM_SVE would not be idempotent, and if a first write preserved the low 128 bits, a subsequent (potentially identical) partial write would discard the low 128 bits. When support for the NT_ARM_SSVE regset was added in commit: e12310a0d30f ("arm64/sme: Implement ptrace support for streaming mode SVE registers") ... the above behaviour was retained for writes to the NT_ARM_SVE regset, though writes to the NT_ARM_SSVE would always zero the SVE registers and would not inherit FPSIMD register state. This happened as fpsimd_sync_to_sve() only copied the FPSIMD regs when TIF_SVE was clear and PSTATE.SM==0. Subsequently, when FPSIMD/SVE state tracking was changed across commits: baa8515281b3 ("arm64/fpsimd: Track the saved FPSIMD state type separately to TIF_SVE") a0136be443d5 (arm64/fpsimd: Load FP state based on recorded data type") bbc6172eefdb ("arm64/fpsimd: SME no longer requires SVE register state") 8c845e273104 ("arm64/sve: Leave SVE enabled on syscall if we don't context switch") ... there was no corresponding update to the ptrace code, nor to fpsimd_sync_to_sve(), which stil considers TIF_SVE and PSTATE.SM rather than the saved fp_type. The saved state can be in the FPSIMD format regardless of whether TIF_SVE is set or clear, and the saved type can change non-deterministically during syscalls. Consequently a subsequent partial write to the NT_ARM_SVE or NT_ARM_SSVE regsets may non-deterministically preserve the FPSIMD state, and may migrate this state between streaming and non-streaming modes. Clean this up by never attempting to preserve ANY state when writing an SVE payload to the NT_ARM_SVE/NT_ARM_SSVE regsets, zeroing all relevant state including FPSR and FPCR. This simplifies the code, makes the behaviour deterministic, and avoids migrating state between streaming and non-streaming modes. As above, I do not believe this should adversely affect existing userspace applications. At the same time, remove fpsimd_sync_to_sve(). It is no longer used, doesn't do what its documentation implies, and gets in the way of other cleanups and fixes. Fixes: 43d4da2c45b2 ("arm64/sve: ptrace and ELF coredump support") Signed-off-by: Mark Rutland Cc: Catalin Marinas Cc: David Spickett Cc: Luis Machado Cc: Marc Zyngier Cc: Mark Brown Cc: Will Deacon Link: https://lore.kernel.org/r/20250508132644.1395904-6-mark.rutland@arm.com Signed-off-by: Will Deacon [ bh: Retain the vendor force-sync helper while deleting the obsolete conditional helper. ] Signed-off-by: Bin Huang --- arch/arm64/include/asm/fpsimd.h | 1 - arch/arm64/kernel/fpsimd.c | 15 --------------- arch/arm64/kernel/ptrace.c | 26 +++++++++----------------- 3 files changed, 9 insertions(+), 33 deletions(-) diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h index ad80cf8de27a..37c2dc992c42 100644 --- a/arch/arm64/include/asm/fpsimd.h +++ b/arch/arm64/include/asm/fpsimd.h @@ -204,7 +204,6 @@ struct vl_info { extern void sve_alloc(struct task_struct *task, bool flush); extern void fpsimd_release_task(struct task_struct *task); -extern void fpsimd_sync_to_sve(struct task_struct *task); extern void fpsimd_force_sync_to_sve(struct task_struct *task); extern void sve_sync_to_fpsimd(struct task_struct *task); extern void sve_sync_from_fpsimd_zeropad(struct task_struct *task); diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c index 2d2e48ebe189..356d9cde0a17 100644 --- a/arch/arm64/kernel/fpsimd.c +++ b/arch/arm64/kernel/fpsimd.c @@ -777,21 +777,6 @@ void fpsimd_force_sync_to_sve(struct task_struct *task) fpsimd_to_sve(task); } -/* - * Ensure that task->thread.sve_state is up to date with respect to - * the user task, irrespective of when SVE is in use or not. - * - * This should only be called by ptrace. task must be non-runnable. - * task->thread.sve_state must point to at least sve_state_size(task) - * bytes of allocated kernel memory. - */ -void fpsimd_sync_to_sve(struct task_struct *task) -{ - if (!test_tsk_thread_flag(task, TIF_SVE) && - !thread_sm_enabled(&task->thread)) - fpsimd_to_sve(task); -} - /* * Ensure that task->thread.uw.fpsimd_state is up to date with respect to * the user task, irrespective of whether SVE is in use or not. diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c index 3d9f4b05bfce..ce928f55b995 100644 --- a/arch/arm64/kernel/ptrace.c +++ b/arch/arm64/kernel/ptrace.c @@ -905,8 +905,6 @@ static int sve_set_common(struct task_struct *target, /* Enter/exit streaming mode */ if (system_supports_sme()) { - u64 old_svcr = target->thread.svcr; - switch (type) { case ARM64_VEC_SVE: target->thread.svcr &= ~SVCR_SM_MASK; @@ -926,23 +924,20 @@ static int sve_set_common(struct task_struct *target, ret = -EINVAL; goto out; } - - /* - * If we switched then invalidate any existing SVE - * state and ensure there's storage. - */ - if (target->thread.svcr != old_svcr) - sve_alloc(target, true); } + /* Always zero V regs, FPSR, and FPCR */ + memset(¤t->thread.uw.fpsimd_state, 0, + sizeof(current->thread.uw.fpsimd_state)); + /* Registers: FPSIMD-only case */ BUILD_BUG_ON(SVE_PT_FPSIMD_OFFSET != sizeof(header)); if ((header.flags & SVE_PT_REGS_MASK) == SVE_PT_REGS_FPSIMD) { - ret = __fpr_set(target, regset, pos, count, kbuf, ubuf, - SVE_PT_FPSIMD_OFFSET); clear_tsk_thread_flag(target, TIF_SVE); target->thread.fp_type = FP_STATE_FPSIMD; + ret = __fpr_set(target, regset, pos, count, kbuf, ubuf, + SVE_PT_FPSIMD_OFFSET); goto out; } @@ -961,6 +956,7 @@ static int sve_set_common(struct task_struct *target, goto out; } + /* Always zero SVE state */ sve_alloc(target, true); if (!target->thread.sve_state) { ret = -ENOMEM; @@ -970,13 +966,9 @@ static int sve_set_common(struct task_struct *target, } /* - * Ensure target->thread.sve_state is up to date with target's - * FPSIMD regs, so that a short copyin leaves trailing - * registers unmodified. Only enable SVE if we are - * configuring normal SVE, a system with streaming SVE may not - * have normal SVE. + * Only enable SVE if we are configuring normal SVE, a system with + * streaming SVE may not have normal SVE. */ - fpsimd_sync_to_sve(target); if (type == ARM64_VEC_SVE) set_tsk_thread_flag(target, TIF_SVE); target->thread.fp_type = FP_STATE_SVE; From 010abc91e7fcc3a3a6ee45194c1053d494d3de55 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Thu, 8 May 2025 14:26:26 +0100 Subject: [PATCH 54/85] arm64/fpsimd: Clarify sve_sync_*() functions commit b255be426913e83dd79b920b77e94d3c47886c50 upstream. The sve_sync_{to,from}_fpsimd*() functions are intended to extract/insert the currently effective FPSIMD state of a task regardless of whether the task's state is saved in FPSIMD format or SVE format. Historically they were only used by ptrace, but sve_sync_to_fpsimd() is now used more widely, and sve_sync_from_fpsimd_zeropad() may be used more widely in future. When FPSIMD/SVE state tracking was changed across commits: baa8515281b3 ("arm64/fpsimd: Track the saved FPSIMD state type separately to TIF_SVE") a0136be443d5 (arm64/fpsimd: Load FP state based on recorded data type") bbc6172eefdb ("arm64/fpsimd: SME no longer requires SVE register state") 8c845e273104 ("arm64/sve: Leave SVE enabled on syscall if we don't context switch") ... sve_sync_to_fpsimd() was updated to consider task->thread.fp_type rather than the task's TIF_SVE and PSTATE.SM, but (apparently due to an oversight) sve_sync_from_fpsimd_zeropad() was left as-is, leaving the two inconsistent. Due to this, sve_sync_from_fpsimd_zeropad() may copy state from task->thread.uw.fpsimd_state into task->thread.sve_state when task->thread.fp_type == FP_STATE_FPSIMD. This is redundant (but benign) as task->thread.uw.fpsimd_state is the effective state that will be restored, and task->thread.sve_state will not be consumed. For consistency, and to avoid the redundant work, it better for sve_sync_from_fpsimd_zeropad() to consider task->thread.fp_type alone, matching sve_sync_to_fpsimd(). The naming of both functions is somehat unfortunate, as it is unclear when and why they copy state. It would be better to describe them in terms of the effective state. Considering all of the above, clean this up: * Adjust sve_sync_from_fpsimd_zeropad() to consider task->thread.fp_type. * Update comments to clarify the intended semantics/usage. I've removed the description that task->thread.sve_state must have been allocated, as this is only necessary when task->thread.fp_type == FP_STATE_SVE, which itself implies that task->thread.sve_state must have been allocated. * Rename the functions to more clearly indicate when/why they copy state: - sve_sync_to_fpsimd() => fpsimd_sync_from_effective_state() - sve_sync_from_fpsimd_zeropad => fpsimd_sync_to_effective_state_zeropad() Signed-off-by: Mark Rutland Cc: Catalin Marinas Cc: Marc Zyngier Cc: Mark Brown Cc: Will Deacon Link: https://lore.kernel.org/r/20250508132644.1395904-7-mark.rutland@arm.com Signed-off-by: Will Deacon [ bh: Retain the vendor force-sync helper and adapt the surrounding declarations. ] Signed-off-by: Bin Huang --- arch/arm64/include/asm/fpsimd.h | 8 ++++---- arch/arm64/kernel/fpsimd.c | 30 ++++++++++++------------------ arch/arm64/kernel/ptrace.c | 6 +++--- arch/arm64/kernel/signal.c | 2 +- 4 files changed, 20 insertions(+), 26 deletions(-) diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h index 37c2dc992c42..985df68edb81 100644 --- a/arch/arm64/include/asm/fpsimd.h +++ b/arch/arm64/include/asm/fpsimd.h @@ -205,8 +205,8 @@ struct vl_info { extern void sve_alloc(struct task_struct *task, bool flush); extern void fpsimd_release_task(struct task_struct *task); extern void fpsimd_force_sync_to_sve(struct task_struct *task); -extern void sve_sync_to_fpsimd(struct task_struct *task); -extern void sve_sync_from_fpsimd_zeropad(struct task_struct *task); +extern void fpsimd_sync_from_effective_state(struct task_struct *task); +extern void fpsimd_sync_to_effective_state_zeropad(struct task_struct *task); extern int vec_set_vector_length(struct task_struct *task, enum vec_type type, unsigned long vl, unsigned long flags); @@ -306,8 +306,8 @@ size_t sve_state_size(struct task_struct const *task); static inline void sve_alloc(struct task_struct *task, bool flush) { } static inline void fpsimd_release_task(struct task_struct *task) { } -static inline void sve_sync_to_fpsimd(struct task_struct *task) { } -static inline void sve_sync_from_fpsimd_zeropad(struct task_struct *task) { } +static inline void fpsimd_sync_from_effective_state(struct task_struct *task) { } +static inline void fpsimd_sync_to_effective_state_zeropad(struct task_struct *task) { } static inline int sve_max_virtualisable_vl(void) { diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c index 356d9cde0a17..f7381de5641c 100644 --- a/arch/arm64/kernel/fpsimd.c +++ b/arch/arm64/kernel/fpsimd.c @@ -778,39 +778,33 @@ void fpsimd_force_sync_to_sve(struct task_struct *task) } /* - * Ensure that task->thread.uw.fpsimd_state is up to date with respect to - * the user task, irrespective of whether SVE is in use or not. + * Ensure that task->thread.uw.fpsimd_state is up to date with respect to the + * task's currently effective FPSIMD/SVE state. * - * This should only be called by ptrace. task must be non-runnable. - * task->thread.sve_state must point to at least sve_state_size(task) - * bytes of allocated kernel memory. + * The task's FPSIMD/SVE/SME state must not be subject to concurrent + * manipulation. */ -void sve_sync_to_fpsimd(struct task_struct *task) +void fpsimd_sync_from_effective_state(struct task_struct *task) { if (task->thread.fp_type == FP_STATE_SVE) sve_to_fpsimd(task); } /* - * Ensure that task->thread.sve_state is up to date with respect to - * the task->thread.uw.fpsimd_state. + * Ensure that the task's currently effective FPSIMD/SVE state is up to date + * with respect to task->thread.uw.fpsimd_state, zeroing any effective + * non-FPSIMD (S)SVE state. * - * This should only be called by ptrace to merge new FPSIMD register - * values into a task for which SVE is currently active. - * task must be non-runnable. - * task->thread.sve_state must point to at least sve_state_size(task) - * bytes of allocated kernel memory. - * task->thread.uw.fpsimd_state must already have been initialised with - * the new FPSIMD register values to be merged in. + * The task's FPSIMD/SVE/SME state must not be subject to concurrent + * manipulation. */ -void sve_sync_from_fpsimd_zeropad(struct task_struct *task) +void fpsimd_sync_to_effective_state_zeropad(struct task_struct *task) { unsigned int vq; void *sst = task->thread.sve_state; struct user_fpsimd_state const *fst = &task->thread.uw.fpsimd_state; - if (!test_tsk_thread_flag(task, TIF_SVE) && - !thread_sm_enabled(&task->thread)) + if (task->thread.fp_type != FP_STATE_SVE) return; vq = sve_vq_from_vl(thread_get_cur_vl(&task->thread)); diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c index ce928f55b995..6e3badd42049 100644 --- a/arch/arm64/kernel/ptrace.c +++ b/arch/arm64/kernel/ptrace.c @@ -593,7 +593,7 @@ static int __fpr_get(struct task_struct *target, { struct user_fpsimd_state *uregs; - sve_sync_to_fpsimd(target); + fpsimd_sync_from_effective_state(target); uregs = &target->thread.uw.fpsimd_state; @@ -625,7 +625,7 @@ static int __fpr_set(struct task_struct *target, * Ensure target->thread.uw.fpsimd_state is up to date, so that a * short copyin can't resurrect stale data. */ - sve_sync_to_fpsimd(target); + fpsimd_sync_from_effective_state(target); newstate = target->thread.uw.fpsimd_state; @@ -652,7 +652,7 @@ static int fpr_set(struct task_struct *target, const struct user_regset *regset, if (ret) return ret; - sve_sync_from_fpsimd_zeropad(target); + fpsimd_sync_to_effective_state_zeropad(target); fpsimd_flush_task_state(target); return ret; diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c index 1dea0d522ac6..4e011d1c5ca7 100644 --- a/arch/arm64/kernel/signal.c +++ b/arch/arm64/kernel/signal.c @@ -194,7 +194,7 @@ static int preserve_fpsimd_context(struct fpsimd_context __user *ctx) ¤t->thread.uw.fpsimd_state; int err; - sve_sync_to_fpsimd(current); + fpsimd_sync_from_effective_state(current); /* copy the FP and status/control registers */ err = __copy_to_user(ctx->vregs, fpsimd->vregs, sizeof(fpsimd->vregs)); From 0f615f7a00e31afe25a2d48b605908bced4f8834 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Thu, 8 May 2025 14:26:27 +0100 Subject: [PATCH 55/85] arm64/fpsimd: Factor out {sve,sme}_state_size() helpers commit 8738288a08b8367cd2a9f656498d7490e4473036 upstream. In subsequent patches we'll need to determine the SVE/SME state size for a given SVE VL and SME VL regardless of whether a task is currently configured with those VLs. Split the sizing logic out of sve_state_size() and sme_state_size() so that we don't need to open-code this logic elsewhere. At the same time, apply minor cleanups: * Move sve_state_size() into fpsimd.h, matching the placement of sme_state_size(). * Remove the feature checks from sve_state_size(). We only call sve_state_size() when at least one of SVE and SME are supported, and when either of the two is not supported, the task's corresponding SVE/SME vector length will be zero. Signed-off-by: Mark Rutland Cc: Catalin Marinas Cc: Marc Zyngier Cc: Mark Brown Cc: Will Deacon Link: https://lore.kernel.org/r/20250508132644.1395904-8-mark.rutland@arm.com Signed-off-by: Will Deacon Signed-off-by: Bin Huang --- arch/arm64/include/asm/fpsimd.h | 47 ++++++++++++++++++++++++++------- arch/arm64/kernel/fpsimd.c | 16 ----------- 2 files changed, 37 insertions(+), 26 deletions(-) diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h index 985df68edb81..ab3a61f2cad5 100644 --- a/arch/arm64/include/asm/fpsimd.h +++ b/arch/arm64/include/asm/fpsimd.h @@ -300,7 +300,22 @@ static inline bool sve_vq_available(unsigned int vq) return vq_available(ARM64_VEC_SVE, vq); } -size_t sve_state_size(struct task_struct const *task); +static inline size_t __sve_state_size(unsigned int sve_vl, unsigned int sme_vl) +{ + unsigned int vl = max(sve_vl, sme_vl); + return SVE_SIG_REGS_SIZE(sve_vq_from_vl(vl)); +} + +/* + * Return how many bytes of memory are required to store the full SVE + * state for task, given task's currently configured vector length. + */ +static inline size_t sve_state_size(struct task_struct const *task) +{ + unsigned int sve_vl = task_get_sve_vl(task); + unsigned int sme_vl = task_get_sme_vl(task); + return __sve_state_size(sve_vl, sme_vl); +} #else /* ! CONFIG_ARM64_SVE */ @@ -341,6 +356,11 @@ static inline void vec_update_vq_map(enum vec_type t) { } static inline int vec_verify_vq_map(enum vec_type t) { return 0; } static inline void sve_setup(void) { } +static inline size_t __sve_state_size(unsigned int sve_vl, unsigned int sme_vl) +{ + return 0; +} + static inline size_t sve_state_size(struct task_struct const *task) { return 0; @@ -393,6 +413,16 @@ extern int sme_set_current_vl(unsigned long arg); extern int sme_get_current_vl(void); extern void sme_suspend_exit(void); +static inline size_t __sme_state_size(unsigned int sme_vl) +{ + size_t size = ZA_SIG_REGS_SIZE(sve_vq_from_vl(sme_vl)); + + if (system_supports_sme2()) + size += ZT_SIG_REG_SIZE; + + return size; +} + /* * Return how many bytes of memory are required to store the full SME * specific state for task, given task's currently configured vector @@ -400,15 +430,7 @@ extern void sme_suspend_exit(void); */ static inline size_t sme_state_size(struct task_struct const *task) { - unsigned int vl = task_get_sme_vl(task); - size_t size; - - size = ZA_SIG_REGS_SIZE(sve_vq_from_vl(vl)); - - if (system_supports_sme2()) - size += ZT_SIG_REG_SIZE; - - return size; + return __sme_state_size(task_get_sme_vl(task)); } #else @@ -429,6 +451,11 @@ static inline int sme_set_current_vl(unsigned long arg) { return -EINVAL; } static inline int sme_get_current_vl(void) { return -EINVAL; } static inline void sme_suspend_exit(void) { } +static inline size_t __sme_state_size(unsigned int sme_vl) +{ + return 0; +} + static inline size_t sme_state_size(struct task_struct const *task) { return 0; diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c index f7381de5641c..c565df23589b 100644 --- a/arch/arm64/kernel/fpsimd.c +++ b/arch/arm64/kernel/fpsimd.c @@ -723,22 +723,6 @@ static void sve_free(struct task_struct *task) __sve_free(task); } -/* - * Return how many bytes of memory are required to store the full SVE - * state for task, given task's currently configured vector length. - */ -size_t sve_state_size(struct task_struct const *task) -{ - unsigned int vl = 0; - - if (system_supports_sve()) - vl = task_get_sve_vl(task); - if (system_supports_sme()) - vl = max(vl, task_get_sme_vl(task)); - - return SVE_SIG_REGS_SIZE(sve_vq_from_vl(vl)); -} - /* * Ensure that task->thread.sve_state is allocated and sufficiently large. * From 0f85bcfb28c6b0d3b0a17d6e0cf762feab00a21a Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Thu, 8 May 2025 14:26:28 +0100 Subject: [PATCH 56/85] arm64/fpsimd: Add task_smstop_sm() commit 6ef1d778ce56c5bde1ac0a1f85fd9710efde6724 upstream. In a few places we want to transition a task from streaming mode to non-streaming mode, e.g. signal delivery where we historically tried to use an SMSTOP SM instruction. Add a new helper to manipulate a task's state in the same way as an SMSTOP SM instruction. I have not added a corresponding helper to simulate the effects of SMSTART SM. Only ptrace transitions a task into streaming mode, and ptrace has distinct semantics for such transitions. Per ARM DDI 0487 L.a, section B1.4.6: | RRSWFQ | When the Effective value of PSTATE.SM is changed by any method from 0 | to 1, an entry to Streaming SVE mode is performed, and all implemented | bits of Streaming SVE register state are set to zero. | RKFRQZ | When the Effective value of PSTATE.SM is changed by any method from 1 | to 0, an exit from Streaming SVE mode is performed, and in the | newly-entered mode, all implemented bits of the SVE scalable vector | registers, SVE predicate registers, and FFR, are set to zero. Per ARM DDI 0487 L.a, section C5.2.9: | On entry to or exit from Streaming SVE mode, FPMR is set to 0 Per ARM DDI 0487 L.a, section C5.2.10: | On entry to or exit from Streaming SVE mode, FPSR.{IOC, DZC, OFC, UFC, | IXC, IDC, QC} are set to 1 and the remaining bits are set to 0. This means bits 0, 1, 2, 3, 4, 7, and 27 respectively, i.e. 0x0800009f Signed-off-by: Mark Rutland Cc: Catalin Marinas Cc: Marc Zyngier Cc: Mark Brown Cc: Will Deacon Link: https://lore.kernel.org/r/20250508132644.1395904-9-mark.rutland@arm.com Signed-off-by: Will Deacon Signed-off-by: Bin Huang --- arch/arm64/include/asm/fpsimd.h | 2 ++ arch/arm64/kernel/fpsimd.c | 22 ++++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h index ab3a61f2cad5..383462315489 100644 --- a/arch/arm64/include/asm/fpsimd.h +++ b/arch/arm64/include/asm/fpsimd.h @@ -114,6 +114,8 @@ static inline bool thread_za_enabled(struct thread_struct *thread) return system_supports_sme() && (thread->svcr & SVCR_ZA_MASK); } +extern void task_smstop_sm(struct task_struct *task); + /* Maximum VL that SVE/SME VL-agnostic software can transparently support */ #define VL_ARCH_MAX 0x100 diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c index c565df23589b..cb0395cfa19e 100644 --- a/arch/arm64/kernel/fpsimd.c +++ b/arch/arm64/kernel/fpsimd.c @@ -699,6 +699,28 @@ static inline void sve_to_fpsimd(struct task_struct *task) } } +static inline void __fpsimd_zero_vregs(struct user_fpsimd_state *fpsimd) +{ + memset(&fpsimd->vregs, 0, sizeof(fpsimd->vregs)); +} + +/* + * Simulate the effects of an SMSTOP SM instruction. + */ +void task_smstop_sm(struct task_struct *task) +{ + if (!thread_sm_enabled(&task->thread)) + return; + + __fpsimd_zero_vregs(&task->thread.uw.fpsimd_state); + task->thread.uw.fpsimd_state.fpsr = 0x0800009f; + if (system_supports_fpmr()) + task->thread.uw.fpmr = 0; + + task->thread.svcr &= ~SVCR_SM_MASK; + task->thread.fp_type = FP_STATE_FPSIMD; +} + void cpu_enable_fpmr(const struct arm64_cpu_capabilities *__always_unused p) { write_sysreg_s(read_sysreg_s(SYS_SCTLR_EL1) | SCTLR_EL1_EnFPM_MASK, From abcc944a90ee86a99e4bdc29c23744f1823d0c9b Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Thu, 8 May 2025 14:26:29 +0100 Subject: [PATCH 57/85] arm64/fpsimd: signal: Use SMSTOP behaviour in setup_return() commit 99560c9452bb9819334bdb9c6e9e27c0f45f8829 upstream. Historically the behaviour of setup_return() was nondeterministic, depending on whether the task's FSIMD/SVE/SME state happened to be live. We fixed most of that in commit: 929fa99b1215 ("arm64/fpsimd: signal: Always save+flush state early") ... but we didn't decide on how clearing PSTATE.SM should behave, and left a TODO comment to that effect. Use the new task_smstop_sm() helper to make this behave as if an SMSTOP instruction was used to exit streaming mode. This would have been the most common behaviour prior to the commit above. Fixes: 40a8e87bb328 ("arm64/sme: Disable ZA and streaming mode when handling signals") Signed-off-by: Mark Rutland Cc: Catalin Marinas Cc: Marc Zyngier Cc: Mark Brown Cc: Will Deacon Link: https://lore.kernel.org/r/20250508132644.1395904-10-mark.rutland@arm.com Signed-off-by: Will Deacon Signed-off-by: Bin Huang --- arch/arm64/kernel/signal.c | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c index 4e011d1c5ca7..724bd2fd75f7 100644 --- a/arch/arm64/kernel/signal.c +++ b/arch/arm64/kernel/signal.c @@ -1146,22 +1146,8 @@ static void setup_return(struct pt_regs *regs, struct k_sigaction *ka, /* Signal handlers are invoked with ZA and streaming mode disabled */ if (system_supports_sme()) { - /* - * If we were in streaming mode the saved register - * state was SVE but we will exit SM and use the - * FPSIMD register state. - * - * TODO: decide if this should behave as SMSTOP (e.g. reset - * FPSR + FPMR), or whether this should only clear the scalable - * registers + ZA state. - */ - if (current->thread.svcr & SVCR_SM_MASK) { - memset(¤t->thread.uw.fpsimd_state, 0, - sizeof(current->thread.uw.fpsimd_state)); - current->thread.fp_type = FP_STATE_FPSIMD; - } - - current->thread.svcr &= ~(SVCR_ZA_MASK | SVCR_SM_MASK); + task_smstop_sm(current); + current->thread.svcr &= ~SVCR_ZA_MASK; write_sysreg_s(0, SYS_TPIDR2_EL0); } From 705b6a782fbf877c227670f712554bf935583b6e Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Thu, 8 May 2025 14:26:30 +0100 Subject: [PATCH 58/85] arm64/fpsimd: Remove redundant task->mm check commit 8d61eef756798cd33721e4c89bc72ce81792a3e8 upstream. For historical reasons, arch_dup_task_struct() only calls fpsimd_preserve_current_state() when current->mm is non-NULL, but this is no longer necessary. Historically TIF_FOREIGN_FPSTATE was only managed for user threads, and was never set for kernel threads. At the time, various functions attempted to avoid saving/restoring state for kernel threads by checking task_struct::mm to try to distinguish user threads from kernel threads. We added the current->mm check to arch_dup_task_struct() in commit: 6eb6c80187c5 ("arm64: kernel thread don't need to save fpsimd context.") ... where the intent was to avoid pointlessly saving state for kernel threads, which never had live state (and the saved state would never be consumed). Subsequently we began setting TIF_FOREIGN_FPSTATE for kernel threads, and removed most of the task_struct::mm checks in commit: df3fb9682045 ("arm64: fpsimd: Eliminate task->mm checks") ... but we missed the check in arch_dup_task_struct(), which is similarly redundant. Remove the redundant check from arch_dup_task_struct(). Signed-off-by: Mark Rutland Cc: Catalin Marinas Cc: Marc Zyngier Cc: Mark Brown Cc: Will Deacon Link: https://lore.kernel.org/r/20250508132644.1395904-11-mark.rutland@arm.com Signed-off-by: Will Deacon [ bh: Adjust arch_dup_task_struct context to the existing task layout. ] Signed-off-by: Bin Huang --- arch/arm64/kernel/process.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index 22ef633fe568..e2a100fdb7dc 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -286,8 +286,7 @@ void arch_release_task_struct(struct task_struct *tsk) int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) { - if (current->mm) - fpsimd_preserve_current_state(); + fpsimd_preserve_current_state(); *dst = *src; /* We rely on the above assignment to initialize dst's thread_flags: */ From 7c14197b73a3ecc295b583e5eefe79c8760ca962 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Thu, 8 May 2025 14:26:31 +0100 Subject: [PATCH 59/85] arm64/fpsimd: Consistently preserve FPSIMD state during clone() commit e0cb0f26594c644c71ee7f48ebaae6b26bf56a12 upstream. In arch_dup_task_struct() we try to ensure that the child task inherits the FPSIMD state of its parent, but this depends on the parent task's saved state being in FPSIMD format, which is not always the case. Consequently the child task may inherit stale FPSIMD state in some cases. This can happen when the parent's state has been modified by ptrace since syscall entry, as writes to the NT_ARM_SVE regset may save state in SVE format. This has been possible since commit: bc0ee4760364 ("arm64/sve: Core task context handling") More recently it has been possible for a task's FPSIMD/SVE state to be saved before lazy discarding was guaranteed to occur, in which case preemption could cause the effective FPSIMD state to be saved in SVE format non-deterministically. This has been possible since commit: f130ac0ae441 ("arm64: syscall: unmask DAIF earlier for SVCs") Fix this by saving the parent task's effective FPSIMD state into FPSIMD format before copying the task_struct. As this requires modifying the parent's fpsimd_state, we must save+flush the state to avoid racing with concurrent manipulation. Similar issues exist when the parent has streaming mode state, and will be addressed by subsequent patches. Fixes: bc0ee4760364 ("arm64/sve: Core task context handling") Fixes: f130ac0ae441 ("arm64: syscall: unmask DAIF earlier for SVCs") Signed-off-by: Mark Rutland Cc: Catalin Marinas Cc: Marc Zyngier Cc: Mark Brown Cc: Will Deacon Link: https://lore.kernel.org/r/20250508132644.1395904-12-mark.rutland@arm.com Signed-off-by: Will Deacon [ bh: Adjust arch_dup_task_struct context to the existing task layout. ] Signed-off-by: Bin Huang --- arch/arm64/kernel/process.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index e2a100fdb7dc..8ad8defd3300 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -286,7 +286,14 @@ void arch_release_task_struct(struct task_struct *tsk) int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) { - fpsimd_preserve_current_state(); + /* + * The current/src task's FPSIMD state may or may not be live, and may + * have been altered by ptrace after entry to the kernel. Save the + * effective FPSIMD state so that this will be copied into dst. + */ + fpsimd_save_and_flush_current_state(); + fpsimd_sync_from_effective_state(src); + *dst = *src; /* We rely on the above assignment to initialize dst's thread_flags: */ From bbfed1ebff24f49620e0536531159c02a760acbc Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Thu, 8 May 2025 14:26:32 +0100 Subject: [PATCH 60/85] arm64/fpsimd: Clear PSTATE.SM during clone() commit a6d066f705747124fb2d662df0acbb45ffe6c406 upstream. Currently arch_dup_task_struct() doesn't handle cases where the parent task has PSTATE.SM==1. Since syscall entry exits streaming mode, the parent will usually have PSTATE.SM==0, but this can be change by ptrace after syscall entry. When this happens, arch_dup_task_struct() will initialise the new task into an invalid state. The new task inherits the parent's configuration of PSTATE.SM, but fp_type is set to FP_STATE_FPSIMD, TIF_SVE and SME may be cleared, and both sve_state and sme_state may be set to NULL. This can result in a variety of problems whenever the new task's state is manipulated, including kernel NULL pointer dereferences and leaking of streaming mode state between tasks. When ptrace is not involved, the parent will have PSTATE.SM==0 as a result of syscall entry, and the documentation in Documentation/arch/arm64/sme.rst says: | On process creation (eg, clone()) the newly created process will have | PSTATE.SM cleared. ... so make this true by using task_smstop_sm() to exit streaming mode in the child task, avoiding the problems above. Fixes: 8bd7f91c03d8 ("arm64/sme: Implement traps and syscall handling for SME") Signed-off-by: Mark Rutland Cc: Catalin Marinas Cc: Marc Zyngier Cc: Mark Brown Cc: Will Deacon Link: https://lore.kernel.org/r/20250508132644.1395904-13-mark.rutland@arm.com Signed-off-by: Will Deacon [ bh: Retain the target thread-info initialization assertion when updating clone state. ] Signed-off-by: Bin Huang --- arch/arm64/kernel/process.c | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index 8ad8defd3300..3178d51a2dc9 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -300,16 +300,13 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) BUILD_BUG_ON(!IS_ENABLED(CONFIG_THREAD_INFO_IN_TASK)); /* - * Detach src's sve_state (if any) from dst so that it does not - * get erroneously used or freed prematurely. dst's copies - * will be allocated on demand later on if dst uses SVE. - * For consistency, also clear TIF_SVE here: this could be done - * later in copy_process(), but to avoid tripping up future - * maintainers it is best not to leave TIF flags and buffers in - * an inconsistent state, even temporarily. + * Drop stale reference to src's sve_state and convert dst to + * non-streaming FPSIMD mode. */ + dst->thread.fp_type = FP_STATE_FPSIMD; dst->thread.sve_state = NULL; clear_tsk_thread_flag(dst, TIF_SVE); + task_smstop_sm(dst); /* * In the unlikely event that we create a new thread with ZA @@ -338,8 +335,6 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) clear_tsk_thread_flag(dst, TIF_SME); } - dst->thread.fp_type = FP_STATE_FPSIMD; - /* clear any pending asynchronous tag fault raised by the parent */ clear_tsk_thread_flag(dst, TIF_MTE_ASYNC_FAULT); From acf62f56c93093caca1b74985306d16a20ac76de Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Thu, 8 May 2025 14:26:33 +0100 Subject: [PATCH 61/85] arm64/fpsimd: Make clone() compatible with ZA lazy saving commit cde5c32db55740659fca6d56c09b88800d88fd29 upstream. Linux is intended to be compatible with userspace written to Arm's AAPCS64 procedure call standard [1,2]. For the Scalable Matrix Extension (SME), AAPCS64 was extended with a "ZA lazy saving scheme", where SME's ZA tile is lazily callee-saved and caller-restored. In this scheme, TPIDR2_EL0 indicates whether the ZA tile is live or has been saved by pointing to a "TPIDR2 block" in memory, which has a "za_save_buffer" pointer. This scheme has been implemented in GCC and LLVM, with necessary runtime support implemented in glibc and bionic. AAPCS64 does not specify how the ZA lazy saving scheme is expected to interact with thread creation mechanisms such as fork() and pthread_create(), which would be implemented in terms of the Linux clone syscall. The behaviour implemented by Linux and glibc/bionic doesn't always compose safely, as explained below. Currently the clone syscall is implemented such that PSTATE.ZA and the ZA tile are always inherited by the new task, and TPIDR2_EL0 is inherited unless the 'flags' argument includes CLONE_SETTLS, in which case TPIDR2_EL0 is set to 0/NULL. This doesn't make much sense: (a) TPIDR2_EL0 is part of the calling convention, and changes as control is passed between functions. It is *NOT* used for thread local storage, despite superficial similarity to TPIDR_EL0, which is is used as the TLS register. (b) TPIDR2_EL0 and PSTATE.ZA are tightly coupled in the procedure call standard, and some combinations of states are illegal. In general, manipulating the two independently is not guaranteed to be safe. In practice, code which is compliant with the procedure call standard may issue a clone syscall while in the "ZA dormant" state, where PSTATE.ZA==1 and TPIDR2_EL0 is non-null and indicates that ZA needs to be saved. This can cause a variety of problems, including: * If the implementation of pthread_create() passes CLONE_SETTLS, the new thread will start with PSTATE.ZA==1 and TPIDR2==NULL. Per the procedure call standard this is not a legitimate state for most functions. This can cause data corruption (e.g. as code may rely on PSTATE.ZA being 0 to guarantee that an SMSTART ZA instruction will zero the ZA tile contents), and may result in other undefined behaviour. * If the implementation of pthread_create() does not pass CLONE_SETTLS, the new thread will start with PSTATE.ZA==1 and TPIDR2 pointing to a TPIDR2 block on the parent thread's stack. This can result in a variety of problems, e.g. - The child may write back to the parent's za_save_buffer, corrupting its contents. - The child may read from the TPIDR2 block after the parent has reused this memory for something else, and consequently the child may abort or clobber arbitrary memory. Ideally we'd require that userspace ensures that a task is in the "ZA off" state (with PSTATE.ZA==0 and TPIDR2_EL0==NULL) prior to issuing a clone syscall, and have the kernel force this state for new threads. Unfortunately, contemporary C libraries do not do this, and simply forcing this state within the implementation of clone would break fork(). Instead, we can bodge around this by considering the CLONE_VM flag, and manipulate PSTATE.ZA and TPIDR2_EL0 as a pair. CLONE_VM indicates that the new task will run in the same address space as its parent, and in that case it doesn't make sense to inherit a stale pointer to the parent's TPIDR2 block: * For fork(), CLONE_VM will not be set, and it is safe to inherit both PSTATE.ZA and TPIDR2_EL0 as the new task will have its own copy of the address space, and cannot clobber its parent's stack. * For pthread_create() and vfork(), CLONE_VM will be set, and discarding PSTATE.ZA and TPIDR2_EL0 for the new task doesn't break any existing assumptions in userspace. Implement this behaviour for clone(). We currently inherit PSTATE.ZA in arch_dup_task_struct(), but this does not have access to the clone flags, so move this logic under copy_thread(). Documentation is updated to describe the new behaviour. [1] https://github.com/ARM-software/abi-aa/releases/download/2025Q1/aapcs64.pdf [2] https://github.com/ARM-software/abi-aa/blob/c51addc3dc03e73a016a1e4edf25440bcac76431/aapcs64/aapcs64.rst Suggested-by: Catalin Marinas Signed-off-by: Mark Rutland Cc: Catalin Marinas Cc: Daniel Kiss Cc: Marc Zyngier Cc: Mark Brown Cc: Richard Sandiford Cc: Sander De Smalen Cc: Tamas Petz Cc: Will Deacon Cc: Yury Khrustalev Acked-by: Yury Khrustalev Link: https://lore.kernel.org/r/20250508132644.1395904-14-mark.rutland@arm.com Signed-off-by: Will Deacon [ bh: Adapt to the target kernel without POE/GCS and declare ret for copy_thread_za() error handling. ] Signed-off-by: Bin Huang --- Documentation/arch/arm64/sme.rst | 4 +- arch/arm64/kernel/process.c | 93 +++++++++++++++++++++----------- 2 files changed, 65 insertions(+), 32 deletions(-) diff --git a/Documentation/arch/arm64/sme.rst b/Documentation/arch/arm64/sme.rst index 316c7b4e6fb7..f9da92312878 100644 --- a/Documentation/arch/arm64/sme.rst +++ b/Documentation/arch/arm64/sme.rst @@ -69,8 +69,8 @@ model features for SME is included in Appendix A. vectors from 0 to VL/8-1 stored in the same endianness invariant format as is used for SVE vectors. -* On thread creation TPIDR2_EL0 is preserved unless CLONE_SETTLS is specified, - in which case it is set to 0. +* On thread creation PSTATE.ZA and TPIDR2_EL0 are preserved unless CLONE_VM + is specified, in which case PSTATE.ZA is set to 0 and TPIDR2_EL0 is set to 0. 2. Vector lengths ------------------ diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index 3178d51a2dc9..ffedc551e9f5 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -309,31 +309,14 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) task_smstop_sm(dst); /* - * In the unlikely event that we create a new thread with ZA - * enabled we should retain the ZA and ZT state so duplicate - * it here. This may be shortly freed if we exec() or if - * CLONE_SETTLS but it's simpler to do it here. To avoid - * confusing the rest of the code ensure that we have a - * sve_state allocated whenever sme_state is allocated. + * Drop stale reference to src's sme_state and ensure dst has ZA + * disabled. + * + * When necessary, ZA will be inherited later in copy_thread_za(). */ - if (thread_za_enabled(&src->thread)) { - dst->thread.sve_state = kzalloc(sve_state_size(src), - GFP_KERNEL); - if (!dst->thread.sve_state) - return -ENOMEM; - - dst->thread.sme_state = kmemdup(src->thread.sme_state, - sme_state_size(src), - GFP_KERNEL); - if (!dst->thread.sme_state) { - kfree(dst->thread.sve_state); - dst->thread.sve_state = NULL; - return -ENOMEM; - } - } else { - dst->thread.sme_state = NULL; - clear_tsk_thread_flag(dst, TIF_SME); - } + dst->thread.sme_state = NULL; + clear_tsk_thread_flag(dst, TIF_SME); + dst->thread.svcr &= ~SVCR_ZA_MASK; /* clear any pending asynchronous tag fault raised by the parent */ clear_tsk_thread_flag(dst, TIF_MTE_ASYNC_FAULT); @@ -341,6 +324,31 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) return 0; } +static int copy_thread_za(struct task_struct *dst, struct task_struct *src) +{ + if (!thread_za_enabled(&src->thread)) + return 0; + + dst->thread.sve_state = kzalloc(sve_state_size(src), + GFP_KERNEL); + if (!dst->thread.sve_state) + return -ENOMEM; + + dst->thread.sme_state = kmemdup(src->thread.sme_state, + sme_state_size(src), + GFP_KERNEL); + if (!dst->thread.sme_state) { + kfree(dst->thread.sve_state); + dst->thread.sve_state = NULL; + return -ENOMEM; + } + + set_tsk_thread_flag(dst, TIF_SME); + dst->thread.svcr |= SVCR_ZA_MASK; + + return 0; +} + asmlinkage void ret_from_fork(void) asm("ret_from_fork"); int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) @@ -349,6 +357,7 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) unsigned long stack_start = args->stack; unsigned long tls = args->tls; struct pt_regs *childregs = task_pt_regs(p); + int ret; memset(&p->thread.cpu_context, 0, sizeof(struct cpu_context)); @@ -372,8 +381,6 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) * out-of-sync with the saved value. */ *task_user_tls(p) = read_sysreg(tpidr_el0); - if (system_supports_tpidr2()) - p->thread.tpidr2_el0 = read_sysreg_s(SYS_TPIDR2_EL0); if (stack_start) { if (is_compat_thread(task_thread_info(p))) @@ -382,14 +389,40 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) childregs->sp = stack_start; } + /* + * Due to the AAPCS64 "ZA lazy saving scheme", PSTATE.ZA and + * TPIDR2 need to be manipulated as a pair, and either both + * need to be inherited or both need to be reset. + * + * Within a process, child threads must not inherit their + * parent's TPIDR2 value or they may clobber their parent's + * stack at some later point. + * + * When a process is fork()'d, the child must inherit ZA and + * TPIDR2 from its parent in case there was dormant ZA state. + * + * Use CLONE_VM to determine when the child will share the + * address space with the parent, and cannot safely inherit the + * state. + */ + if (system_supports_sme()) { + if (!(clone_flags & CLONE_VM)) { + p->thread.tpidr2_el0 = read_sysreg_s(SYS_TPIDR2_EL0); + ret = copy_thread_za(p, current); + if (ret) + return ret; + } else { + p->thread.tpidr2_el0 = 0; + WARN_ON_ONCE(p->thread.svcr & SVCR_ZA_MASK); + } + } + /* * If a TLS pointer was passed to clone, use it for the new - * thread. We also reset TPIDR2 if it's in use. + * thread. */ - if (clone_flags & CLONE_SETTLS) { + if (clone_flags & CLONE_SETTLS) p->thread.uw.tp_value = tls; - p->thread.tpidr2_el0 = 0; - } } else { /* * A kthread has no context to ERET to, so ensure any buggy From 737e32f9f980d9b9d11f03ff57df5b7de28d779d Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Thu, 8 May 2025 14:26:34 +0100 Subject: [PATCH 62/85] arm64/fpsimd: ptrace/prctl: Ensure VL changes do not resurrect stale data commit 49ce484187f72a94c202348179a9a4e63a0f864b upstream. The SVE/SME vector lengths can be changed via prctl/ptrace syscalls. Changes to the SVE/SME vector lengths are documented as preserving the lower 128 bits of the Z registers (i.e. the bits shared with the FPSIMD V registers). To ensure this, vec_set_vector_length() explicitly copies register values from a task's saved SVE state to its saved FPSIMD state when dropping the task to FPSIMD-only. The logic for this was not updated when when FPSIMD/SVE state tracking was changed across commits: baa8515281b3 ("arm64/fpsimd: Track the saved FPSIMD state type separately to TIF_SVE") a0136be443d5 (arm64/fpsimd: Load FP state based on recorded data type") bbc6172eefdb ("arm64/fpsimd: SME no longer requires SVE register state") 8c845e273104 ("arm64/sve: Leave SVE enabled on syscall if we don't context switch") Since the last commit above, a task's FPSIMD/SVE state may be stored in FPSIMD format while TIF_SVE is set, and the stored SVE state is stale. When vec_set_vector_length() encounters this case, it will erroneously clobber the live FPSIMD state with stale SVE state by using sve_to_fpsimd(). Fix this by using fpsimd_sync_from_effective_state() instead. Related issues with streaming mode state will be addressed in subsequent patches. Fixes: 8c845e273104 ("arm64/sve: Leave SVE enabled on syscall if we don't context switch") Signed-off-by: Mark Rutland Cc: Catalin Marinas Cc: David Spickett Cc: Luis Machado Cc: Marc Zyngier Cc: Mark Brown Cc: Will Deacon Link: https://lore.kernel.org/r/20250508132644.1395904-15-mark.rutland@arm.com Signed-off-by: Will Deacon Signed-off-by: Bin Huang --- arch/arm64/kernel/fpsimd.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c index cb0395cfa19e..89f71c096244 100644 --- a/arch/arm64/kernel/fpsimd.c +++ b/arch/arm64/kernel/fpsimd.c @@ -870,7 +870,7 @@ int vec_set_vector_length(struct task_struct *task, enum vec_type type, fpsimd_flush_task_state(task); if (test_and_clear_tsk_thread_flag(task, TIF_SVE) || thread_sm_enabled(&task->thread)) { - sve_to_fpsimd(task); + fpsimd_sync_from_effective_state(task); task->thread.fp_type = FP_STATE_FPSIMD; } From 925e468959ccc4a8e4ee82e734d23ddc8e9ee686 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Thu, 8 May 2025 14:26:35 +0100 Subject: [PATCH 63/85] arm64/fpsimd: ptrace/prctl: Ensure VL changes leave task in a valid state commit b87c8c4aca1163ae4791507089007863e9c3ca1b upstream. Currently, vec_set_vector_length() can manipulate a task into an invalid state as a result of a prctl/ptrace syscall which changes the SVE/SME vector length, resulting in several problems: (1) When changing the SVE vector length, if the task initially has PSTATE.ZA==1, and sve_alloc() fails to allocate memory, the task will be left with PSTATE.ZA==1 and sve_state==NULL. This is not a legitimate state, and could result in a subsequent null pointer dereference. (2) When changing the SVE vector length, if the task initially has PSTATE.SM==1, the task will be left with PSTATE.SM==1 and fp_type==FP_STATE_FPSIMD. Streaming mode state always needs to be saved in SVE format, so this is not a legitimate state. Attempting to restore this state may cause a task to erroneously inherit stale streaming mode predicate registers and FFR contents, behaving non-deterministically and potentially leaving information from another task. While in this state, reads of the NT_ARM_SSVE regset will indicate that the registers are not stored in SVE format. For the NT_ARM_SSVE regset specifically, debuggers interpret this as meaning that PSTATE.SM==0. (3) When changing the SME vector length, if the task initially has PSTATE.SM==1, the lower 128 bits of task's streaming mode vector state will be migrated to non-streaming mode, rather than these bits being zeroed as is usually the case for changes to PSTATE.SM. To fix the first issue, we can eagerly allocate the new sve_state and sme_state before modifying the task. This makes it possible to handle memory allocation failure without modifying the task state at all, and removes the need to clear TIF_SVE and TIF_SME. To fix the second issue, we either need to clear PSTATE.SM or not change the saved fp_type. Given we're going to eagerly allocate sve_state and sme_state, the simplest option is to preserve PSTATE.SM and the saves fp_type, and consistently truncate the SVE state. This ensures that the task always stays in a valid state, and by virtue of not exiting streaming mode, this also sidesteps the third issue. I believe these changes should not be problematic for realistic usage: * When the SVE/SME vector length is changed via prctl(), syscall entry will have cleared PSTATE.SM. Unless the task's state has been manipulated via ptrace after entry, the task will have PSTATE.SM==0. * When the SVE/SME vector length is changed via a write to the NT_ARM_SVE or NT_ARM_SSVE regsets, PSTATE.SM will be forced immediately after the length change, and new vector state will be copied from userspace. * When the SME vector length is changed via a write to the NT_ARM_ZA regset, the (S)SVE state is clobbered today, so anyone who cares about the specific state would need to install this after writing to the NT_ARM_ZA regset. As we need to free the old SVE state while TIF_SVE may still be set, we cannot use sve_free(), and using kfree() directly makes it clear that the free pairs with the subsequent assignment. As this leaves sve_free() unused, I've removed the existing sve_free() and renamed __sve_free() to mirror sme_free(). Fixes: 8bd7f91c03d8 ("arm64/sme: Implement traps and syscall handling for SME") Fixes: baa8515281b3 ("arm64/fpsimd: Track the saved FPSIMD state type separately to TIF_SVE") Signed-off-by: Mark Rutland Cc: Catalin Marinas Cc: David Spickett Cc: Luis Machado Cc: Marc Zyngier Cc: Mark Brown Cc: Will Deacon Link: https://lore.kernel.org/r/20250508132644.1395904-16-mark.rutland@arm.com Signed-off-by: Will Deacon [ bh: Replace the older VL transition in full and retain its documentation flag spelling. ] Signed-off-by: Bin Huang --- Documentation/arch/arm64/sme.rst | 2 +- arch/arm64/kernel/fpsimd.c | 148 +++++++++++++++---------------- 2 files changed, 73 insertions(+), 77 deletions(-) diff --git a/Documentation/arch/arm64/sme.rst b/Documentation/arch/arm64/sme.rst index f9da92312878..7a1ff0dc9e4c 100644 --- a/Documentation/arch/arm64/sme.rst +++ b/Documentation/arch/arm64/sme.rst @@ -241,7 +241,7 @@ prctl(PR_SME_SET_VL, unsigned long arg) length, or calling PR_SME_SET_VL with the PR_SVE_SET_VL_ONEXEC flag, does not constitute a change to the vector length for this purpose. - * Changing the vector length causes PSTATE.ZA and PSTATE.SM to be cleared. + * Changing the vector length causes PSTATE.ZA to be cleared. Calling PR_SME_SET_VL with vl equal to the thread's current vector length, or calling PR_SME_SET_VL with the PR_SVE_SET_VL_ONEXEC flag, does not constitute a change to the vector length for this purpose. diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c index 89f71c096244..5c0a7fecb15d 100644 --- a/arch/arm64/kernel/fpsimd.c +++ b/arch/arm64/kernel/fpsimd.c @@ -728,23 +728,12 @@ void cpu_enable_fpmr(const struct arm64_cpu_capabilities *__always_unused p) } #ifdef CONFIG_ARM64_SVE -/* - * Call __sve_free() directly only if you know task can't be scheduled - * or preempted. - */ -static void __sve_free(struct task_struct *task) +static void sve_free(struct task_struct *task) { kfree(task->thread.sve_state); task->thread.sve_state = NULL; } -static void sve_free(struct task_struct *task) -{ - WARN_ON(test_tsk_thread_flag(task, TIF_SVE)); - - __sve_free(task); -} - /* * Ensure that task->thread.sve_state is allocated and sufficiently large. * @@ -819,10 +808,73 @@ void fpsimd_sync_to_effective_state_zeropad(struct task_struct *task) __fpsimd_to_sve(sst, fst, vq); } +static int change_live_vector_length(struct task_struct *task, + enum vec_type type, + unsigned long vl) +{ + unsigned int sve_vl = task_get_sve_vl(task); + unsigned int sme_vl = task_get_sme_vl(task); + void *sve_state = NULL, *sme_state = NULL; + + if (type == ARM64_VEC_SME) + sme_vl = vl; + else + sve_vl = vl; + + /* + * Allocate the new sve_state and sme_state before freeing the old + * copies so that allocation failure can be handled without needing to + * mutate the task's state in any way. + * + * Changes to the SVE vector length must not discard live ZA state or + * clear PSTATE.ZA, as userspace code which is unaware of the AAPCS64 + * ZA lazy saving scheme may attempt to change the SVE vector length + * while unsaved/dormant ZA state exists. + */ + sve_state = kzalloc(__sve_state_size(sve_vl, sme_vl), GFP_KERNEL); + if (!sve_state) + goto out_mem; + + if (type == ARM64_VEC_SME) { + sme_state = kzalloc(__sme_state_size(sme_vl), GFP_KERNEL); + if (!sme_state) + goto out_mem; + } + + if (task == current) + fpsimd_save_and_flush_current_state(); + else + fpsimd_flush_task_state(task); + + /* + * Always preserve PSTATE.SM and the effective FPSIMD state, zeroing + * other SVE state. + */ + fpsimd_sync_from_effective_state(task); + task_set_vl(task, type, vl); + kfree(task->thread.sve_state); + task->thread.sve_state = sve_state; + fpsimd_sync_to_effective_state_zeropad(task); + + if (type == ARM64_VEC_SME) { + task->thread.svcr &= ~SVCR_ZA_MASK; + kfree(task->thread.sme_state); + task->thread.sme_state = sme_state; + } + + return 0; + +out_mem: + kfree(sve_state); + kfree(sme_state); + return -ENOMEM; +} + int vec_set_vector_length(struct task_struct *task, enum vec_type type, unsigned long vl, unsigned long flags) { - bool free_sme = false; + bool onexec = flags & PR_SVE_SET_VL_ONEXEC; + bool inherit = flags & PR_SVE_VL_INHERIT; if (flags & ~(unsigned long)(PR_SVE_VL_INHERIT | PR_SVE_SET_VL_ONEXEC)) @@ -842,73 +894,17 @@ int vec_set_vector_length(struct task_struct *task, enum vec_type type, vl = find_supported_vector_length(type, vl); - if (flags & (PR_SVE_VL_INHERIT | - PR_SVE_SET_VL_ONEXEC)) + if (!onexec && vl != task_get_vl(task, type)) { + if (change_live_vector_length(task, type, vl)) + return -ENOMEM; + } + + if (onexec || inherit) task_set_vl_onexec(task, type, vl); else /* Reset VL to system default on next exec: */ task_set_vl_onexec(task, type, 0); - /* Only actually set the VL if not deferred: */ - if (flags & PR_SVE_SET_VL_ONEXEC) - goto out; - - if (vl == task_get_vl(task, type)) - goto out; - - /* - * To ensure the FPSIMD bits of the SVE vector registers are preserved, - * write any live register state back to task_struct, and convert to a - * regular FPSIMD thread. - */ - if (task == current) { - get_cpu_fpsimd_context(); - - fpsimd_save_user_state(); - } - - fpsimd_flush_task_state(task); - if (test_and_clear_tsk_thread_flag(task, TIF_SVE) || - thread_sm_enabled(&task->thread)) { - fpsimd_sync_from_effective_state(task); - task->thread.fp_type = FP_STATE_FPSIMD; - } - - if (system_supports_sme()) { - if (type == ARM64_VEC_SME || - !(task->thread.svcr & (SVCR_SM_MASK | SVCR_ZA_MASK))) { - /* - * We are changing the SME VL or weren't using - * SME anyway, discard the state and force a - * reallocation. - */ - task->thread.svcr &= ~(SVCR_SM_MASK | - SVCR_ZA_MASK); - clear_tsk_thread_flag(task, TIF_SME); - free_sme = true; - } - } - - if (task == current) - put_cpu_fpsimd_context(); - - task_set_vl(task, type, vl); - - /* - * Free the changed states if they are not in use, SME will be - * reallocated to the correct size on next use and we just - * allocate SVE now in case it is needed for use in streaming - * mode. - */ - if (system_supports_sve()) { - sve_free(task); - sve_alloc(task, true); - } - - if (free_sme) - sme_free(task); - -out: update_tsk_thread_flag(task, vec_vl_inherit_flag(type), flags & PR_SVE_VL_INHERIT); @@ -1230,7 +1226,7 @@ void __init sve_setup(void) */ void fpsimd_release_task(struct task_struct *dead_task) { - __sve_free(dead_task); + sve_free(dead_task); sme_free(dead_task); } From 761e61ff9b7686a7e4dde3258bff705a1fc64ba3 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Thu, 8 May 2025 14:26:36 +0100 Subject: [PATCH 64/85] arm64/fpsimd: ptrace: Save task state before generating SVE header commit 054d627c5554bdd38228174b275d62113124e3ad upstream. As sve_init_header_from_task() consumes the saved value of PSTATE.SM and the saved fp_type, both must be saved before the header is generated. When generating a coredump for the current task, sve_get_common() calls sve_init_header_from_task() before saving the task's state. Consequently the header may be bogus, and the contents of the regset may be misleading. Fix this by saving the task's state before generting the header. Fixes: e12310a0d30f ("arm64/sme: Implement ptrace support for streaming mode SVE registers") Fixes: b017a0cea627 ("arm64/ptrace: Use saved floating point state type to determine SVE layout") Signed-off-by: Mark Rutland Cc: Catalin Marinas Cc: David Spickett Cc: Luis Machado Cc: Marc Zyngier Cc: Mark Brown Cc: Will Deacon Link: https://lore.kernel.org/r/20250508132644.1395904-17-mark.rutland@arm.com Signed-off-by: Will Deacon Signed-off-by: Bin Huang --- arch/arm64/kernel/ptrace.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c index 6e3badd42049..63b80fb48f39 100644 --- a/arch/arm64/kernel/ptrace.c +++ b/arch/arm64/kernel/ptrace.c @@ -819,15 +819,15 @@ static int sve_get_common(struct task_struct *target, unsigned int vq; unsigned long start, end; + if (target == current) + fpsimd_preserve_current_state(); + /* Header */ sve_init_header_from_task(&header, target, type); vq = sve_vq_from_vl(header.vl); membuf_write(&to, &header, sizeof(header)); - if (target == current) - fpsimd_preserve_current_state(); - BUILD_BUG_ON(SVE_PT_FPSIMD_OFFSET != sizeof(header)); BUILD_BUG_ON(SVE_PT_SVE_OFFSET != sizeof(header)); From c9e522c791faae05b85df8b5e6819f7ea7c44cf8 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Thu, 8 May 2025 14:26:37 +0100 Subject: [PATCH 65/85] arm64/fpsimd: ptrace: Do not present register data for inactive mode commit b93e685ecff77e0b231c12802fb632ef36a62140 upstream. The SME ptrace ABI is written around the incorrect assumption that SVE_PT_REGS_FPSIMD and SVE_PT_REGS_SVE are independent bit flags, where it is possible for both to be clear. In reality they are different values for bit 0 of the header flags, where SVE_PT_REGS_FPSIMD is 0 and SVE_PT_REGS_SVE is 1. In cases where code was written expecting that neither bit flag would be set, the value is equivalent to SVE_PT_REGS_FPSIMD. One consequence of this is that reads of the NT_ARM_SVE or NT_ARM_SSVE will erroneously present data from the other mode: * When PSTATE.SM==1, reads of NT_ARM_SVE will present a header with SVE_PT_REGS_FPSIMD, and FPSIMD-formatted data from streaming mode. * When PSTATE.SM==0, reads of NT_ARM_SSVE will present a header with SVE_PT_REGS_FPSIMD, and FPSIMD-formatted data from non-streaming mode. The original intent was that no register data would be provided in these cases, as described in commit: e12310a0d30f ("arm64/sme: Implement ptrace support for streaming mode SVE registers") Luckily, debuggers do not consume the bogus register data. Both GDB and LLDB read the NT_ARM_SSVE regset before the NT_ARM_SVE regset, and assume that when the NT_ARM_SSVE header presents SVE_PT_REGS_FPSIMD, it is necessary to read register contents from the NT_ARM_SVE regset, regardless of whether the NT_ARM_SSVE regset provided bogus register data. Fix the code to stop presenting register data from the inactive mode. At the same time, make the manipulation of the flag clearer, and remove the bogus comment from sve_set_common(). I've given this a quick spin with GDB and LLDB, and both seem happy. Fixes: e12310a0d30f ("arm64/sme: Implement ptrace support for streaming mode SVE registers") Signed-off-by: Mark Rutland Cc: Catalin Marinas Cc: David Spickett Cc: Luis Machado Cc: Marc Zyngier Cc: Mark Brown Cc: Will Deacon Link: https://lore.kernel.org/r/20250508132644.1395904-18-mark.rutland@arm.com Signed-off-by: Will Deacon Signed-off-by: Bin Huang --- arch/arm64/kernel/ptrace.c | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c index 63b80fb48f39..40be1ff5b406 100644 --- a/arch/arm64/kernel/ptrace.c +++ b/arch/arm64/kernel/ptrace.c @@ -774,6 +774,11 @@ static void sve_init_header_from_task(struct user_sve_header *header, task_type = ARM64_VEC_SVE; active = (task_type == type); + if (active && target->thread.fp_type == FP_STATE_SVE) + header->flags = SVE_PT_REGS_SVE; + else + header->flags = SVE_PT_REGS_FPSIMD; + switch (type) { case ARM64_VEC_SVE: if (test_tsk_thread_flag(target, TIF_SVE_VL_INHERIT)) @@ -788,19 +793,14 @@ static void sve_init_header_from_task(struct user_sve_header *header, return; } - if (active) { - if (target->thread.fp_type == FP_STATE_FPSIMD) { - header->flags |= SVE_PT_REGS_FPSIMD; - } else { - header->flags |= SVE_PT_REGS_SVE; - } - } - header->vl = task_get_vl(target, type); vq = sve_vq_from_vl(header->vl); header->max_vl = vec_max_vl(type); - header->size = SVE_PT_SIZE(vq, header->flags); + if (active) + header->size = SVE_PT_SIZE(vq, header->flags); + else + header->size = sizeof(header); header->max_size = SVE_PT_SIZE(sve_vq_from_vl(header->max_vl), SVE_PT_REGS_SVE); } @@ -831,6 +831,13 @@ static int sve_get_common(struct task_struct *target, BUILD_BUG_ON(SVE_PT_FPSIMD_OFFSET != sizeof(header)); BUILD_BUG_ON(SVE_PT_SVE_OFFSET != sizeof(header)); + /* + * When the requested vector type is not active, do not present data + * from the other mode to userspace. + */ + if (header.size == sizeof(header)) + return 0; + switch ((header.flags & SVE_PT_REGS_MASK)) { case SVE_PT_REGS_FPSIMD: return __fpr_get(target, regset, to); @@ -858,7 +865,7 @@ static int sve_get_common(struct task_struct *target, return membuf_zero(&to, end - start); default: - return 0; + BUILD_BUG(); } } @@ -941,10 +948,7 @@ static int sve_set_common(struct task_struct *target, goto out; } - /* - * Otherwise: no registers or full SVE case. For backwards - * compatibility reasons we treat empty flags as SVE registers. - */ + /* Otherwise: no registers or full SVE case. */ /* * If setting a different VL from the requested VL and there is From 59c5fa6f2b21b4f82b70954ce55e9219996ccfaa Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Thu, 8 May 2025 14:26:38 +0100 Subject: [PATCH 66/85] arm64/fpsimd: ptrace: Mandate SVE payload for streaming-mode state commit f916dd32a943a7ab40497718aa7bcf3648d2bb39 upstream. When a task has PSTATE.SM==1, reads of NT_ARM_SSVE are required to always present a header with SVE_PT_REGS_SVE, and register data in SVE format. Reads of NT_ARM_SSVE must never present register data in FPSIMD format. Within the kernel, we always expect streaming SVE data to be stored in SVE format. Currently a user can write to NT_ARM_SSVE with a header presenting SVE_PT_REGS_FPSIMD rather than SVE_PT_REGS_SVE, placing the task's FPSIMD/SVE data into an invalid state. To fix this we can either: (a) Forbid such writes. (b) Accept such writes, and immediately convert data into SVE format. Take the simple option and forbid such writes. Fixes: e12310a0d30f ("arm64/sme: Implement ptrace support for streaming mode SVE registers") Signed-off-by: Mark Rutland Cc: Catalin Marinas Cc: David Spickett Cc: Luis Machado Cc: Marc Zyngier Cc: Mark Brown Cc: Will Deacon Reviewed-by: Mark Brown Link: https://lore.kernel.org/r/20250508132644.1395904-19-mark.rutland@arm.com Signed-off-by: Will Deacon Signed-off-by: Bin Huang --- arch/arm64/kernel/ptrace.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c index 40be1ff5b406..857fe05a9c4b 100644 --- a/arch/arm64/kernel/ptrace.c +++ b/arch/arm64/kernel/ptrace.c @@ -889,6 +889,7 @@ static int sve_set_common(struct task_struct *target, struct user_sve_header header; unsigned int vq; unsigned long start, end; + bool fpsimd; /* Header */ if (count < sizeof(header)) @@ -898,6 +899,15 @@ static int sve_set_common(struct task_struct *target, if (ret) goto out; + /* + * Streaming SVE data is always stored and presented in SVE format. + * Require the user to provide SVE formatted data for consistency, and + * to avoid the risk that we configure the task into an invalid state. + */ + fpsimd = (header.flags & SVE_PT_REGS_MASK) == SVE_PT_REGS_FPSIMD; + if (fpsimd && type == ARM64_VEC_SME) + return -EINVAL; + /* * Apart from SVE_PT_REGS_MASK, all SVE_PT_* flags are consumed by * vec_set_vector_length(), which will also validate them for us: @@ -940,7 +950,7 @@ static int sve_set_common(struct task_struct *target, /* Registers: FPSIMD-only case */ BUILD_BUG_ON(SVE_PT_FPSIMD_OFFSET != sizeof(header)); - if ((header.flags & SVE_PT_REGS_MASK) == SVE_PT_REGS_FPSIMD) { + if (fpsimd) { clear_tsk_thread_flag(target, TIF_SVE); target->thread.fp_type = FP_STATE_FPSIMD; ret = __fpr_set(target, regset, pos, count, kbuf, ubuf, From 06363ca5293a5a8bb672843020b9b7cc561d2a6d Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Thu, 8 May 2025 14:26:39 +0100 Subject: [PATCH 67/85] arm64/fpsimd: ptrace: Gracefully handle errors commit 9f8bf718f29230e38a048d08fc3063e316cd60c1 upstream. Within sve_set_common() we do not handle error conditions correctly: * When writing to NT_ARM_SSVE, if sme_alloc() fails, the task will be left with task->thread.sme_state==NULL, but TIF_SME will be set and task->thread.fp_type==FP_STATE_SVE. This will result in a subsequent null pointer dereference when the task's state is loaded or otherwise manipulated. * When writing to NT_ARM_SSVE, if sve_alloc() fails, the task will be left with task->thread.sve_state==NULL, but TIF_SME will be set, PSTATE.SM will be set, and task->thread.fp_type==FP_STATE_FPSIMD. This is not a legitimate state, and can result in various problems, including a subsequent null pointer dereference and/or the task inheriting stale streaming mode register state the next time its state is loaded into hardware. * When writing to NT_ARM_SSVE, if the VL is changed but the resulting VL differs from that in the header, the task will be left with TIF_SME set, PSTATE.SM set, but task->thread.fp_type==FP_STATE_FPSIMD. This is not a legitimate state, and can result in various problems as described above. Avoid these problems by allocating memory earlier, and by changing the task's saved fp_type to FP_STATE_SVE before skipping register writes due to a change of VL. To make early returns simpler, I've moved the call to fpsimd_flush_task_state() earlier. As the tracee's state has already been saved, and the tracee is known to be blocked for the duration of sve_set_common(), it doesn't matter whether this is called at the start or the end. For consistency I've moved the setting of TIF_SVE earlier. This will be cleared when loading FPSIMD-only state, and so moving this has no resulting functional change. Note that we only allocate the memory for SVE state when SVE register contents are provided, avoiding unnecessary memory allocations for tasks which only use FPSIMD. Fixes: e12310a0d30f ("arm64/sme: Implement ptrace support for streaming mode SVE registers") Fixes: baa8515281b3 ("arm64/fpsimd: Track the saved FPSIMD state type separately to TIF_SVE") Fixes: 5d0a8d2fba50 ("arm64/ptrace: Ensure that SME is set up for target when writing SSVE state") Signed-off-by: Mark Rutland Cc: Catalin Marinas Cc: David Spickett Cc: Luis Machado Cc: Marc Zyngier Cc: Mark Brown Cc: Will Deacon Link: https://lore.kernel.org/r/20250508132644.1395904-20-mark.rutland@arm.com Signed-off-by: Will Deacon [ bh: Adjust SVE regset error-handling context to the existing ptrace implementation. ] Signed-off-by: Bin Huang --- arch/arm64/kernel/ptrace.c | 61 ++++++++++++++++---------------------- 1 file changed, 26 insertions(+), 35 deletions(-) diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c index 857fe05a9c4b..c510cacc34b8 100644 --- a/arch/arm64/kernel/ptrace.c +++ b/arch/arm64/kernel/ptrace.c @@ -891,13 +891,15 @@ static int sve_set_common(struct task_struct *target, unsigned long start, end; bool fpsimd; + fpsimd_flush_task_state(target); + /* Header */ if (count < sizeof(header)) return -EINVAL; ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &header, 0, sizeof(header)); if (ret) - goto out; + return ret; /* * Streaming SVE data is always stored and presented in SVE format. @@ -915,7 +917,21 @@ static int sve_set_common(struct task_struct *target, ret = vec_set_vector_length(target, type, header.vl, ((unsigned long)header.flags & ~SVE_PT_REGS_MASK) << 16); if (ret) - goto out; + return ret; + + /* Allocate SME storage if necessary, preserving any existing ZA/ZT state */ + if (type == ARM64_VEC_SME) { + sme_alloc(target, false); + if (!target->thread.sme_state) + return -ENOMEM; + } + + /* Allocate SVE storage if necessary, zeroing any existing SVE state */ + if (!fpsimd) { + sve_alloc(target, true); + if (!target->thread.sve_state) + return -ENOMEM; + } /* Actual VL set may be less than the user asked for: */ vq = sve_vq_from_vl(task_get_vl(target, type)); @@ -925,21 +941,15 @@ static int sve_set_common(struct task_struct *target, switch (type) { case ARM64_VEC_SVE: target->thread.svcr &= ~SVCR_SM_MASK; + set_tsk_thread_flag(target, TIF_SVE); break; case ARM64_VEC_SME: target->thread.svcr |= SVCR_SM_MASK; - - /* - * Disable traps and ensure there is SME storage but - * preserve any currently set values in ZA/ZT. - */ - sme_alloc(target, false); set_tsk_thread_flag(target, TIF_SME); break; default: WARN_ON_ONCE(1); - ret = -EINVAL; - goto out; + return -EINVAL; } } @@ -955,37 +965,20 @@ static int sve_set_common(struct task_struct *target, target->thread.fp_type = FP_STATE_FPSIMD; ret = __fpr_set(target, regset, pos, count, kbuf, ubuf, SVE_PT_FPSIMD_OFFSET); - goto out; + return ret; } /* Otherwise: no registers or full SVE case. */ + target->thread.fp_type = FP_STATE_SVE; + /* * If setting a different VL from the requested VL and there is * register data, the data layout will be wrong: don't even * try to set the registers in this case. */ - if (count && vq != sve_vq_from_vl(header.vl)) { - ret = -EIO; - goto out; - } - - /* Always zero SVE state */ - sve_alloc(target, true); - if (!target->thread.sve_state) { - ret = -ENOMEM; - clear_tsk_thread_flag(target, TIF_SVE); - target->thread.fp_type = FP_STATE_FPSIMD; - goto out; - } - - /* - * Only enable SVE if we are configuring normal SVE, a system with - * streaming SVE may not have normal SVE. - */ - if (type == ARM64_VEC_SVE) - set_tsk_thread_flag(target, TIF_SVE); - target->thread.fp_type = FP_STATE_SVE; + if (count && vq != sve_vq_from_vl(header.vl)) + return -EIO; BUILD_BUG_ON(SVE_PT_SVE_OFFSET != sizeof(header)); start = SVE_PT_SVE_OFFSET; @@ -994,7 +987,7 @@ static int sve_set_common(struct task_struct *target, target->thread.sve_state, start, end); if (ret) - goto out; + return ret; start = end; end = SVE_PT_SVE_FPSR_OFFSET(vq); @@ -1010,8 +1003,6 @@ static int sve_set_common(struct task_struct *target, &target->thread.uw.fpsimd_state.fpsr, start, end); -out: - fpsimd_flush_task_state(target); return ret; } From 6861f3cdb98bb677ab4ca152ab50a7b7ddc1ecd8 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Thu, 8 May 2025 14:26:40 +0100 Subject: [PATCH 68/85] arm64/fpsimd: Allow CONFIG_ARM64_SME to be selected commit 33c4618d0ac04139b737dcc0870b9dc3ed4dd170 upstream. Now that the known issues with SME have been addressed, allow SME to be selected. Signed-off-by: Mark Rutland Cc: Catalin Marinas Cc: Daniel Kiss Cc: David Spickett Cc: Fuad Tabba Cc: Luis Machado Cc: Marc Zyngier Cc: Mark Brown Cc: Richard Sandiford Cc: Sander De Smalen Cc: Tamas Petz Cc: Todd Kjos Cc: Will Deacon Cc: Yury Khrustalev Tested-By: Luis Machado Link: https://lore.kernel.org/r/20250508132644.1395904-21-mark.rutland@arm.com Signed-off-by: Will Deacon Signed-off-by: Bin Huang --- arch/arm64/Kconfig | 1 - 1 file changed, 1 deletion(-) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 67aa4825271b..ebe18a411488 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -2244,7 +2244,6 @@ config ARM64_SME bool "ARM Scalable Matrix Extension support" default y depends on ARM64_SVE - depends on BROKEN help The Scalable Matrix Extension (SME) is an extension to the AArch64 execution state which utilises a substantial subset of the SVE From 6af2ed5a10ef9f15e92b42c12d4a74583b2c4a24 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Thu, 8 May 2025 14:26:42 +0100 Subject: [PATCH 69/85] kselftest/arm64: tpidr2: Adjust to new clone() behaviour commit be45e63f79ecfea8373f18f50330838d77553a6b upstream. In order to fix an ABI problem, we recently changed the way that a clone() syscall manipulates TPIDR2 and PSTATE.ZA. Historically the child would inherit the parent's TPIDR2 value unless CLONE_SETTLS was set, and now the child will inherit the parent's TPIDR2 value unless CLONE_VM is set. Update the tpidr2 test for the new behaviour. Signed-off-by: Mark Rutland Cc: Catalin Marinas Cc: Daniel Kiss Cc: Marc Zyngier Cc: Mark Brown Cc: Richard Sandiford Cc: Sander De Smalen Cc: Tamas Petz Cc: Will Deacon Cc: Yury Khrustalev Link: https://lore.kernel.org/r/20250508132644.1395904-23-mark.rutland@arm.com Signed-off-by: Will Deacon Signed-off-by: Bin Huang --- tools/testing/selftests/arm64/abi/tpidr2.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/arm64/abi/tpidr2.c b/tools/testing/selftests/arm64/abi/tpidr2.c index 351a098b503a..a03981b51500 100644 --- a/tools/testing/selftests/arm64/abi/tpidr2.c +++ b/tools/testing/selftests/arm64/abi/tpidr2.c @@ -169,8 +169,10 @@ static int sys_clone(unsigned long clone_flags, unsigned long newsp, child_tidptr); } +#define __STACK_SIZE (8 * 1024 * 1024) + /* - * If we clone with CLONE_SETTLS then the value in the parent should + * If we clone with CLONE_VM then the value in the parent should * be unchanged and the child should start with zero and be able to * set its own value. */ @@ -179,11 +181,19 @@ static int write_clone_read(void) int parent_tid, child_tid; pid_t parent, waiting; int ret, status; + void *stack; parent = getpid(); set_tpidr2(parent); - ret = sys_clone(CLONE_SETTLS, 0, &parent_tid, 0, &child_tid); + stack = malloc(__STACK_SIZE); + if (!stack) { + putstr("# malloc() failed\n"); + return 0; + } + + ret = sys_clone(CLONE_VM, (unsigned long)stack + __STACK_SIZE, + &parent_tid, 0, &child_tid); if (ret == -1) { putstr("# clone() failed\n"); putnum(errno); From fc3919acc5388c7e488cb41737476c0c5207039a Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Mon, 22 Jan 2024 21:05:03 +0000 Subject: [PATCH 70/85] kselftest/arm64: Test that ptrace takes effect in the target process commit c745b15c1f9cea5680c2906ae868302108f8daf0 upstream. While we have test coverage for the ptrace interface in our selftests the current programs have a number of gaps. The testing is done per regset so does not cover interactions and at no point do any of the tests actually run the traced processes meaning that there is no validation that anything we read or write corresponds to register values the process actually sees. Let's add a new program which attempts to cover these gaps. Each test we do performs a single ptrace write. For each test we generate some random initial register data in memory and then fork() and trace a child. The child will load the generated data into the registers then trigger a breakpoint. The parent waits for the breakpoint then reads the entire child register state via ptrace, verifying that the values expected were actually loaded by the child. It then does the write being tested and resumes the child. Once resumed the child saves the register state it sees to memory and executes another breakpoint. The parent uses process_vm_readv() to get these values from the child and verifies that the values were as expected before cleaning up the child. We generate configurations with combinations of vector lengths and SVCR values and then try every ptrace write which will implement the transition we generated. In order to control execution time (especially in emulation) we only cover the minimum and maximum VL for each of SVE and SME, this will ensure we generate both increasing and decreasing changes in vector length. In order to provide a baseline test we also check the case where we resume the child without doing a ptrace write. In order to simplify the generation of the test count for kselftest we will report but skip a substantial number of tests that can't actually be expressed via a single ptrace write, several times more than we actually run. This is noisy and will add some overhead but is very much simpler so is probably worth the tradeoff. Signed-off-by: Mark Brown Link: https://lore.kernel.org/r/20240122-arm64-test-ptrace-regs-v1-1-0897f822d73e@kernel.org Signed-off-by: Catalin Marinas Signed-off-by: Bin Huang --- tools/testing/selftests/arm64/fp/.gitignore | 1 + tools/testing/selftests/arm64/fp/Makefile | 5 +- .../selftests/arm64/fp/fp-ptrace-asm.S | 279 +++ tools/testing/selftests/arm64/fp/fp-ptrace.c | 1503 +++++++++++++++++ tools/testing/selftests/arm64/fp/fp-ptrace.h | 13 + 5 files changed, 1800 insertions(+), 1 deletion(-) create mode 100644 tools/testing/selftests/arm64/fp/fp-ptrace-asm.S create mode 100644 tools/testing/selftests/arm64/fp/fp-ptrace.c create mode 100644 tools/testing/selftests/arm64/fp/fp-ptrace.h diff --git a/tools/testing/selftests/arm64/fp/.gitignore b/tools/testing/selftests/arm64/fp/.gitignore index ebc86757bdd8..00e52c966281 100644 --- a/tools/testing/selftests/arm64/fp/.gitignore +++ b/tools/testing/selftests/arm64/fp/.gitignore @@ -1,4 +1,5 @@ fp-pidbench +fp-ptrace fp-stress fpsimd-test rdvl-sme diff --git a/tools/testing/selftests/arm64/fp/Makefile b/tools/testing/selftests/arm64/fp/Makefile index b413b0af07f9..55d4f00d9e8e 100644 --- a/tools/testing/selftests/arm64/fp/Makefile +++ b/tools/testing/selftests/arm64/fp/Makefile @@ -5,7 +5,9 @@ top_srcdir = $(realpath ../../../../../) CFLAGS += $(KHDR_INCLUDES) -TEST_GEN_PROGS := fp-stress \ +TEST_GEN_PROGS := \ + fp-ptrace \ + fp-stress \ sve-ptrace sve-probe-vls \ vec-syscfg \ za-fork za-ptrace @@ -24,6 +26,7 @@ EXTRA_CLEAN += $(OUTPUT)/asm-utils.o $(OUTPUT)/rdvl.o $(OUTPUT)/za-fork-asm.o # Build with nolibc to avoid effects due to libc's clone() support $(OUTPUT)/fp-pidbench: fp-pidbench.S $(OUTPUT)/asm-utils.o $(CC) -nostdlib $^ -o $@ +$(OUTPUT)/fp-ptrace: fp-ptrace.c fp-ptrace-asm.S $(OUTPUT)/fpsimd-test: fpsimd-test.S $(OUTPUT)/asm-utils.o $(CC) -nostdlib $^ -o $@ $(OUTPUT)/rdvl-sve: rdvl-sve.c $(OUTPUT)/rdvl.o diff --git a/tools/testing/selftests/arm64/fp/fp-ptrace-asm.S b/tools/testing/selftests/arm64/fp/fp-ptrace-asm.S new file mode 100644 index 000000000000..7ad59d92d02b --- /dev/null +++ b/tools/testing/selftests/arm64/fp/fp-ptrace-asm.S @@ -0,0 +1,279 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (C) 2021-3 ARM Limited. +// +// Assembly portion of the FP ptrace test + +// +// Load values from memory into registers, break on a breakpoint, then +// break on a further breakpoint +// + +#include "fp-ptrace.h" +#include "sme-inst.h" + +.arch_extension sve + +// Load and save register values with pauses for ptrace +// +// x0 - SVE in use +// x1 - SME in use +// x2 - SME2 in use +// x3 - FA64 supported + +.globl load_and_save +load_and_save: + stp x11, x12, [sp, #-0x10]! + + // This should be redundant in the SVE case + ldr x7, =v_in + ldp q0, q1, [x7] + ldp q2, q3, [x7, #16 * 2] + ldp q4, q5, [x7, #16 * 4] + ldp q6, q7, [x7, #16 * 6] + ldp q8, q9, [x7, #16 * 8] + ldp q10, q11, [x7, #16 * 10] + ldp q12, q13, [x7, #16 * 12] + ldp q14, q15, [x7, #16 * 14] + ldp q16, q17, [x7, #16 * 16] + ldp q18, q19, [x7, #16 * 18] + ldp q20, q21, [x7, #16 * 20] + ldp q22, q23, [x7, #16 * 22] + ldp q24, q25, [x7, #16 * 24] + ldp q26, q27, [x7, #16 * 26] + ldp q28, q29, [x7, #16 * 28] + ldp q30, q31, [x7, #16 * 30] + + // SME? + cbz x1, check_sve_in + + adrp x7, svcr_in + ldr x7, [x7, :lo12:svcr_in] + // SVCR is 0 by default, avoid triggering SME if not in use + cbz x7, check_sve_in + msr S3_3_C4_C2_2, x7 + + // ZA? + tbz x7, #SVCR_ZA_SHIFT, check_sm_in + rdsvl 11, 1 + mov w12, #0 + ldr x6, =za_in +1: _ldr_za 12, 6 + add x6, x6, x11 + add x12, x12, #1 + cmp x11, x12 + bne 1b + + // ZT? + cbz x2, check_sm_in + adrp x6, zt_in + add x6, x6, :lo12:zt_in + _ldr_zt 6 + + // In streaming mode? +check_sm_in: + tbz x7, #SVCR_SM_SHIFT, check_sve_in + mov x4, x3 // Load FFR if we have FA64 + b load_sve + + // SVE? +check_sve_in: + cbz x0, wait_for_writes + mov x4, #1 + +load_sve: + ldr x7, =z_in + ldr z0, [x7, #0, MUL VL] + ldr z1, [x7, #1, MUL VL] + ldr z2, [x7, #2, MUL VL] + ldr z3, [x7, #3, MUL VL] + ldr z4, [x7, #4, MUL VL] + ldr z5, [x7, #5, MUL VL] + ldr z6, [x7, #6, MUL VL] + ldr z7, [x7, #7, MUL VL] + ldr z8, [x7, #8, MUL VL] + ldr z9, [x7, #9, MUL VL] + ldr z10, [x7, #10, MUL VL] + ldr z11, [x7, #11, MUL VL] + ldr z12, [x7, #12, MUL VL] + ldr z13, [x7, #13, MUL VL] + ldr z14, [x7, #14, MUL VL] + ldr z15, [x7, #15, MUL VL] + ldr z16, [x7, #16, MUL VL] + ldr z17, [x7, #17, MUL VL] + ldr z18, [x7, #18, MUL VL] + ldr z19, [x7, #19, MUL VL] + ldr z20, [x7, #20, MUL VL] + ldr z21, [x7, #21, MUL VL] + ldr z22, [x7, #22, MUL VL] + ldr z23, [x7, #23, MUL VL] + ldr z24, [x7, #24, MUL VL] + ldr z25, [x7, #25, MUL VL] + ldr z26, [x7, #26, MUL VL] + ldr z27, [x7, #27, MUL VL] + ldr z28, [x7, #28, MUL VL] + ldr z29, [x7, #29, MUL VL] + ldr z30, [x7, #30, MUL VL] + ldr z31, [x7, #31, MUL VL] + + // FFR is not present in base SME + cbz x4, 1f + ldr x7, =ffr_in + ldr p0, [x7] + ldr x7, [x7, #0] + cbz x7, 1f + wrffr p0.b +1: + + ldr x7, =p_in + ldr p0, [x7, #0, MUL VL] + ldr p1, [x7, #1, MUL VL] + ldr p2, [x7, #2, MUL VL] + ldr p3, [x7, #3, MUL VL] + ldr p4, [x7, #4, MUL VL] + ldr p5, [x7, #5, MUL VL] + ldr p6, [x7, #6, MUL VL] + ldr p7, [x7, #7, MUL VL] + ldr p8, [x7, #8, MUL VL] + ldr p9, [x7, #9, MUL VL] + ldr p10, [x7, #10, MUL VL] + ldr p11, [x7, #11, MUL VL] + ldr p12, [x7, #12, MUL VL] + ldr p13, [x7, #13, MUL VL] + ldr p14, [x7, #14, MUL VL] + ldr p15, [x7, #15, MUL VL] + +wait_for_writes: + // Wait for the parent + brk #0 + + // Save values + ldr x7, =v_out + stp q0, q1, [x7] + stp q2, q3, [x7, #16 * 2] + stp q4, q5, [x7, #16 * 4] + stp q6, q7, [x7, #16 * 6] + stp q8, q9, [x7, #16 * 8] + stp q10, q11, [x7, #16 * 10] + stp q12, q13, [x7, #16 * 12] + stp q14, q15, [x7, #16 * 14] + stp q16, q17, [x7, #16 * 16] + stp q18, q19, [x7, #16 * 18] + stp q20, q21, [x7, #16 * 20] + stp q22, q23, [x7, #16 * 22] + stp q24, q25, [x7, #16 * 24] + stp q26, q27, [x7, #16 * 26] + stp q28, q29, [x7, #16 * 28] + stp q30, q31, [x7, #16 * 30] + + // SME? + cbz x1, check_sve_out + + rdsvl 11, 1 + adrp x6, sme_vl_out + str x11, [x6, :lo12:sme_vl_out] + + mrs x7, S3_3_C4_C2_2 + adrp x6, svcr_out + str x7, [x6, :lo12:svcr_out] + + // ZA? + tbz x7, #SVCR_ZA_SHIFT, check_sm_out + mov w12, #0 + ldr x6, =za_out +1: _str_za 12, 6 + add x6, x6, x11 + add x12, x12, #1 + cmp x11, x12 + bne 1b + + // ZT? + cbz x2, check_sm_out + adrp x6, zt_out + add x6, x6, :lo12:zt_out + _str_zt 6 + + // In streaming mode? +check_sm_out: + tbz x7, #SVCR_SM_SHIFT, check_sve_out + mov x4, x3 // FFR? + b read_sve + + // SVE? +check_sve_out: + cbz x0, wait_for_reads + mov x4, #1 + + rdvl x7, #1 + adrp x6, sve_vl_out + str x7, [x6, :lo12:sve_vl_out] + +read_sve: + ldr x7, =z_out + str z0, [x7, #0, MUL VL] + str z1, [x7, #1, MUL VL] + str z2, [x7, #2, MUL VL] + str z3, [x7, #3, MUL VL] + str z4, [x7, #4, MUL VL] + str z5, [x7, #5, MUL VL] + str z6, [x7, #6, MUL VL] + str z7, [x7, #7, MUL VL] + str z8, [x7, #8, MUL VL] + str z9, [x7, #9, MUL VL] + str z10, [x7, #10, MUL VL] + str z11, [x7, #11, MUL VL] + str z12, [x7, #12, MUL VL] + str z13, [x7, #13, MUL VL] + str z14, [x7, #14, MUL VL] + str z15, [x7, #15, MUL VL] + str z16, [x7, #16, MUL VL] + str z17, [x7, #17, MUL VL] + str z18, [x7, #18, MUL VL] + str z19, [x7, #19, MUL VL] + str z20, [x7, #20, MUL VL] + str z21, [x7, #21, MUL VL] + str z22, [x7, #22, MUL VL] + str z23, [x7, #23, MUL VL] + str z24, [x7, #24, MUL VL] + str z25, [x7, #25, MUL VL] + str z26, [x7, #26, MUL VL] + str z27, [x7, #27, MUL VL] + str z28, [x7, #28, MUL VL] + str z29, [x7, #29, MUL VL] + str z30, [x7, #30, MUL VL] + str z31, [x7, #31, MUL VL] + + ldr x7, =p_out + str p0, [x7, #0, MUL VL] + str p1, [x7, #1, MUL VL] + str p2, [x7, #2, MUL VL] + str p3, [x7, #3, MUL VL] + str p4, [x7, #4, MUL VL] + str p5, [x7, #5, MUL VL] + str p6, [x7, #6, MUL VL] + str p7, [x7, #7, MUL VL] + str p8, [x7, #8, MUL VL] + str p9, [x7, #9, MUL VL] + str p10, [x7, #10, MUL VL] + str p11, [x7, #11, MUL VL] + str p12, [x7, #12, MUL VL] + str p13, [x7, #13, MUL VL] + str p14, [x7, #14, MUL VL] + str p15, [x7, #15, MUL VL] + + // Only save FFR if it exists + cbz x4, wait_for_reads + ldr x7, =ffr_out + rdffr p0.b + str p0, [x7] + +wait_for_reads: + // Wait for the parent + brk #0 + + // Ensure we don't leave ourselves in streaming mode + cbz x1, out + msr S3_3_C4_C2_2, xzr + +out: + ldp x11, x12, [sp, #-0x10] + ret diff --git a/tools/testing/selftests/arm64/fp/fp-ptrace.c b/tools/testing/selftests/arm64/fp/fp-ptrace.c new file mode 100644 index 000000000000..c7ceafe5f471 --- /dev/null +++ b/tools/testing/selftests/arm64/fp/fp-ptrace.c @@ -0,0 +1,1503 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2023 ARM Limited. + * Original author: Mark Brown + */ + +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +#include "../../kselftest.h" + +#include "fp-ptrace.h" + +/* and don't like each other, so: */ +#ifndef NT_ARM_SVE +#define NT_ARM_SVE 0x405 +#endif + +#ifndef NT_ARM_SSVE +#define NT_ARM_SSVE 0x40b +#endif + +#ifndef NT_ARM_ZA +#define NT_ARM_ZA 0x40c +#endif + +#ifndef NT_ARM_ZT +#define NT_ARM_ZT 0x40d +#endif + +#define ARCH_VQ_MAX 256 + +/* VL 128..2048 in powers of 2 */ +#define MAX_NUM_VLS 5 + +#define NUM_FPR 32 +__uint128_t v_in[NUM_FPR]; +__uint128_t v_expected[NUM_FPR]; +__uint128_t v_out[NUM_FPR]; + +char z_in[__SVE_ZREGS_SIZE(ARCH_VQ_MAX)]; +char z_expected[__SVE_ZREGS_SIZE(ARCH_VQ_MAX)]; +char z_out[__SVE_ZREGS_SIZE(ARCH_VQ_MAX)]; + +char p_in[__SVE_PREGS_SIZE(ARCH_VQ_MAX)]; +char p_expected[__SVE_PREGS_SIZE(ARCH_VQ_MAX)]; +char p_out[__SVE_PREGS_SIZE(ARCH_VQ_MAX)]; + +char ffr_in[__SVE_PREG_SIZE(ARCH_VQ_MAX)]; +char ffr_expected[__SVE_PREG_SIZE(ARCH_VQ_MAX)]; +char ffr_out[__SVE_PREG_SIZE(ARCH_VQ_MAX)]; + +char za_in[ZA_SIG_REGS_SIZE(ARCH_VQ_MAX)]; +char za_expected[ZA_SIG_REGS_SIZE(ARCH_VQ_MAX)]; +char za_out[ZA_SIG_REGS_SIZE(ARCH_VQ_MAX)]; + +char zt_in[ZT_SIG_REG_BYTES]; +char zt_expected[ZT_SIG_REG_BYTES]; +char zt_out[ZT_SIG_REG_BYTES]; + +uint64_t sve_vl_out; +uint64_t sme_vl_out; +uint64_t svcr_in, svcr_expected, svcr_out; + +void load_and_save(int sve, int sme, int sme2, int fa64); + +static bool got_alarm; + +static void handle_alarm(int sig, siginfo_t *info, void *context) +{ + got_alarm = true; +} + +#ifdef CONFIG_CPU_BIG_ENDIAN +static __uint128_t arm64_cpu_to_le128(__uint128_t x) +{ + u64 a = swab64(x); + u64 b = swab64(x >> 64); + + return ((__uint128_t)a << 64) | b; +} +#else +static __uint128_t arm64_cpu_to_le128(__uint128_t x) +{ + return x; +} +#endif + +#define arm64_le128_to_cpu(x) arm64_cpu_to_le128(x) + +static bool sve_supported(void) +{ + return getauxval(AT_HWCAP) & HWCAP_SVE; +} + +static bool sme_supported(void) +{ + return getauxval(AT_HWCAP2) & HWCAP2_SME; +} + +static bool sme2_supported(void) +{ + return getauxval(AT_HWCAP2) & HWCAP2_SME2; +} + +static bool fa64_supported(void) +{ + return getauxval(AT_HWCAP2) & HWCAP2_SME_FA64; +} + +static bool compare_buffer(const char *name, void *out, + void *expected, size_t size) +{ + void *tmp; + + if (memcmp(out, expected, size) == 0) + return true; + + ksft_print_msg("Mismatch in %s\n", name); + + /* Did we just get zeros back? */ + tmp = malloc(size); + if (!tmp) { + ksft_print_msg("OOM allocating %lu bytes for %s\n", + size, name); + ksft_exit_fail(); + } + memset(tmp, 0, size); + + if (memcmp(out, tmp, size) == 0) + ksft_print_msg("%s is zero\n", name); + + free(tmp); + + return false; +} + +struct test_config { + int sve_vl_in; + int sve_vl_expected; + int sme_vl_in; + int sme_vl_expected; + int svcr_in; + int svcr_expected; +}; + +struct test_definition { + const char *name; + bool sve_vl_change; + bool (*supported)(struct test_config *config); + void (*set_expected_values)(struct test_config *config); + void (*modify_values)(pid_t child, struct test_config *test_config); +}; + +static int vl_in(struct test_config *config) +{ + int vl; + + if (config->svcr_in & SVCR_SM) + vl = config->sme_vl_in; + else + vl = config->sve_vl_in; + + return vl; +} + +static int vl_expected(struct test_config *config) +{ + int vl; + + if (config->svcr_expected & SVCR_SM) + vl = config->sme_vl_expected; + else + vl = config->sve_vl_expected; + + return vl; +} + +static void run_child(struct test_config *config) +{ + int ret; + + /* Let the parent attach to us */ + ret = ptrace(PTRACE_TRACEME, 0, 0, 0); + if (ret < 0) + ksft_exit_fail_msg("PTRACE_TRACEME failed: %s (%d)\n", + strerror(errno), errno); + + /* VL setup */ + if (sve_supported()) { + ret = prctl(PR_SVE_SET_VL, config->sve_vl_in); + if (ret != config->sve_vl_in) { + ksft_print_msg("Failed to set SVE VL %d: %d\n", + config->sve_vl_in, ret); + } + } + + if (sme_supported()) { + ret = prctl(PR_SME_SET_VL, config->sme_vl_in); + if (ret != config->sme_vl_in) { + ksft_print_msg("Failed to set SME VL %d: %d\n", + config->sme_vl_in, ret); + } + } + + /* Load values and wait for the parent */ + load_and_save(sve_supported(), sme_supported(), + sme2_supported(), fa64_supported()); + + exit(0); +} + +static void read_one_child_regs(pid_t child, char *name, + struct iovec *iov_parent, + struct iovec *iov_child) +{ + int len = iov_parent->iov_len; + int ret; + + ret = process_vm_readv(child, iov_parent, 1, iov_child, 1, 0); + if (ret == -1) + ksft_print_msg("%s read failed: %s (%d)\n", + name, strerror(errno), errno); + else if (ret != len) + ksft_print_msg("Short read of %s: %d\n", name, ret); +} + +static void read_child_regs(pid_t child) +{ + struct iovec iov_parent, iov_child; + + /* + * Since the child fork()ed from us the buffer addresses are + * the same in parent and child. + */ + iov_parent.iov_base = &v_out; + iov_parent.iov_len = sizeof(v_out); + iov_child.iov_base = &v_out; + iov_child.iov_len = sizeof(v_out); + read_one_child_regs(child, "FPSIMD", &iov_parent, &iov_child); + + if (sve_supported() || sme_supported()) { + iov_parent.iov_base = &sve_vl_out; + iov_parent.iov_len = sizeof(sve_vl_out); + iov_child.iov_base = &sve_vl_out; + iov_child.iov_len = sizeof(sve_vl_out); + read_one_child_regs(child, "SVE VL", &iov_parent, &iov_child); + + iov_parent.iov_base = &z_out; + iov_parent.iov_len = sizeof(z_out); + iov_child.iov_base = &z_out; + iov_child.iov_len = sizeof(z_out); + read_one_child_regs(child, "Z", &iov_parent, &iov_child); + + iov_parent.iov_base = &p_out; + iov_parent.iov_len = sizeof(p_out); + iov_child.iov_base = &p_out; + iov_child.iov_len = sizeof(p_out); + read_one_child_regs(child, "P", &iov_parent, &iov_child); + + iov_parent.iov_base = &ffr_out; + iov_parent.iov_len = sizeof(ffr_out); + iov_child.iov_base = &ffr_out; + iov_child.iov_len = sizeof(ffr_out); + read_one_child_regs(child, "FFR", &iov_parent, &iov_child); + } + + if (sme_supported()) { + iov_parent.iov_base = &sme_vl_out; + iov_parent.iov_len = sizeof(sme_vl_out); + iov_child.iov_base = &sme_vl_out; + iov_child.iov_len = sizeof(sme_vl_out); + read_one_child_regs(child, "SME VL", &iov_parent, &iov_child); + + iov_parent.iov_base = &svcr_out; + iov_parent.iov_len = sizeof(svcr_out); + iov_child.iov_base = &svcr_out; + iov_child.iov_len = sizeof(svcr_out); + read_one_child_regs(child, "SVCR", &iov_parent, &iov_child); + + iov_parent.iov_base = &za_out; + iov_parent.iov_len = sizeof(za_out); + iov_child.iov_base = &za_out; + iov_child.iov_len = sizeof(za_out); + read_one_child_regs(child, "ZA", &iov_parent, &iov_child); + } + + if (sme2_supported()) { + iov_parent.iov_base = &zt_out; + iov_parent.iov_len = sizeof(zt_out); + iov_child.iov_base = &zt_out; + iov_child.iov_len = sizeof(zt_out); + read_one_child_regs(child, "ZT", &iov_parent, &iov_child); + } +} + +static bool continue_breakpoint(pid_t child, + enum __ptrace_request restart_type) +{ + struct user_pt_regs pt_regs; + struct iovec iov; + int ret; + + /* Get PC */ + iov.iov_base = &pt_regs; + iov.iov_len = sizeof(pt_regs); + ret = ptrace(PTRACE_GETREGSET, child, NT_PRSTATUS, &iov); + if (ret < 0) { + ksft_print_msg("Failed to get PC: %s (%d)\n", + strerror(errno), errno); + return false; + } + + /* Skip over the BRK */ + pt_regs.pc += 4; + ret = ptrace(PTRACE_SETREGSET, child, NT_PRSTATUS, &iov); + if (ret < 0) { + ksft_print_msg("Failed to skip BRK: %s (%d)\n", + strerror(errno), errno); + return false; + } + + /* Restart */ + ret = ptrace(restart_type, child, 0, 0); + if (ret < 0) { + ksft_print_msg("Failed to restart child: %s (%d)\n", + strerror(errno), errno); + return false; + } + + return true; +} + +static bool check_ptrace_values_sve(pid_t child, struct test_config *config) +{ + struct user_sve_header *sve; + struct user_fpsimd_state *fpsimd; + struct iovec iov; + int ret, vq; + bool pass = true; + + if (!sve_supported()) + return true; + + vq = __sve_vq_from_vl(config->sve_vl_in); + + iov.iov_len = SVE_PT_SVE_OFFSET + SVE_PT_SVE_SIZE(vq, SVE_PT_REGS_SVE); + iov.iov_base = malloc(iov.iov_len); + if (!iov.iov_base) { + ksft_print_msg("OOM allocating %lu byte SVE buffer\n", + iov.iov_len); + return false; + } + + ret = ptrace(PTRACE_GETREGSET, child, NT_ARM_SVE, &iov); + if (ret != 0) { + ksft_print_msg("Failed to read initial SVE: %s (%d)\n", + strerror(errno), errno); + pass = false; + goto out; + } + + sve = iov.iov_base; + + if (sve->vl != config->sve_vl_in) { + ksft_print_msg("Mismatch in initial SVE VL: %d != %d\n", + sve->vl, config->sve_vl_in); + pass = false; + } + + /* If we are in streaming mode we should just read FPSIMD */ + if ((config->svcr_in & SVCR_SM) && (sve->flags & SVE_PT_REGS_SVE)) { + ksft_print_msg("NT_ARM_SVE reports SVE with PSTATE.SM\n"); + pass = false; + } + + if (sve->size != SVE_PT_SIZE(vq, sve->flags)) { + ksft_print_msg("Mismatch in SVE header size: %d != %lu\n", + sve->size, SVE_PT_SIZE(vq, sve->flags)); + pass = false; + } + + /* The registers might be in completely different formats! */ + if (sve->flags & SVE_PT_REGS_SVE) { + if (!compare_buffer("initial SVE Z", + iov.iov_base + SVE_PT_SVE_ZREG_OFFSET(vq, 0), + z_in, SVE_PT_SVE_ZREGS_SIZE(vq))) + pass = false; + + if (!compare_buffer("initial SVE P", + iov.iov_base + SVE_PT_SVE_PREG_OFFSET(vq, 0), + p_in, SVE_PT_SVE_PREGS_SIZE(vq))) + pass = false; + + if (!compare_buffer("initial SVE FFR", + iov.iov_base + SVE_PT_SVE_FFR_OFFSET(vq), + ffr_in, SVE_PT_SVE_PREG_SIZE(vq))) + pass = false; + } else { + fpsimd = iov.iov_base + SVE_PT_FPSIMD_OFFSET; + if (!compare_buffer("initial V via SVE", &fpsimd->vregs[0], + v_in, sizeof(v_in))) + pass = false; + } + +out: + free(iov.iov_base); + return pass; +} + +static bool check_ptrace_values_ssve(pid_t child, struct test_config *config) +{ + struct user_sve_header *sve; + struct user_fpsimd_state *fpsimd; + struct iovec iov; + int ret, vq; + bool pass = true; + + if (!sme_supported()) + return true; + + vq = __sve_vq_from_vl(config->sme_vl_in); + + iov.iov_len = SVE_PT_SVE_OFFSET + SVE_PT_SVE_SIZE(vq, SVE_PT_REGS_SVE); + iov.iov_base = malloc(iov.iov_len); + if (!iov.iov_base) { + ksft_print_msg("OOM allocating %lu byte SSVE buffer\n", + iov.iov_len); + return false; + } + + ret = ptrace(PTRACE_GETREGSET, child, NT_ARM_SSVE, &iov); + if (ret != 0) { + ksft_print_msg("Failed to read initial SSVE: %s (%d)\n", + strerror(errno), errno); + pass = false; + goto out; + } + + sve = iov.iov_base; + + if (sve->vl != config->sme_vl_in) { + ksft_print_msg("Mismatch in initial SSVE VL: %d != %d\n", + sve->vl, config->sme_vl_in); + pass = false; + } + + if ((config->svcr_in & SVCR_SM) && !(sve->flags & SVE_PT_REGS_SVE)) { + ksft_print_msg("NT_ARM_SSVE reports FPSIMD with PSTATE.SM\n"); + pass = false; + } + + if (sve->size != SVE_PT_SIZE(vq, sve->flags)) { + ksft_print_msg("Mismatch in SSVE header size: %d != %lu\n", + sve->size, SVE_PT_SIZE(vq, sve->flags)); + pass = false; + } + + /* The registers might be in completely different formats! */ + if (sve->flags & SVE_PT_REGS_SVE) { + if (!compare_buffer("initial SSVE Z", + iov.iov_base + SVE_PT_SVE_ZREG_OFFSET(vq, 0), + z_in, SVE_PT_SVE_ZREGS_SIZE(vq))) + pass = false; + + if (!compare_buffer("initial SSVE P", + iov.iov_base + SVE_PT_SVE_PREG_OFFSET(vq, 0), + p_in, SVE_PT_SVE_PREGS_SIZE(vq))) + pass = false; + + if (!compare_buffer("initial SSVE FFR", + iov.iov_base + SVE_PT_SVE_FFR_OFFSET(vq), + ffr_in, SVE_PT_SVE_PREG_SIZE(vq))) + pass = false; + } else { + fpsimd = iov.iov_base + SVE_PT_FPSIMD_OFFSET; + if (!compare_buffer("initial V via SSVE", + &fpsimd->vregs[0], v_in, sizeof(v_in))) + pass = false; + } + +out: + free(iov.iov_base); + return pass; +} + +static bool check_ptrace_values_za(pid_t child, struct test_config *config) +{ + struct user_za_header *za; + struct iovec iov; + int ret, vq; + bool pass = true; + + if (!sme_supported()) + return true; + + vq = __sve_vq_from_vl(config->sme_vl_in); + + iov.iov_len = ZA_SIG_CONTEXT_SIZE(vq); + iov.iov_base = malloc(iov.iov_len); + if (!iov.iov_base) { + ksft_print_msg("OOM allocating %lu byte ZA buffer\n", + iov.iov_len); + return false; + } + + ret = ptrace(PTRACE_GETREGSET, child, NT_ARM_ZA, &iov); + if (ret != 0) { + ksft_print_msg("Failed to read initial ZA: %s (%d)\n", + strerror(errno), errno); + pass = false; + goto out; + } + + za = iov.iov_base; + + if (za->vl != config->sme_vl_in) { + ksft_print_msg("Mismatch in initial SME VL: %d != %d\n", + za->vl, config->sme_vl_in); + pass = false; + } + + /* If PSTATE.ZA is not set we should just read the header */ + if (config->svcr_in & SVCR_ZA) { + if (za->size != ZA_PT_SIZE(vq)) { + ksft_print_msg("Unexpected ZA ptrace read size: %d != %lu\n", + za->size, ZA_PT_SIZE(vq)); + pass = false; + } + + if (!compare_buffer("initial ZA", + iov.iov_base + ZA_PT_ZA_OFFSET, + za_in, ZA_PT_ZA_SIZE(vq))) + pass = false; + } else { + if (za->size != sizeof(*za)) { + ksft_print_msg("Unexpected ZA ptrace read size: %d != %lu\n", + za->size, sizeof(*za)); + pass = false; + } + } + +out: + free(iov.iov_base); + return pass; +} + +static bool check_ptrace_values_zt(pid_t child, struct test_config *config) +{ + uint8_t buf[512]; + struct iovec iov; + int ret; + + if (!sme2_supported()) + return true; + + iov.iov_base = &buf; + iov.iov_len = ZT_SIG_REG_BYTES; + ret = ptrace(PTRACE_GETREGSET, child, NT_ARM_ZT, &iov); + if (ret != 0) { + ksft_print_msg("Failed to read initial ZT: %s (%d)\n", + strerror(errno), errno); + return false; + } + + return compare_buffer("initial ZT", buf, zt_in, ZT_SIG_REG_BYTES); +} + + +static bool check_ptrace_values(pid_t child, struct test_config *config) +{ + bool pass = true; + struct user_fpsimd_state fpsimd; + struct iovec iov; + int ret; + + iov.iov_base = &fpsimd; + iov.iov_len = sizeof(fpsimd); + ret = ptrace(PTRACE_GETREGSET, child, NT_PRFPREG, &iov); + if (ret == 0) { + if (!compare_buffer("initial V", &fpsimd.vregs, v_in, + sizeof(v_in))) { + pass = false; + } + } else { + ksft_print_msg("Failed to read initial V: %s (%d)\n", + strerror(errno), errno); + pass = false; + } + + if (!check_ptrace_values_sve(child, config)) + pass = false; + + if (!check_ptrace_values_ssve(child, config)) + pass = false; + + if (!check_ptrace_values_za(child, config)) + pass = false; + + if (!check_ptrace_values_zt(child, config)) + pass = false; + + return pass; +} + +static bool run_parent(pid_t child, struct test_definition *test, + struct test_config *config) +{ + int wait_status, ret; + pid_t pid; + bool pass; + + /* Initial attach */ + while (1) { + pid = waitpid(child, &wait_status, 0); + if (pid < 0) { + if (errno == EINTR) + continue; + ksft_exit_fail_msg("waitpid() failed: %s (%d)\n", + strerror(errno), errno); + } + + if (pid == child) + break; + } + + if (WIFEXITED(wait_status)) { + ksft_print_msg("Child exited loading values with status %d\n", + WEXITSTATUS(wait_status)); + pass = false; + goto out; + } + + if (WIFSIGNALED(wait_status)) { + ksft_print_msg("Child died from signal %d loading values\n", + WTERMSIG(wait_status)); + pass = false; + goto out; + } + + /* Read initial values via ptrace */ + pass = check_ptrace_values(child, config); + + /* Do whatever writes we want to do */ + if (test->modify_values) + test->modify_values(child, config); + + if (!continue_breakpoint(child, PTRACE_CONT)) + goto cleanup; + + while (1) { + pid = waitpid(child, &wait_status, 0); + if (pid < 0) { + if (errno == EINTR) + continue; + ksft_exit_fail_msg("waitpid() failed: %s (%d)\n", + strerror(errno), errno); + } + + if (pid == child) + break; + } + + if (WIFEXITED(wait_status)) { + ksft_print_msg("Child exited saving values with status %d\n", + WEXITSTATUS(wait_status)); + pass = false; + goto out; + } + + if (WIFSIGNALED(wait_status)) { + ksft_print_msg("Child died from signal %d saving values\n", + WTERMSIG(wait_status)); + pass = false; + goto out; + } + + /* See what happened as a result */ + read_child_regs(child); + + if (!continue_breakpoint(child, PTRACE_DETACH)) + goto cleanup; + + /* The child should exit cleanly */ + got_alarm = false; + alarm(1); + while (1) { + if (got_alarm) { + ksft_print_msg("Wait for child timed out\n"); + goto cleanup; + } + + pid = waitpid(child, &wait_status, 0); + if (pid < 0) { + if (errno == EINTR) + continue; + ksft_exit_fail_msg("waitpid() failed: %s (%d)\n", + strerror(errno), errno); + } + + if (pid == child) + break; + } + alarm(0); + + if (got_alarm) { + ksft_print_msg("Timed out waiting for child\n"); + pass = false; + goto cleanup; + } + + if (pid == child && WIFSIGNALED(wait_status)) { + ksft_print_msg("Child died from signal %d cleaning up\n", + WTERMSIG(wait_status)); + pass = false; + goto out; + } + + if (pid == child && WIFEXITED(wait_status)) { + if (WEXITSTATUS(wait_status) != 0) { + ksft_print_msg("Child exited with error %d\n", + WEXITSTATUS(wait_status)); + pass = false; + } + } else { + ksft_print_msg("Child did not exit cleanly\n"); + pass = false; + goto cleanup; + } + + goto out; + +cleanup: + ret = kill(child, SIGKILL); + if (ret != 0) { + ksft_print_msg("kill() failed: %s (%d)\n", + strerror(errno), errno); + return false; + } + + while (1) { + pid = waitpid(child, &wait_status, 0); + if (pid < 0) { + if (errno == EINTR) + continue; + ksft_exit_fail_msg("waitpid() failed: %s (%d)\n", + strerror(errno), errno); + } + + if (pid == child) + break; + } + +out: + return pass; +} + +static void fill_random(void *buf, size_t size) +{ + int i; + uint32_t *lbuf = buf; + + /* random() returns a 32 bit number regardless of the size of long */ + for (i = 0; i < size / sizeof(uint32_t); i++) + lbuf[i] = random(); +} + +static void fill_random_ffr(void *buf, size_t vq) +{ + uint8_t *lbuf = buf; + int bits, i; + + /* + * Only values with a continuous set of 0..n bits set are + * valid for FFR, set all bits then clear a random number of + * high bits. + */ + memset(buf, 0, __SVE_FFR_SIZE(vq)); + + bits = random() % (__SVE_FFR_SIZE(vq) * 8); + for (i = 0; i < bits / 8; i++) + lbuf[i] = 0xff; + if (bits / 8 != __SVE_FFR_SIZE(vq)) + lbuf[i] = (1 << (bits % 8)) - 1; +} + +static void fpsimd_to_sve(__uint128_t *v, char *z, int vl) +{ + int vq = __sve_vq_from_vl(vl); + int i; + __uint128_t *p; + + if (!vl) + return; + + for (i = 0; i < __SVE_NUM_ZREGS; i++) { + p = (__uint128_t *)&z[__SVE_ZREG_OFFSET(vq, i)]; + *p = arm64_cpu_to_le128(v[i]); + } +} + +static void set_initial_values(struct test_config *config) +{ + int vq = __sve_vq_from_vl(vl_in(config)); + int sme_vq = __sve_vq_from_vl(config->sme_vl_in); + + svcr_in = config->svcr_in; + svcr_expected = config->svcr_expected; + svcr_out = 0; + + fill_random(&v_in, sizeof(v_in)); + memcpy(v_expected, v_in, sizeof(v_in)); + memset(v_out, 0, sizeof(v_out)); + + /* Changes will be handled in the test case */ + if (sve_supported() || (config->svcr_in & SVCR_SM)) { + /* The low 128 bits of Z are shared with the V registers */ + fill_random(&z_in, __SVE_ZREGS_SIZE(vq)); + fpsimd_to_sve(v_in, z_in, vl_in(config)); + memcpy(z_expected, z_in, __SVE_ZREGS_SIZE(vq)); + memset(z_out, 0, sizeof(z_out)); + + fill_random(&p_in, __SVE_PREGS_SIZE(vq)); + memcpy(p_expected, p_in, __SVE_PREGS_SIZE(vq)); + memset(p_out, 0, sizeof(p_out)); + + if ((config->svcr_in & SVCR_SM) && !fa64_supported()) + memset(ffr_in, 0, __SVE_PREG_SIZE(vq)); + else + fill_random_ffr(&ffr_in, vq); + memcpy(ffr_expected, ffr_in, __SVE_PREG_SIZE(vq)); + memset(ffr_out, 0, __SVE_PREG_SIZE(vq)); + } + + if (config->svcr_in & SVCR_ZA) + fill_random(za_in, ZA_SIG_REGS_SIZE(sme_vq)); + else + memset(za_in, 0, ZA_SIG_REGS_SIZE(sme_vq)); + if (config->svcr_expected & SVCR_ZA) + memcpy(za_expected, za_in, ZA_SIG_REGS_SIZE(sme_vq)); + else + memset(za_expected, 0, ZA_SIG_REGS_SIZE(sme_vq)); + if (sme_supported()) + memset(za_out, 0, sizeof(za_out)); + + if (sme2_supported()) { + if (config->svcr_in & SVCR_ZA) + fill_random(zt_in, ZT_SIG_REG_BYTES); + else + memset(zt_in, 0, ZT_SIG_REG_BYTES); + if (config->svcr_expected & SVCR_ZA) + memcpy(zt_expected, zt_in, ZT_SIG_REG_BYTES); + else + memset(zt_expected, 0, ZT_SIG_REG_BYTES); + memset(zt_out, 0, sizeof(zt_out)); + } +} + +static bool check_memory_values(struct test_config *config) +{ + bool pass = true; + int vq, sme_vq; + + if (!compare_buffer("saved V", v_out, v_expected, sizeof(v_out))) + pass = false; + + vq = __sve_vq_from_vl(vl_expected(config)); + sme_vq = __sve_vq_from_vl(config->sme_vl_expected); + + if (svcr_out != svcr_expected) { + ksft_print_msg("Mismatch in saved SVCR %lx != %lx\n", + svcr_out, svcr_expected); + pass = false; + } + + if (sve_vl_out != config->sve_vl_expected) { + ksft_print_msg("Mismatch in SVE VL: %ld != %d\n", + sve_vl_out, config->sve_vl_expected); + pass = false; + } + + if (sme_vl_out != config->sme_vl_expected) { + ksft_print_msg("Mismatch in SME VL: %ld != %d\n", + sme_vl_out, config->sme_vl_expected); + pass = false; + } + + if (!compare_buffer("saved Z", z_out, z_expected, + __SVE_ZREGS_SIZE(vq))) + pass = false; + + if (!compare_buffer("saved P", p_out, p_expected, + __SVE_PREGS_SIZE(vq))) + pass = false; + + if (!compare_buffer("saved FFR", ffr_out, ffr_expected, + __SVE_PREG_SIZE(vq))) + pass = false; + + if (!compare_buffer("saved ZA", za_out, za_expected, + ZA_PT_ZA_SIZE(sme_vq))) + pass = false; + + if (!compare_buffer("saved ZT", zt_out, zt_expected, ZT_SIG_REG_BYTES)) + pass = false; + + return pass; +} + +static bool sve_sme_same(struct test_config *config) +{ + if (config->sve_vl_in != config->sve_vl_expected) + return false; + + if (config->sme_vl_in != config->sme_vl_expected) + return false; + + if (config->svcr_in != config->svcr_expected) + return false; + + return true; +} + +static bool sve_write_supported(struct test_config *config) +{ + if (!sve_supported() && !sme_supported()) + return false; + + if ((config->svcr_in & SVCR_ZA) != (config->svcr_expected & SVCR_ZA)) + return false; + + if (config->svcr_expected & SVCR_SM) { + if (config->sve_vl_in != config->sve_vl_expected) { + return false; + } + + /* Changing the SME VL disables ZA */ + if ((config->svcr_expected & SVCR_ZA) && + (config->sme_vl_in != config->sme_vl_expected)) { + return false; + } + } else { + if (config->sme_vl_in != config->sme_vl_expected) { + return false; + } + } + + return true; +} + +static void fpsimd_write_expected(struct test_config *config) +{ + int vl; + + fill_random(&v_expected, sizeof(v_expected)); + + /* The SVE registers are flushed by a FPSIMD write */ + vl = vl_expected(config); + + memset(z_expected, 0, __SVE_ZREGS_SIZE(__sve_vq_from_vl(vl))); + memset(p_expected, 0, __SVE_PREGS_SIZE(__sve_vq_from_vl(vl))); + memset(ffr_expected, 0, __SVE_PREG_SIZE(__sve_vq_from_vl(vl))); + + fpsimd_to_sve(v_expected, z_expected, vl); +} + +static void fpsimd_write(pid_t child, struct test_config *test_config) +{ + struct user_fpsimd_state fpsimd; + struct iovec iov; + int ret; + + memset(&fpsimd, 0, sizeof(fpsimd)); + memcpy(&fpsimd.vregs, v_expected, sizeof(v_expected)); + + iov.iov_base = &fpsimd; + iov.iov_len = sizeof(fpsimd); + ret = ptrace(PTRACE_SETREGSET, child, NT_PRFPREG, &iov); + if (ret == -1) + ksft_print_msg("FPSIMD set failed: (%s) %d\n", + strerror(errno), errno); +} + +static void sve_write_expected(struct test_config *config) +{ + int vl = vl_expected(config); + int sme_vq = __sve_vq_from_vl(config->sme_vl_expected); + + fill_random(z_expected, __SVE_ZREGS_SIZE(__sve_vq_from_vl(vl))); + fill_random(p_expected, __SVE_PREGS_SIZE(__sve_vq_from_vl(vl))); + + if ((svcr_expected & SVCR_SM) && !fa64_supported()) + memset(ffr_expected, 0, __SVE_PREG_SIZE(sme_vq)); + else + fill_random_ffr(ffr_expected, __sve_vq_from_vl(vl)); + + /* Share the low bits of Z with V */ + fill_random(&v_expected, sizeof(v_expected)); + fpsimd_to_sve(v_expected, z_expected, vl); + + if (config->sme_vl_in != config->sme_vl_expected) { + memset(za_expected, 0, ZA_PT_ZA_SIZE(sme_vq)); + memset(zt_expected, 0, sizeof(zt_expected)); + } +} + +static void sve_write(pid_t child, struct test_config *config) +{ + struct user_sve_header *sve; + struct iovec iov; + int ret, vl, vq, regset; + + vl = vl_expected(config); + vq = __sve_vq_from_vl(vl); + + iov.iov_len = SVE_PT_SVE_OFFSET + SVE_PT_SVE_SIZE(vq, SVE_PT_REGS_SVE); + iov.iov_base = malloc(iov.iov_len); + if (!iov.iov_base) { + ksft_print_msg("Failed allocating %lu byte SVE write buffer\n", + iov.iov_len); + return; + } + memset(iov.iov_base, 0, iov.iov_len); + + sve = iov.iov_base; + sve->size = iov.iov_len; + sve->flags = SVE_PT_REGS_SVE; + sve->vl = vl; + + memcpy(iov.iov_base + SVE_PT_SVE_ZREG_OFFSET(vq, 0), + z_expected, SVE_PT_SVE_ZREGS_SIZE(vq)); + memcpy(iov.iov_base + SVE_PT_SVE_PREG_OFFSET(vq, 0), + p_expected, SVE_PT_SVE_PREGS_SIZE(vq)); + memcpy(iov.iov_base + SVE_PT_SVE_FFR_OFFSET(vq), + ffr_expected, SVE_PT_SVE_PREG_SIZE(vq)); + + if (svcr_expected & SVCR_SM) + regset = NT_ARM_SSVE; + else + regset = NT_ARM_SVE; + + ret = ptrace(PTRACE_SETREGSET, child, regset, &iov); + if (ret != 0) + ksft_print_msg("Failed to write SVE: %s (%d)\n", + strerror(errno), errno); + + free(iov.iov_base); +} + +static bool za_write_supported(struct test_config *config) +{ + if (config->svcr_expected & SVCR_SM) { + if (!(config->svcr_in & SVCR_SM)) + return false; + + /* Changing the SME VL exits streaming mode */ + if (config->sme_vl_in != config->sme_vl_expected) { + return false; + } + } + + /* Can't disable SM outside a VL change */ + if ((config->svcr_in & SVCR_SM) && + !(config->svcr_expected & SVCR_SM)) + return false; + + return true; +} + +static void za_write_expected(struct test_config *config) +{ + int sme_vq, sve_vq; + + sme_vq = __sve_vq_from_vl(config->sme_vl_expected); + + if (config->svcr_expected & SVCR_ZA) { + fill_random(za_expected, ZA_PT_ZA_SIZE(sme_vq)); + } else { + memset(za_expected, 0, ZA_PT_ZA_SIZE(sme_vq)); + memset(zt_expected, 0, sizeof(zt_expected)); + } + + /* Changing the SME VL flushes ZT, SVE state and exits SM */ + if (config->sme_vl_in != config->sme_vl_expected) { + svcr_expected &= ~SVCR_SM; + + sve_vq = __sve_vq_from_vl(vl_expected(config)); + memset(z_expected, 0, __SVE_ZREGS_SIZE(sve_vq)); + memset(p_expected, 0, __SVE_PREGS_SIZE(sve_vq)); + memset(ffr_expected, 0, __SVE_PREG_SIZE(sve_vq)); + memset(zt_expected, 0, sizeof(zt_expected)); + + fpsimd_to_sve(v_expected, z_expected, vl_expected(config)); + } +} + +static void za_write(pid_t child, struct test_config *config) +{ + struct user_za_header *za; + struct iovec iov; + int ret, vq; + + vq = __sve_vq_from_vl(config->sme_vl_expected); + + if (config->svcr_expected & SVCR_ZA) + iov.iov_len = ZA_PT_SIZE(vq); + else + iov.iov_len = sizeof(*za); + iov.iov_base = malloc(iov.iov_len); + if (!iov.iov_base) { + ksft_print_msg("Failed allocating %lu byte ZA write buffer\n", + iov.iov_len); + return; + } + memset(iov.iov_base, 0, iov.iov_len); + + za = iov.iov_base; + za->size = iov.iov_len; + za->vl = config->sme_vl_expected; + if (config->svcr_expected & SVCR_ZA) + memcpy(iov.iov_base + ZA_PT_ZA_OFFSET, za_expected, + ZA_PT_ZA_SIZE(vq)); + + ret = ptrace(PTRACE_SETREGSET, child, NT_ARM_ZA, &iov); + if (ret != 0) + ksft_print_msg("Failed to write ZA: %s (%d)\n", + strerror(errno), errno); + + free(iov.iov_base); +} + +static bool zt_write_supported(struct test_config *config) +{ + if (!sme2_supported()) + return false; + if (config->sme_vl_in != config->sme_vl_expected) + return false; + if (!(config->svcr_expected & SVCR_ZA)) + return false; + if ((config->svcr_in & SVCR_SM) != (config->svcr_expected & SVCR_SM)) + return false; + + return true; +} + +static void zt_write_expected(struct test_config *config) +{ + int sme_vq; + + sme_vq = __sve_vq_from_vl(config->sme_vl_expected); + + if (config->svcr_expected & SVCR_ZA) { + fill_random(zt_expected, sizeof(zt_expected)); + } else { + memset(za_expected, 0, ZA_PT_ZA_SIZE(sme_vq)); + memset(zt_expected, 0, sizeof(zt_expected)); + } +} + +static void zt_write(pid_t child, struct test_config *config) +{ + struct iovec iov; + int ret; + + iov.iov_len = ZT_SIG_REG_BYTES; + iov.iov_base = zt_expected; + ret = ptrace(PTRACE_SETREGSET, child, NT_ARM_ZT, &iov); + if (ret != 0) + ksft_print_msg("Failed to write ZT: %s (%d)\n", + strerror(errno), errno); +} + +/* Actually run a test */ +static void run_test(struct test_definition *test, struct test_config *config) +{ + pid_t child; + char name[1024]; + bool pass; + + if (sve_supported() && sme_supported()) + snprintf(name, sizeof(name), "%s, SVE %d->%d, SME %d/%x->%d/%x", + test->name, + config->sve_vl_in, config->sve_vl_expected, + config->sme_vl_in, config->svcr_in, + config->sme_vl_expected, config->svcr_expected); + else if (sve_supported()) + snprintf(name, sizeof(name), "%s, SVE %d->%d", test->name, + config->sve_vl_in, config->sve_vl_expected); + else if (sme_supported()) + snprintf(name, sizeof(name), "%s, SME %d/%x->%d/%x", + test->name, + config->sme_vl_in, config->svcr_in, + config->sme_vl_expected, config->svcr_expected); + else + snprintf(name, sizeof(name), "%s", test->name); + + if (test->supported && !test->supported(config)) { + ksft_test_result_skip("%s\n", name); + return; + } + + set_initial_values(config); + + if (test->set_expected_values) + test->set_expected_values(config); + + child = fork(); + if (child < 0) + ksft_exit_fail_msg("fork() failed: %s (%d)\n", + strerror(errno), errno); + /* run_child() never returns */ + if (child == 0) + run_child(config); + + pass = run_parent(child, test, config); + if (!check_memory_values(config)) + pass = false; + + ksft_test_result(pass, "%s\n", name); +} + +static void run_tests(struct test_definition defs[], int count, + struct test_config *config) +{ + int i; + + for (i = 0; i < count; i++) + run_test(&defs[i], config); +} + +static struct test_definition base_test_defs[] = { + { + .name = "No writes", + .supported = sve_sme_same, + }, + { + .name = "FPSIMD write", + .supported = sve_sme_same, + .set_expected_values = fpsimd_write_expected, + .modify_values = fpsimd_write, + }, +}; + +static struct test_definition sve_test_defs[] = { + { + .name = "SVE write", + .supported = sve_write_supported, + .set_expected_values = sve_write_expected, + .modify_values = sve_write, + }, +}; + +static struct test_definition za_test_defs[] = { + { + .name = "ZA write", + .supported = za_write_supported, + .set_expected_values = za_write_expected, + .modify_values = za_write, + }, +}; + +static struct test_definition zt_test_defs[] = { + { + .name = "ZT write", + .supported = zt_write_supported, + .set_expected_values = zt_write_expected, + .modify_values = zt_write, + }, +}; + +static int sve_vls[MAX_NUM_VLS], sme_vls[MAX_NUM_VLS]; +static int sve_vl_count, sme_vl_count; + +static void probe_vls(const char *name, int vls[], int *vl_count, int set_vl) +{ + unsigned int vq; + int vl; + + *vl_count = 0; + + for (vq = ARCH_VQ_MAX; vq > 0; vq /= 2) { + vl = prctl(set_vl, vq * 16); + if (vl == -1) + ksft_exit_fail_msg("SET_VL failed: %s (%d)\n", + strerror(errno), errno); + + vl &= PR_SVE_VL_LEN_MASK; + + if (*vl_count && (vl == vls[*vl_count - 1])) + break; + + vq = sve_vq_from_vl(vl); + + vls[*vl_count] = vl; + *vl_count += 1; + } + + if (*vl_count > 2) { + /* Just use the minimum and maximum */ + vls[1] = vls[*vl_count - 1]; + ksft_print_msg("%d %s VLs, using %d and %d\n", + *vl_count, name, vls[0], vls[1]); + *vl_count = 2; + } else { + ksft_print_msg("%d %s VLs\n", *vl_count, name); + } +} + +static struct { + int svcr_in, svcr_expected; +} svcr_combinations[] = { + { .svcr_in = 0, .svcr_expected = 0, }, + { .svcr_in = 0, .svcr_expected = SVCR_SM, }, + { .svcr_in = 0, .svcr_expected = SVCR_ZA, }, + /* Can't enable both SM and ZA with a single ptrace write */ + + { .svcr_in = SVCR_SM, .svcr_expected = 0, }, + { .svcr_in = SVCR_SM, .svcr_expected = SVCR_SM, }, + { .svcr_in = SVCR_SM, .svcr_expected = SVCR_ZA, }, + { .svcr_in = SVCR_SM, .svcr_expected = SVCR_SM | SVCR_ZA, }, + + { .svcr_in = SVCR_ZA, .svcr_expected = 0, }, + { .svcr_in = SVCR_ZA, .svcr_expected = SVCR_SM, }, + { .svcr_in = SVCR_ZA, .svcr_expected = SVCR_ZA, }, + { .svcr_in = SVCR_ZA, .svcr_expected = SVCR_SM | SVCR_ZA, }, + + { .svcr_in = SVCR_SM | SVCR_ZA, .svcr_expected = 0, }, + { .svcr_in = SVCR_SM | SVCR_ZA, .svcr_expected = SVCR_SM, }, + { .svcr_in = SVCR_SM | SVCR_ZA, .svcr_expected = SVCR_ZA, }, + { .svcr_in = SVCR_SM | SVCR_ZA, .svcr_expected = SVCR_SM | SVCR_ZA, }, +}; + +static void run_sve_tests(void) +{ + struct test_config test_config; + int i, j; + + if (!sve_supported()) + return; + + test_config.sme_vl_in = sme_vls[0]; + test_config.sme_vl_expected = sme_vls[0]; + test_config.svcr_in = 0; + test_config.svcr_expected = 0; + + for (i = 0; i < sve_vl_count; i++) { + test_config.sve_vl_in = sve_vls[i]; + + for (j = 0; j < sve_vl_count; j++) { + test_config.sve_vl_expected = sve_vls[j]; + + run_tests(base_test_defs, + ARRAY_SIZE(base_test_defs), + &test_config); + if (sve_supported()) + run_tests(sve_test_defs, + ARRAY_SIZE(sve_test_defs), + &test_config); + } + } + +} + +static void run_sme_tests(void) +{ + struct test_config test_config; + int i, j, k; + + if (!sme_supported()) + return; + + test_config.sve_vl_in = sve_vls[0]; + test_config.sve_vl_expected = sve_vls[0]; + + /* + * Every SME VL/SVCR combination + */ + for (i = 0; i < sme_vl_count; i++) { + test_config.sme_vl_in = sme_vls[i]; + + for (j = 0; j < sme_vl_count; j++) { + test_config.sme_vl_expected = sme_vls[j]; + + for (k = 0; k < ARRAY_SIZE(svcr_combinations); k++) { + test_config.svcr_in = svcr_combinations[k].svcr_in; + test_config.svcr_expected = svcr_combinations[k].svcr_expected; + + run_tests(base_test_defs, + ARRAY_SIZE(base_test_defs), + &test_config); + run_tests(sve_test_defs, + ARRAY_SIZE(sve_test_defs), + &test_config); + run_tests(za_test_defs, + ARRAY_SIZE(za_test_defs), + &test_config); + + if (sme2_supported()) + run_tests(zt_test_defs, + ARRAY_SIZE(zt_test_defs), + &test_config); + } + } + } +} + +int main(void) +{ + struct test_config test_config; + struct sigaction sa; + int tests, ret, tmp; + + srandom(getpid()); + + ksft_print_header(); + + if (sve_supported()) { + probe_vls("SVE", sve_vls, &sve_vl_count, PR_SVE_SET_VL); + + tests = ARRAY_SIZE(base_test_defs) + + ARRAY_SIZE(sve_test_defs); + tests *= sve_vl_count * sve_vl_count; + } else { + /* Only run the FPSIMD tests */ + sve_vl_count = 1; + tests = ARRAY_SIZE(base_test_defs); + } + + if (sme_supported()) { + probe_vls("SME", sme_vls, &sme_vl_count, PR_SME_SET_VL); + + tmp = ARRAY_SIZE(base_test_defs) + ARRAY_SIZE(sve_test_defs) + + ARRAY_SIZE(za_test_defs); + + if (sme2_supported()) + tmp += ARRAY_SIZE(zt_test_defs); + + tmp *= sme_vl_count * sme_vl_count; + tmp *= ARRAY_SIZE(svcr_combinations); + tests += tmp; + } else { + sme_vl_count = 1; + } + + if (sme2_supported()) + ksft_print_msg("SME2 supported\n"); + + if (fa64_supported()) + ksft_print_msg("FA64 supported\n"); + + ksft_set_plan(tests); + + /* Get signal handers ready before we start any children */ + memset(&sa, 0, sizeof(sa)); + sa.sa_sigaction = handle_alarm; + sa.sa_flags = SA_RESTART | SA_SIGINFO; + sigemptyset(&sa.sa_mask); + ret = sigaction(SIGALRM, &sa, NULL); + if (ret < 0) + ksft_print_msg("Failed to install SIGALRM handler: %s (%d)\n", + strerror(errno), errno); + + /* + * Run the test set if there is no SVE or SME, with those we + * have to pick a VL for each run. + */ + if (!sve_supported()) { + test_config.sve_vl_in = 0; + test_config.sve_vl_expected = 0; + test_config.sme_vl_in = 0; + test_config.sme_vl_expected = 0; + test_config.svcr_in = 0; + test_config.svcr_expected = 0; + + run_tests(base_test_defs, ARRAY_SIZE(base_test_defs), + &test_config); + } + + run_sve_tests(); + run_sme_tests(); + + ksft_finished(); +} diff --git a/tools/testing/selftests/arm64/fp/fp-ptrace.h b/tools/testing/selftests/arm64/fp/fp-ptrace.h new file mode 100644 index 000000000000..db4f2c4d750c --- /dev/null +++ b/tools/testing/selftests/arm64/fp/fp-ptrace.h @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (C) 2021-3 ARM Limited. + +#ifndef FP_PTRACE_H +#define FP_PTRACE_H + +#define SVCR_SM_SHIFT 0 +#define SVCR_ZA_SHIFT 1 + +#define SVCR_SM (1 << SVCR_SM_SHIFT) +#define SVCR_ZA (1 << SVCR_ZA_SHIFT) + +#endif From 3f9b9df8d2809bf7d49ed46df460b4c57ffa9a11 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Tue, 12 Nov 2024 13:08:14 +0000 Subject: [PATCH 71/85] kselftets/arm64: Use flag bits for features in fp-ptrace assembler code commit c0350076c13eac4f1d7f7ab6acd43bb252baef7a upstream. The assembler portions of fp-ptrace are passed feature flags by the C code indicating which architectural features are supported. Currently these use an entire register for each flag which is wasteful and gets cumbersome as new flags are added. Switch to using flag bits in a single register to make things easier to maintain. No functional change. Signed-off-by: Mark Brown Link: https://lore.kernel.org/r/20241112-arm64-fp-ptrace-fpmr-v2-1-250b57c61254@kernel.org Signed-off-by: Catalin Marinas Signed-off-by: Bin Huang --- .../selftests/arm64/fp/fp-ptrace-asm.S | 32 +++++++++++-------- tools/testing/selftests/arm64/fp/fp-ptrace.c | 17 +++++++--- tools/testing/selftests/arm64/fp/fp-ptrace.h | 10 ++++++ 3 files changed, 41 insertions(+), 18 deletions(-) diff --git a/tools/testing/selftests/arm64/fp/fp-ptrace-asm.S b/tools/testing/selftests/arm64/fp/fp-ptrace-asm.S index 7ad59d92d02b..5e7e9c878f2c 100644 --- a/tools/testing/selftests/arm64/fp/fp-ptrace-asm.S +++ b/tools/testing/selftests/arm64/fp/fp-ptrace-asm.S @@ -15,10 +15,7 @@ // Load and save register values with pauses for ptrace // -// x0 - SVE in use -// x1 - SME in use -// x2 - SME2 in use -// x3 - FA64 supported +// x0 - HAVE_ flags indicating which features are in use .globl load_and_save load_and_save: @@ -44,7 +41,7 @@ load_and_save: ldp q30, q31, [x7, #16 * 30] // SME? - cbz x1, check_sve_in + tbz x0, #HAVE_SME_SHIFT, check_sve_in adrp x7, svcr_in ldr x7, [x7, :lo12:svcr_in] @@ -64,7 +61,7 @@ load_and_save: bne 1b // ZT? - cbz x2, check_sm_in + tbz x0, #HAVE_SME2_SHIFT, check_sm_in adrp x6, zt_in add x6, x6, :lo12:zt_in _ldr_zt 6 @@ -72,12 +69,16 @@ load_and_save: // In streaming mode? check_sm_in: tbz x7, #SVCR_SM_SHIFT, check_sve_in - mov x4, x3 // Load FFR if we have FA64 + + // Load FFR if we have FA64 + mov x4, #0 + tbz x0, #HAVE_FA64_SHIFT, load_sve + mov x4, #1 b load_sve // SVE? check_sve_in: - cbz x0, wait_for_writes + tbz x0, #HAVE_SVE_SHIFT, wait_for_writes mov x4, #1 load_sve: @@ -165,8 +166,7 @@ wait_for_writes: stp q28, q29, [x7, #16 * 28] stp q30, q31, [x7, #16 * 30] - // SME? - cbz x1, check_sve_out + tbz x0, #HAVE_SME_SHIFT, check_sve_out rdsvl 11, 1 adrp x6, sme_vl_out @@ -187,7 +187,7 @@ wait_for_writes: bne 1b // ZT? - cbz x2, check_sm_out + tbz x0, #HAVE_SME2_SHIFT, check_sm_out adrp x6, zt_out add x6, x6, :lo12:zt_out _str_zt 6 @@ -195,12 +195,16 @@ wait_for_writes: // In streaming mode? check_sm_out: tbz x7, #SVCR_SM_SHIFT, check_sve_out - mov x4, x3 // FFR? + + // Do we have FA64 and FFR? + mov x4, #0 + tbz x0, #HAVE_FA64_SHIFT, read_sve + mov x4, #1 b read_sve // SVE? check_sve_out: - cbz x0, wait_for_reads + tbz x0, #HAVE_SVE_SHIFT, wait_for_reads mov x4, #1 rdvl x7, #1 @@ -271,7 +275,7 @@ wait_for_reads: brk #0 // Ensure we don't leave ourselves in streaming mode - cbz x1, out + tbz x0, #HAVE_SME_SHIFT, out msr S3_3_C4_C2_2, xzr out: diff --git a/tools/testing/selftests/arm64/fp/fp-ptrace.c b/tools/testing/selftests/arm64/fp/fp-ptrace.c index c7ceafe5f471..d96af27487fa 100644 --- a/tools/testing/selftests/arm64/fp/fp-ptrace.c +++ b/tools/testing/selftests/arm64/fp/fp-ptrace.c @@ -82,7 +82,7 @@ uint64_t sve_vl_out; uint64_t sme_vl_out; uint64_t svcr_in, svcr_expected, svcr_out; -void load_and_save(int sve, int sme, int sme2, int fa64); +void load_and_save(int flags); static bool got_alarm; @@ -198,7 +198,7 @@ static int vl_expected(struct test_config *config) static void run_child(struct test_config *config) { - int ret; + int ret, flags; /* Let the parent attach to us */ ret = ptrace(PTRACE_TRACEME, 0, 0, 0); @@ -224,8 +224,17 @@ static void run_child(struct test_config *config) } /* Load values and wait for the parent */ - load_and_save(sve_supported(), sme_supported(), - sme2_supported(), fa64_supported()); + flags = 0; + if (sve_supported()) + flags |= HAVE_SVE; + if (sme_supported()) + flags |= HAVE_SME; + if (sme2_supported()) + flags |= HAVE_SME2; + if (fa64_supported()) + flags |= HAVE_FA64; + + load_and_save(flags); exit(0); } diff --git a/tools/testing/selftests/arm64/fp/fp-ptrace.h b/tools/testing/selftests/arm64/fp/fp-ptrace.h index db4f2c4d750c..36ca627e1980 100644 --- a/tools/testing/selftests/arm64/fp/fp-ptrace.h +++ b/tools/testing/selftests/arm64/fp/fp-ptrace.h @@ -10,4 +10,14 @@ #define SVCR_SM (1 << SVCR_SM_SHIFT) #define SVCR_ZA (1 << SVCR_ZA_SHIFT) +#define HAVE_SVE_SHIFT 0 +#define HAVE_SME_SHIFT 1 +#define HAVE_SME2_SHIFT 2 +#define HAVE_FA64_SHIFT 3 + +#define HAVE_SVE (1 << HAVE_SVE_SHIFT) +#define HAVE_SME (1 << HAVE_SME_SHIFT) +#define HAVE_SME2 (1 << HAVE_SME2_SHIFT) +#define HAVE_FA64 (1 << HAVE_FA64_SHIFT) + #endif From 4c3c5a63d437fc300814143f99b79ed3c1369ca4 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Tue, 12 Nov 2024 13:08:15 +0000 Subject: [PATCH 72/85] kselftest/arm64: Expand the set of ZA writes fp-ptrace does commit 7e9c5b00009a625cc304c865192978c01c0cc077 upstream. Currently our test for implementable ZA writes is written in a bit of a convoluted fashion which excludes all changes where we clear SVCR.SM even though we can actually support that since changing the vector length resets SVCR. Make the logic more direct, enabling us to actually run these cases. Signed-off-by: Mark Brown Link: https://lore.kernel.org/r/20241112-arm64-fp-ptrace-fpmr-v2-2-250b57c61254@kernel.org Signed-off-by: Catalin Marinas Signed-off-by: Bin Huang --- tools/testing/selftests/arm64/fp/fp-ptrace.c | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/tools/testing/selftests/arm64/fp/fp-ptrace.c b/tools/testing/selftests/arm64/fp/fp-ptrace.c index d96af27487fa..56cf6e02c535 100644 --- a/tools/testing/selftests/arm64/fp/fp-ptrace.c +++ b/tools/testing/selftests/arm64/fp/fp-ptrace.c @@ -1078,21 +1078,19 @@ static void sve_write(pid_t child, struct test_config *config) static bool za_write_supported(struct test_config *config) { - if (config->svcr_expected & SVCR_SM) { - if (!(config->svcr_in & SVCR_SM)) + if (config->sme_vl_in != config->sme_vl_expected) { + /* Changing the SME VL exits streaming mode. */ + if (config->svcr_expected & SVCR_SM) { return false; - - /* Changing the SME VL exits streaming mode */ - if (config->sme_vl_in != config->sme_vl_expected) { + } + } else { + /* Otherwise we can't change streaming mode */ + if ((config->svcr_in & SVCR_SM) != + (config->svcr_expected & SVCR_SM)) { return false; } } - /* Can't disable SM outside a VL change */ - if ((config->svcr_in & SVCR_SM) && - !(config->svcr_expected & SVCR_SM)) - return false; - return true; } From ad742485a3ccbf1e20a3e777821ba88deaf7b1e6 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Tue, 12 Nov 2024 13:08:16 +0000 Subject: [PATCH 73/85] kselftest/arm64: Add FPMR coverage to fp-ptrace commit 7dbd26d0b22d69d36ab3e76ee7f152482a19cbed upstream. Add coverage for FPMR to fp-ptrace. FPMR can be available independently of SVE and SME, if SME is supported then FPMR is cleared by entering and exiting streaming mode. As with other registers we generate random values to load into the register, we restrict these to bitfields which are always defined. We also leave bitfields where the valid values are affected by the set of supported FP8 formats zero to reduce complexity, it is unlikely that specific bitfields will be affected by ptrace issues. Signed-off-by: Mark Brown Link: https://lore.kernel.org/r/20241112-arm64-fp-ptrace-fpmr-v2-3-250b57c61254@kernel.org [catalin.marinas@arm.com: use REG_FPMR instead of FPMR] Signed-off-by: Catalin Marinas Signed-off-by: Bin Huang --- .../selftests/arm64/fp/fp-ptrace-asm.S | 23 +++- tools/testing/selftests/arm64/fp/fp-ptrace.c | 126 ++++++++++++++++++ tools/testing/selftests/arm64/fp/fp-ptrace.h | 2 + tools/testing/selftests/arm64/fp/sme-inst.h | 2 + 4 files changed, 146 insertions(+), 7 deletions(-) diff --git a/tools/testing/selftests/arm64/fp/fp-ptrace-asm.S b/tools/testing/selftests/arm64/fp/fp-ptrace-asm.S index 5e7e9c878f2c..82c3ab70e1cf 100644 --- a/tools/testing/selftests/arm64/fp/fp-ptrace-asm.S +++ b/tools/testing/selftests/arm64/fp/fp-ptrace-asm.S @@ -71,14 +71,12 @@ check_sm_in: tbz x7, #SVCR_SM_SHIFT, check_sve_in // Load FFR if we have FA64 - mov x4, #0 - tbz x0, #HAVE_FA64_SHIFT, load_sve - mov x4, #1 + ubfx x4, x0, #HAVE_FA64_SHIFT, #1 b load_sve // SVE? check_sve_in: - tbz x0, #HAVE_SVE_SHIFT, wait_for_writes + tbz x0, #HAVE_SVE_SHIFT, check_fpmr_in mov x4, #1 load_sve: @@ -143,6 +141,13 @@ load_sve: ldr p14, [x7, #14, MUL VL] ldr p15, [x7, #15, MUL VL] + // This has to come after we set PSTATE.SM +check_fpmr_in: + tbz x0, #HAVE_FPMR_SHIFT, wait_for_writes + adrp x7, fpmr_in + ldr x7, [x7, :lo12:fpmr_in] + msr REG_FPMR, x7 + wait_for_writes: // Wait for the parent brk #0 @@ -166,6 +171,12 @@ wait_for_writes: stp q28, q29, [x7, #16 * 28] stp q30, q31, [x7, #16 * 30] + tbz x0, #HAVE_FPMR_SHIFT, check_sme_out + mrs x7, REG_FPMR + adrp x6, fpmr_out + str x7, [x6, :lo12:fpmr_out] + +check_sme_out: tbz x0, #HAVE_SME_SHIFT, check_sve_out rdsvl 11, 1 @@ -197,9 +208,7 @@ check_sm_out: tbz x7, #SVCR_SM_SHIFT, check_sve_out // Do we have FA64 and FFR? - mov x4, #0 - tbz x0, #HAVE_FA64_SHIFT, read_sve - mov x4, #1 + ubfx x4, x0, #HAVE_FA64_SHIFT, #1 b read_sve // SVE? diff --git a/tools/testing/selftests/arm64/fp/fp-ptrace.c b/tools/testing/selftests/arm64/fp/fp-ptrace.c index 56cf6e02c535..4930e03a7b99 100644 --- a/tools/testing/selftests/arm64/fp/fp-ptrace.c +++ b/tools/testing/selftests/arm64/fp/fp-ptrace.c @@ -31,6 +31,14 @@ #include "fp-ptrace.h" +#include + +#define FPMR_LSCALE2_MASK GENMASK(37, 32) +#define FPMR_NSCALE_MASK GENMASK(31, 24) +#define FPMR_LSCALE_MASK GENMASK(22, 16) +#define FPMR_OSC_MASK GENMASK(15, 15) +#define FPMR_OSM_MASK GENMASK(14, 14) + /* and don't like each other, so: */ #ifndef NT_ARM_SVE #define NT_ARM_SVE 0x405 @@ -48,11 +56,22 @@ #define NT_ARM_ZT 0x40d #endif +#ifndef NT_ARM_FPMR +#define NT_ARM_FPMR 0x40e +#endif + #define ARCH_VQ_MAX 256 /* VL 128..2048 in powers of 2 */ #define MAX_NUM_VLS 5 +/* + * FPMR bits we can set without doing feature checks to see if values + * are valid. + */ +#define FPMR_SAFE_BITS (FPMR_LSCALE2_MASK | FPMR_NSCALE_MASK | \ + FPMR_LSCALE_MASK | FPMR_OSC_MASK | FPMR_OSM_MASK) + #define NUM_FPR 32 __uint128_t v_in[NUM_FPR]; __uint128_t v_expected[NUM_FPR]; @@ -78,6 +97,8 @@ char zt_in[ZT_SIG_REG_BYTES]; char zt_expected[ZT_SIG_REG_BYTES]; char zt_out[ZT_SIG_REG_BYTES]; +uint64_t fpmr_in, fpmr_expected, fpmr_out; + uint64_t sve_vl_out; uint64_t sme_vl_out; uint64_t svcr_in, svcr_expected, svcr_out; @@ -128,6 +149,11 @@ static bool fa64_supported(void) return getauxval(AT_HWCAP2) & HWCAP2_SME_FA64; } +static bool fpmr_supported(void) +{ + return getauxval(AT_HWCAP2) & HWCAP2_FPMR; +} + static bool compare_buffer(const char *name, void *out, void *expected, size_t size) { @@ -233,6 +259,8 @@ static void run_child(struct test_config *config) flags |= HAVE_SME2; if (fa64_supported()) flags |= HAVE_FA64; + if (fpmr_supported()) + flags |= HAVE_FPMR; load_and_save(flags); @@ -321,6 +349,14 @@ static void read_child_regs(pid_t child) iov_child.iov_len = sizeof(zt_out); read_one_child_regs(child, "ZT", &iov_parent, &iov_child); } + + if (fpmr_supported()) { + iov_parent.iov_base = &fpmr_out; + iov_parent.iov_len = sizeof(fpmr_out); + iov_child.iov_base = &fpmr_out; + iov_child.iov_len = sizeof(fpmr_out); + read_one_child_regs(child, "FPMR", &iov_parent, &iov_child); + } } static bool continue_breakpoint(pid_t child, @@ -595,6 +631,26 @@ static bool check_ptrace_values_zt(pid_t child, struct test_config *config) return compare_buffer("initial ZT", buf, zt_in, ZT_SIG_REG_BYTES); } +static bool check_ptrace_values_fpmr(pid_t child, struct test_config *config) +{ + uint64_t val; + struct iovec iov; + int ret; + + if (!fpmr_supported()) + return true; + + iov.iov_base = &val; + iov.iov_len = sizeof(val); + ret = ptrace(PTRACE_GETREGSET, child, NT_ARM_FPMR, &iov); + if (ret != 0) { + ksft_print_msg("Failed to read initial FPMR: %s (%d)\n", + strerror(errno), errno); + return false; + } + + return compare_buffer("initial FPMR", &val, &fpmr_in, sizeof(val)); +} static bool check_ptrace_values(pid_t child, struct test_config *config) { @@ -629,6 +685,9 @@ static bool check_ptrace_values(pid_t child, struct test_config *config) if (!check_ptrace_values_zt(child, config)) pass = false; + if (!check_ptrace_values_fpmr(child, config)) + pass = false; + return pass; } @@ -832,11 +891,18 @@ static void set_initial_values(struct test_config *config) { int vq = __sve_vq_from_vl(vl_in(config)); int sme_vq = __sve_vq_from_vl(config->sme_vl_in); + bool sm_change; svcr_in = config->svcr_in; svcr_expected = config->svcr_expected; svcr_out = 0; + if (sme_supported() && + (svcr_in & SVCR_SM) != (svcr_expected & SVCR_SM)) + sm_change = true; + else + sm_change = false; + fill_random(&v_in, sizeof(v_in)); memcpy(v_expected, v_in, sizeof(v_in)); memset(v_out, 0, sizeof(v_out)); @@ -883,6 +949,21 @@ static void set_initial_values(struct test_config *config) memset(zt_expected, 0, ZT_SIG_REG_BYTES); memset(zt_out, 0, sizeof(zt_out)); } + + if (fpmr_supported()) { + fill_random(&fpmr_in, sizeof(fpmr_in)); + fpmr_in &= FPMR_SAFE_BITS; + + /* Entering or exiting streaming mode clears FPMR */ + if (sm_change) + fpmr_expected = 0; + else + fpmr_expected = fpmr_in; + } else { + fpmr_in = 0; + fpmr_expected = 0; + fpmr_out = 0; + } } static bool check_memory_values(struct test_config *config) @@ -933,6 +1014,12 @@ static bool check_memory_values(struct test_config *config) if (!compare_buffer("saved ZT", zt_out, zt_expected, ZT_SIG_REG_BYTES)) pass = false; + if (fpmr_out != fpmr_expected) { + ksft_print_msg("Mismatch in saved FPMR: %lx != %lx\n", + fpmr_out, fpmr_expected); + pass = false; + } + return pass; } @@ -1010,6 +1097,36 @@ static void fpsimd_write(pid_t child, struct test_config *test_config) strerror(errno), errno); } +static bool fpmr_write_supported(struct test_config *config) +{ + if (!fpmr_supported()) + return false; + + if (!sve_sme_same(config)) + return false; + + return true; +} + +static void fpmr_write_expected(struct test_config *config) +{ + fill_random(&fpmr_expected, sizeof(fpmr_expected)); + fpmr_expected &= FPMR_SAFE_BITS; +} + +static void fpmr_write(pid_t child, struct test_config *config) +{ + struct iovec iov; + int ret; + + iov.iov_len = sizeof(fpmr_expected); + iov.iov_base = &fpmr_expected; + ret = ptrace(PTRACE_SETREGSET, child, NT_ARM_FPMR, &iov); + if (ret != 0) + ksft_print_msg("Failed to write FPMR: %s (%d)\n", + strerror(errno), errno); +} + static void sve_write_expected(struct test_config *config) { int vl = vl_expected(config); @@ -1266,6 +1383,12 @@ static struct test_definition base_test_defs[] = { .set_expected_values = fpsimd_write_expected, .modify_values = fpsimd_write, }, + { + .name = "FPMR write", + .supported = fpmr_write_supported, + .set_expected_values = fpmr_write_expected, + .modify_values = fpmr_write, + }, }; static struct test_definition sve_test_defs[] = { @@ -1475,6 +1598,9 @@ int main(void) if (fa64_supported()) ksft_print_msg("FA64 supported\n"); + if (fpmr_supported()) + ksft_print_msg("FPMR supported\n"); + ksft_set_plan(tests); /* Get signal handers ready before we start any children */ diff --git a/tools/testing/selftests/arm64/fp/fp-ptrace.h b/tools/testing/selftests/arm64/fp/fp-ptrace.h index 36ca627e1980..c06919aaf1f7 100644 --- a/tools/testing/selftests/arm64/fp/fp-ptrace.h +++ b/tools/testing/selftests/arm64/fp/fp-ptrace.h @@ -14,10 +14,12 @@ #define HAVE_SME_SHIFT 1 #define HAVE_SME2_SHIFT 2 #define HAVE_FA64_SHIFT 3 +#define HAVE_FPMR_SHIFT 4 #define HAVE_SVE (1 << HAVE_SVE_SHIFT) #define HAVE_SME (1 << HAVE_SME_SHIFT) #define HAVE_SME2 (1 << HAVE_SME2_SHIFT) #define HAVE_FA64 (1 << HAVE_FA64_SHIFT) +#define HAVE_FPMR (1 << HAVE_FPMR_SHIFT) #endif diff --git a/tools/testing/selftests/arm64/fp/sme-inst.h b/tools/testing/selftests/arm64/fp/sme-inst.h index 9292bba5400b..85b9184e0835 100644 --- a/tools/testing/selftests/arm64/fp/sme-inst.h +++ b/tools/testing/selftests/arm64/fp/sme-inst.h @@ -5,6 +5,8 @@ #ifndef SME_INST_H #define SME_INST_H +#define REG_FPMR S3_3_C4_C4_2 + /* * RDSVL X\nx, #\imm */ From 5b77244b068668cec22b59b2d2aded8a2fb1527c Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Thu, 8 May 2025 14:26:41 +0100 Subject: [PATCH 74/85] kselftest/arm64: fp-ptrace: Fix expected FPMR value when PSTATE.SM is changed commit 78b23877dbba7dbcda2b89383d17bed82ce8f663 upstream. The fp-ptrace test suite expects that FPMR is set to zero when PSTATE.SM is changed via ptrace, but ptrace has never altered FPMR in this way, and the test logic erroneously relies upon (and has concealed) a bug where task_fpsimd_load() would unexpectedly and non-deterministically clobber FPMR. Using ptrace, FPMR can only be altered by writing to the NT_ARM_FPMR regset. The value of PSTATE.SM can be altered by writing to the NT_ARM_SVE or NT_ARM_SSVE regsets, and/or by changing the SME vector length (when writing to the NT_ARM_SVE, NT_ARM_SSVE, or NT_ARM_ZA regsets), but none of these writes will change the value of FPMR. The task_fpsimd_load() bug was introduced with the initial FPMR support in commit: 203f2b95a882 ("arm64/fpsimd: Support FEAT_FPMR") The incorrect FPMR test code was introduced in commit: 7dbd26d0b22d ("kselftest/arm64: Add FPMR coverage to fp-ptrace") Subsequently, the task_fpsimd_load() bug was fixed in commit: e5fa85fce08b ("arm64/fpsimd: Don't corrupt FPMR when streaming mode changes") ... whereupon the fp-ptrace FPMR tests started failing reliably, e.g. | # # Mismatch in saved FPMR: 915058000 != 0 | # not ok 25 SVE write, SVE 64->64, SME 64/0->64/1 Fix this by changing the test to expect that FPMR is *NOT* changed when PSTATE.SM is changed via ptrace, matching the extant behaviour. I've chosen to update the test code rather than modifying ptrace to zero FPMR when PSTATE.SM changes. Not zeroing FPMR is simpler overall, and allows the NT_ARM_FPMR regset to be handled independently from other regsets, leaving less scope for error. Fixes: 7dbd26d0b22d ("kselftest/arm64: Add FPMR coverage to fp-ptrace") Signed-off-by: Mark Rutland Cc: Catalin Marinas Cc: David Spickett Cc: Luis Machado Cc: Marc Zyngier Cc: Mark Brown Cc: Will Deacon Link: https://lore.kernel.org/r/20250508132644.1395904-22-mark.rutland@arm.com Signed-off-by: Will Deacon Signed-off-by: Bin Huang --- tools/testing/selftests/arm64/fp/fp-ptrace.c | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/tools/testing/selftests/arm64/fp/fp-ptrace.c b/tools/testing/selftests/arm64/fp/fp-ptrace.c index 4930e03a7b99..762048eb354f 100644 --- a/tools/testing/selftests/arm64/fp/fp-ptrace.c +++ b/tools/testing/selftests/arm64/fp/fp-ptrace.c @@ -891,18 +891,11 @@ static void set_initial_values(struct test_config *config) { int vq = __sve_vq_from_vl(vl_in(config)); int sme_vq = __sve_vq_from_vl(config->sme_vl_in); - bool sm_change; svcr_in = config->svcr_in; svcr_expected = config->svcr_expected; svcr_out = 0; - if (sme_supported() && - (svcr_in & SVCR_SM) != (svcr_expected & SVCR_SM)) - sm_change = true; - else - sm_change = false; - fill_random(&v_in, sizeof(v_in)); memcpy(v_expected, v_in, sizeof(v_in)); memset(v_out, 0, sizeof(v_out)); @@ -953,12 +946,7 @@ static void set_initial_values(struct test_config *config) if (fpmr_supported()) { fill_random(&fpmr_in, sizeof(fpmr_in)); fpmr_in &= FPMR_SAFE_BITS; - - /* Entering or exiting streaming mode clears FPMR */ - if (sm_change) - fpmr_expected = 0; - else - fpmr_expected = fpmr_in; + fpmr_expected = fpmr_in; } else { fpmr_in = 0; fpmr_expected = 0; From 759398362b2dda798090fb62126de31747ae2dbd Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Thu, 8 May 2025 14:26:43 +0100 Subject: [PATCH 75/85] kselftest/arm64: fp-ptrace: Adjust to new VL change behaviour commit 031a2acaa1cdab3c89dfb810a8cb75ddfdf5a65c upstream. In order to fix an ABI problem, we recently changed the way that changing the SVE/SME vector length affects PSTATE.SM. Historically, changing the SME vector length would clear PSTATE.SM. Now, changing the SME vector length preserves PSTATE.SM. Update the fp-ptrace test for the new behaviour. Signed-off-by: Mark Rutland Cc: Catalin Marinas Cc: David Spickett Cc: Luis Machado Cc: Marc Zyngier Cc: Mark Brown Cc: Will Deacon Link: https://lore.kernel.org/r/20250508132644.1395904-24-mark.rutland@arm.com Signed-off-by: Will Deacon Signed-off-by: Bin Huang --- tools/testing/selftests/arm64/fp/fp-ptrace.c | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/tools/testing/selftests/arm64/fp/fp-ptrace.c b/tools/testing/selftests/arm64/fp/fp-ptrace.c index 762048eb354f..c2882a5a5cc0 100644 --- a/tools/testing/selftests/arm64/fp/fp-ptrace.c +++ b/tools/testing/selftests/arm64/fp/fp-ptrace.c @@ -1183,18 +1183,8 @@ static void sve_write(pid_t child, struct test_config *config) static bool za_write_supported(struct test_config *config) { - if (config->sme_vl_in != config->sme_vl_expected) { - /* Changing the SME VL exits streaming mode. */ - if (config->svcr_expected & SVCR_SM) { - return false; - } - } else { - /* Otherwise we can't change streaming mode */ - if ((config->svcr_in & SVCR_SM) != - (config->svcr_expected & SVCR_SM)) { - return false; - } - } + if ((config->svcr_in & SVCR_SM) != (config->svcr_expected & SVCR_SM)) + return false; return true; } @@ -1212,10 +1202,8 @@ static void za_write_expected(struct test_config *config) memset(zt_expected, 0, sizeof(zt_expected)); } - /* Changing the SME VL flushes ZT, SVE state and exits SM */ + /* Changing the SME VL flushes ZT, SVE state */ if (config->sme_vl_in != config->sme_vl_expected) { - svcr_expected &= ~SVCR_SM; - sve_vq = __sve_vq_from_vl(vl_expected(config)); memset(z_expected, 0, __SVE_ZREGS_SIZE(sve_vq)); memset(p_expected, 0, __SVE_PREGS_SIZE(sve_vq)); From a9e05e5f23cd5e98137c8db1c2face3d8fbca1e7 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Thu, 8 May 2025 14:26:44 +0100 Subject: [PATCH 76/85] kselftest/arm64: fp-ptrace: Adjust to new inactive mode behaviour commit 864f3ddcd7153fdb3f2d3822e563985135d8eafa upstream. In order to fix an ABI problem, we recently changed the way that reads of the NT_ARM_SVE and NT_ARM_SSVE regsets behave when their corresponding vector state is inactive. Update the fp-ptrace test for the new behaviour. Signed-off-by: Mark Rutland Cc: Catalin Marinas Cc: David Spickett Cc: Luis Machado Cc: Marc Zyngier Cc: Mark Brown Cc: Will Deacon Link: https://lore.kernel.org/r/20250508132644.1395904-25-mark.rutland@arm.com Signed-off-by: Will Deacon Signed-off-by: Bin Huang --- tools/testing/selftests/arm64/fp/fp-ptrace.c | 30 ++++++++++++++------ 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/tools/testing/selftests/arm64/fp/fp-ptrace.c b/tools/testing/selftests/arm64/fp/fp-ptrace.c index c2882a5a5cc0..191c47ca0ed8 100644 --- a/tools/testing/selftests/arm64/fp/fp-ptrace.c +++ b/tools/testing/selftests/arm64/fp/fp-ptrace.c @@ -439,10 +439,17 @@ static bool check_ptrace_values_sve(pid_t child, struct test_config *config) pass = false; } - if (sve->size != SVE_PT_SIZE(vq, sve->flags)) { - ksft_print_msg("Mismatch in SVE header size: %d != %lu\n", - sve->size, SVE_PT_SIZE(vq, sve->flags)); - pass = false; + if (svcr_in & SVCR_SM) { + if (sve->size != sizeof(sve)) { + ksft_print_msg("NT_ARM_SVE reports data with PSTATE.SM\n"); + pass = false; + } + } else { + if (sve->size != SVE_PT_SIZE(vq, sve->flags)) { + ksft_print_msg("Mismatch in SVE header size: %d != %lu\n", + sve->size, SVE_PT_SIZE(vq, sve->flags)); + pass = false; + } } /* The registers might be in completely different formats! */ @@ -515,10 +522,17 @@ static bool check_ptrace_values_ssve(pid_t child, struct test_config *config) pass = false; } - if (sve->size != SVE_PT_SIZE(vq, sve->flags)) { - ksft_print_msg("Mismatch in SSVE header size: %d != %lu\n", - sve->size, SVE_PT_SIZE(vq, sve->flags)); - pass = false; + if (!(svcr_in & SVCR_SM)) { + if (sve->size != sizeof(sve)) { + ksft_print_msg("NT_ARM_SSVE reports data without PSTATE.SM\n"); + pass = false; + } + } else { + if (sve->size != SVE_PT_SIZE(vq, sve->flags)) { + ksft_print_msg("Mismatch in SSVE header size: %d != %lu\n", + sve->size, SVE_PT_SIZE(vq, sve->flags)); + pass = false; + } } /* The registers might be in completely different formats! */ From 388921748765285b57364634bd0620aeaba90882 Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Tue, 5 May 2026 09:02:13 -0700 Subject: [PATCH 77/85] arm64/fpsimd: ptrace: zero target's fpsimd_state, not the tracer's commit 5cbb61bf4168859d97c068d88d364f4f1f440325 upstream. sve_set_common() is the backend for PTRACE_SETREGSET(NT_ARM_SVE) and PTRACE_SETREGSET(NT_ARM_SSVE). Every write in the function operates on the tracee (target) - except a single memset that uses current instead, zeroing the tracer's saved V0-V31 / FPSR / FPCR shadow on every ptrace SETREGSET call. The memset is meant to give the tracee a defined zero register image before the user-supplied payload is copied in (for partial writes, header-only writes, and FPSIMD<->SVE format switches). Aiming it at current both denies the tracee that clean slate and silently corrupts the tracer. The corruption of the tracer's saved FPSIMD state is not always observable. Where the tracer's state is live on a CPU, this may be reused without loading the corrupted state from memory, and will eventually be written back over the corrupted state. Where the tracer's state is saved in SVE_PT_REGS_SVE format, only the FPSR and FPCR are clobbered, and the effective copy of the vectors is in the task's sve_state. Reproducible on an arm64 kernel with SVE: a single-threaded tracer that loads a known pattern into V0-V31, issues PTRACE_SETREGSET(NT_ARM_SVE) on a child, and reads V0-V31 back observes them all zeroed within tens of thousands of iterations when a sibling thread keeps stealing the FPSIMD CPU binding. Fixes: 316283f276eb ("arm64/fpsimd: ptrace: Consistently handle partial writes to NT_ARM_(S)SVE") Cc: Signed-off-by: Breno Leitao Acked-by: Mark Rutland Signed-off-by: Catalin Marinas Signed-off-by: Bin Huang --- arch/arm64/kernel/ptrace.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c index c510cacc34b8..fbaae3d98935 100644 --- a/arch/arm64/kernel/ptrace.c +++ b/arch/arm64/kernel/ptrace.c @@ -954,8 +954,8 @@ static int sve_set_common(struct task_struct *target, } /* Always zero V regs, FPSR, and FPCR */ - memset(¤t->thread.uw.fpsimd_state, 0, - sizeof(current->thread.uw.fpsimd_state)); + memset(&target->thread.uw.fpsimd_state, 0, + sizeof(target->thread.uw.fpsimd_state)); /* Registers: FPSIMD-only case */ From 43a9e6f63f5d8b003eeeea2e3b864a336b8d1c62 Mon Sep 17 00:00:00 2001 From: Karl Mehltretter Date: Wed, 29 Jul 2026 02:42:54 +0200 Subject: [PATCH 78/85] arm64/fpsimd: ptrace: Fix inactive SVE and SSVE regsets commit c3f83d021162571bcd87b062ec9e587828d2b7f3 upstream. sve_init_header_from_task() takes header as a pointer, so for the inactive mode header->size = sizeof(header); stores 8 rather than sizeof(struct user_sve_header), which is 16. Userspace sees an impossible size smaller than the header it describes. The inactive-mode check in sve_get_common() compares header.size against sizeof(header) as well, but there header is a struct, so the check can never fire. Reads of NT_ARM_SVE and NT_ARM_SSVE for the inactive mode therefore still return the other mode's FPSIMD data, exactly the situation the check was added to prevent. Fix the size, and make the check return the remaining membuf space instead of 0, which regset_get() would interpret as the entire (zero-filled) buffer having been populated. Fixes: b93e685ecff7 ("arm64/fpsimd: ptrace: Do not present register data for inactive mode") Assisted-by: Claude:claude-opus-5 Signed-off-by: Karl Mehltretter Signed-off-by: Will Deacon Signed-off-by: Bin Huang --- arch/arm64/kernel/ptrace.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c index fbaae3d98935..e662a622c7f0 100644 --- a/arch/arm64/kernel/ptrace.c +++ b/arch/arm64/kernel/ptrace.c @@ -800,7 +800,7 @@ static void sve_init_header_from_task(struct user_sve_header *header, if (active) header->size = SVE_PT_SIZE(vq, header->flags); else - header->size = sizeof(header); + header->size = sizeof(*header); header->max_size = SVE_PT_SIZE(sve_vq_from_vl(header->max_vl), SVE_PT_REGS_SVE); } @@ -836,7 +836,7 @@ static int sve_get_common(struct task_struct *target, * from the other mode to userspace. */ if (header.size == sizeof(header)) - return 0; + return to.left; switch ((header.flags & SVE_PT_REGS_MASK)) { case SVE_PT_REGS_FPSIMD: From a2e6430a12a36f7827d9554e400294cb3c58b0e2 Mon Sep 17 00:00:00 2001 From: Karl Mehltretter Date: Wed, 29 Jul 2026 02:42:55 +0200 Subject: [PATCH 79/85] kselftest/arm64: fp-ptrace: Fix checks for inactive SVE and SSVE regsets commit bd290e7fc245f9f85607f305fe8213c6c47a416c upstream. The checks on the header size reported for the inactive regset of the NT_ARM_SVE/NT_ARM_SSVE pair compare it against sizeof(sve), but sve is a struct user_sve_header *, so this is 8 rather than the intended 16. The kernel carried the identical typo when filling in the header, so kernel and test agreed on the wrong value and the test passed. Compare against sizeof(*sve), stop after the header checks for an inactive regset since it has no payload to compare, and prefill the buffer with a sentinel to verify that reading an inactive regset leaves everything after the header untouched. This also covers the getter's return value, which determines how many bytes ptrace copies back to userspace. Fixes: 864f3ddcd715 ("kselftest/arm64: fp-ptrace: Adjust to new inactive mode behaviour") Assisted-by: Claude:claude-opus-5 Signed-off-by: Karl Mehltretter Signed-off-by: Will Deacon Signed-off-by: Bin Huang --- tools/testing/selftests/arm64/fp/fp-ptrace.c | 47 +++++++++++++++++--- 1 file changed, 41 insertions(+), 6 deletions(-) diff --git a/tools/testing/selftests/arm64/fp/fp-ptrace.c b/tools/testing/selftests/arm64/fp/fp-ptrace.c index 191c47ca0ed8..0ad8ff199857 100644 --- a/tools/testing/selftests/arm64/fp/fp-ptrace.c +++ b/tools/testing/selftests/arm64/fp/fp-ptrace.c @@ -65,6 +65,9 @@ /* VL 128..2048 in powers of 2 */ #define MAX_NUM_VLS 5 +/* Sentinel for detecting buffer bytes the kernel did not write */ +#define REGSET_SENTINEL 0xa5 + /* * FPMR bits we can set without doing feature checks to see if values * are valid. @@ -181,6 +184,20 @@ static bool compare_buffer(const char *name, void *out, return false; } +static bool buffer_is_filled(const void *buffer, size_t size, + unsigned char value) +{ + const unsigned char *bytes = buffer; + size_t i; + + for (i = 0; i < size; i++) { + if (bytes[i] != value) + return false; + } + + return true; +} + struct test_config { int sve_vl_in; int sve_vl_expected; @@ -401,6 +418,7 @@ static bool check_ptrace_values_sve(pid_t child, struct test_config *config) struct user_sve_header *sve; struct user_fpsimd_state *fpsimd; struct iovec iov; + size_t buf_size; int ret, vq; bool pass = true; @@ -409,14 +427,16 @@ static bool check_ptrace_values_sve(pid_t child, struct test_config *config) vq = __sve_vq_from_vl(config->sve_vl_in); - iov.iov_len = SVE_PT_SVE_OFFSET + SVE_PT_SVE_SIZE(vq, SVE_PT_REGS_SVE); - iov.iov_base = malloc(iov.iov_len); + buf_size = SVE_PT_SVE_OFFSET + SVE_PT_SVE_SIZE(vq, SVE_PT_REGS_SVE); + iov.iov_len = buf_size; + iov.iov_base = malloc(buf_size); if (!iov.iov_base) { ksft_print_msg("OOM allocating %lu byte SVE buffer\n", iov.iov_len); return false; } + memset(iov.iov_base, REGSET_SENTINEL, buf_size); ret = ptrace(PTRACE_GETREGSET, child, NT_ARM_SVE, &iov); if (ret != 0) { ksft_print_msg("Failed to read initial SVE: %s (%d)\n", @@ -440,10 +460,16 @@ static bool check_ptrace_values_sve(pid_t child, struct test_config *config) } if (svcr_in & SVCR_SM) { - if (sve->size != sizeof(sve)) { + if (sve->size != sizeof(*sve)) { ksft_print_msg("NT_ARM_SVE reports data with PSTATE.SM\n"); pass = false; } + if (!buffer_is_filled(iov.iov_base + sizeof(*sve), + buf_size - sizeof(*sve), REGSET_SENTINEL)) { + ksft_print_msg("NT_ARM_SVE wrote beyond its header with PSTATE.SM\n"); + pass = false; + } + goto out; } else { if (sve->size != SVE_PT_SIZE(vq, sve->flags)) { ksft_print_msg("Mismatch in SVE header size: %d != %lu\n", @@ -485,6 +511,7 @@ static bool check_ptrace_values_ssve(pid_t child, struct test_config *config) struct user_sve_header *sve; struct user_fpsimd_state *fpsimd; struct iovec iov; + size_t buf_size; int ret, vq; bool pass = true; @@ -493,14 +520,16 @@ static bool check_ptrace_values_ssve(pid_t child, struct test_config *config) vq = __sve_vq_from_vl(config->sme_vl_in); - iov.iov_len = SVE_PT_SVE_OFFSET + SVE_PT_SVE_SIZE(vq, SVE_PT_REGS_SVE); - iov.iov_base = malloc(iov.iov_len); + buf_size = SVE_PT_SVE_OFFSET + SVE_PT_SVE_SIZE(vq, SVE_PT_REGS_SVE); + iov.iov_len = buf_size; + iov.iov_base = malloc(buf_size); if (!iov.iov_base) { ksft_print_msg("OOM allocating %lu byte SSVE buffer\n", iov.iov_len); return false; } + memset(iov.iov_base, REGSET_SENTINEL, buf_size); ret = ptrace(PTRACE_GETREGSET, child, NT_ARM_SSVE, &iov); if (ret != 0) { ksft_print_msg("Failed to read initial SSVE: %s (%d)\n", @@ -523,10 +552,16 @@ static bool check_ptrace_values_ssve(pid_t child, struct test_config *config) } if (!(svcr_in & SVCR_SM)) { - if (sve->size != sizeof(sve)) { + if (sve->size != sizeof(*sve)) { ksft_print_msg("NT_ARM_SSVE reports data without PSTATE.SM\n"); pass = false; } + if (!buffer_is_filled(iov.iov_base + sizeof(*sve), + buf_size - sizeof(*sve), REGSET_SENTINEL)) { + ksft_print_msg("NT_ARM_SSVE wrote beyond its header without PSTATE.SM\n"); + pass = false; + } + goto out; } else { if (sve->size != SVE_PT_SIZE(vq, sve->flags)) { ksft_print_msg("Mismatch in SSVE header size: %d != %lu\n", From 24cc45e7d03e402fd33acb5e01138a075888aa2f Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Tue, 20 Jan 2026 14:51:05 +0000 Subject: [PATCH 80/85] arm64/fpsimd: ptrace: Fix SVE writes on !SME systems commit 128a7494a9f15aad60cc6b7e3546bf481ac54a13 upstream. When SVE is supported but SME is not supported, a ptrace write to the NT_ARM_SVE regset can place the tracee into an invalid state where (non-streaming) SVE register data is stored in FP_STATE_SVE format but TIF_SVE is clear. This can result in a later warning from fpsimd_restore_current_state(), e.g. WARNING: CPU: 0 PID: 7214 at arch/arm64/kernel/fpsimd.c:383 fpsimd_restore_current_state+0x50c/0x748 When this happens, fpsimd_restore_current_state() will set TIF_SVE, placing the task into the correct state. This occurs before any other check of TIF_SVE can possibly occur, as other checks of TIF_SVE only happen while the FPSIMD/SVE/SME state is live. Thus, aside from the warning, there is no functional issue. This bug was introduced during rework to error handling in commit: 9f8bf718f2923 ("arm64/fpsimd: ptrace: Gracefully handle errors") ... where the setting of TIF_SVE was moved into a block which is only executed when system_supports_sme() is true. Fix this by removing the system_supports_sme() check. This ensures that TIF_SVE is set for (SVE-formatted) writes to NT_ARM_SVE, at the cost of unconditionally manipulating the tracee's saved svcr value. The manipulation of svcr is benign and inexpensive, and we already do similar elsewhere (e.g. during signal handling), so I don't think it's worth guarding this with system_supports_sme() checks. Aside from the above, there is no functional change. The 'type' argument to sve_set_common() is only set to ARM64_VEC_SME (in ssve_set())) when system_supports_sme(), so the ARM64_VEC_SME case in the switch statement is still unreachable when !system_supports_sme(). When CONFIG_ARM64_SME=n, the only caller of sve_set_common() is sve_set(), and the compiler can constant-fold for the case where type is ARM64_VEC_SVE, removing the logic for other cases. Reported-by: syzbot+d4ab35af21e99d07ce67@syzkaller.appspotmail.com Fixes: 9f8bf718f292 ("arm64/fpsimd: ptrace: Gracefully handle errors") Signed-off-by: Mark Rutland Cc: Cc: Mark Brown Cc: Will Deacon Reviewed-by: Mark Brown Signed-off-by: Catalin Marinas Signed-off-by: Bin Huang --- arch/arm64/kernel/ptrace.c | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c index e662a622c7f0..2514270df620 100644 --- a/arch/arm64/kernel/ptrace.c +++ b/arch/arm64/kernel/ptrace.c @@ -937,20 +937,18 @@ static int sve_set_common(struct task_struct *target, vq = sve_vq_from_vl(task_get_vl(target, type)); /* Enter/exit streaming mode */ - if (system_supports_sme()) { - switch (type) { - case ARM64_VEC_SVE: - target->thread.svcr &= ~SVCR_SM_MASK; - set_tsk_thread_flag(target, TIF_SVE); - break; - case ARM64_VEC_SME: - target->thread.svcr |= SVCR_SM_MASK; - set_tsk_thread_flag(target, TIF_SME); - break; - default: - WARN_ON_ONCE(1); - return -EINVAL; - } + switch (type) { + case ARM64_VEC_SVE: + target->thread.svcr &= ~SVCR_SM_MASK; + set_tsk_thread_flag(target, TIF_SVE); + break; + case ARM64_VEC_SME: + target->thread.svcr |= SVCR_SM_MASK; + set_tsk_thread_flag(target, TIF_SME); + break; + default: + WARN_ON_ONCE(1); + return -EINVAL; } /* Always zero V regs, FPSR, and FPCR */ From cafbb6e0265fec9918d9c01492a4327062797226 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Fri, 18 Jul 2025 23:14:50 +0100 Subject: [PATCH 81/85] kselftest/arm64: Test SME on SME only systems in fp-ptrace commit b021f45d39f37f67005948a96abea84736890463 upstream. When checking that the vector extensions are supported fp-ptrace currently only checks for SVE being supported which means that we get into a confused half configured state for SME only systems. Check for SME as well. Signed-off-by: Mark Brown Link: https://lore.kernel.org/r/20250718-arm64-fp-ptrace-sme-only-v1-1-3b96dd19a503@kernel.org Signed-off-by: Catalin Marinas Signed-off-by: Bin Huang --- tools/testing/selftests/arm64/fp/fp-ptrace.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/arm64/fp/fp-ptrace.c b/tools/testing/selftests/arm64/fp/fp-ptrace.c index 0ad8ff199857..04338d12d55e 100644 --- a/tools/testing/selftests/arm64/fp/fp-ptrace.c +++ b/tools/testing/selftests/arm64/fp/fp-ptrace.c @@ -1642,7 +1642,7 @@ int main(void) * Run the test set if there is no SVE or SME, with those we * have to pick a VL for each run. */ - if (!sve_supported()) { + if (!sve_supported() && !sme_supported()) { test_config.sve_vl_in = 0; test_config.sve_vl_expected = 0; test_config.sme_vl_in = 0; From 267796b3372bc6f6d353841186b480e089853a71 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Fri, 18 Jul 2025 23:14:51 +0100 Subject: [PATCH 82/85] kselftest/arm64: Fix SVE write data generation for SME only systems commit aa7d3c8bc27d32dec940c924d6d270fa312e731f upstream. fp-ptrace does not handle SME only systems correctly when generating data, on SME only systems scenarios where we are not in streaming mode will not have an expected vector length. This leads to attempts to do memcpy()s of zero byte arrays which can crash, fix this by skipping generation of SVE data for cases where we do not expect to have an active vector length. Signed-off-by: Mark Brown Link: https://lore.kernel.org/r/20250718-arm64-fp-ptrace-sme-only-v1-2-3b96dd19a503@kernel.org Signed-off-by: Catalin Marinas [ bh: Apply the same zero-length guards to the older sve_write function before its upstream rename. ] Signed-off-by: Bin Huang --- tools/testing/selftests/arm64/fp/fp-ptrace.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/testing/selftests/arm64/fp/fp-ptrace.c b/tools/testing/selftests/arm64/fp/fp-ptrace.c index 04338d12d55e..776de888d321 100644 --- a/tools/testing/selftests/arm64/fp/fp-ptrace.c +++ b/tools/testing/selftests/arm64/fp/fp-ptrace.c @@ -1169,6 +1169,9 @@ static void sve_write_expected(struct test_config *config) int vl = vl_expected(config); int sme_vq = __sve_vq_from_vl(config->sme_vl_expected); + if (!vl) + return; + fill_random(z_expected, __SVE_ZREGS_SIZE(__sve_vq_from_vl(vl))); fill_random(p_expected, __SVE_PREGS_SIZE(__sve_vq_from_vl(vl))); @@ -1196,6 +1199,9 @@ static void sve_write(pid_t child, struct test_config *config) vl = vl_expected(config); vq = __sve_vq_from_vl(vl); + if (!vl) + return; + iov.iov_len = SVE_PT_SVE_OFFSET + SVE_PT_SVE_SIZE(vq, SVE_PT_REGS_SVE); iov.iov_base = malloc(iov.iov_len); if (!iov.iov_base) { From 489a3e5b2001977c9fcdebcf9fbe36b121593347 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Fri, 18 Jul 2025 23:14:52 +0100 Subject: [PATCH 83/85] kselftest/arm64: Handle attempts to disable SM on SME only systems commit 4752dcc156f2090143296ff45f8e35c8ec3e1730 upstream. The ABI for disabling streaming mode via ptrace is to do a write via the SVE register set. Following the recent round of fixes to the ptrace code we don't support this operation on systems without SVE, which is detected as failures by fp-ptrace. Update the program so that it knows that this operation is not currently supported. Signed-off-by: Mark Brown Link: https://lore.kernel.org/r/20250718-arm64-fp-ptrace-sme-only-v1-3-3b96dd19a503@kernel.org Signed-off-by: Catalin Marinas Signed-off-by: Bin Huang --- tools/testing/selftests/arm64/fp/fp-ptrace.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/testing/selftests/arm64/fp/fp-ptrace.c b/tools/testing/selftests/arm64/fp/fp-ptrace.c index 776de888d321..e15fdc6a22c6 100644 --- a/tools/testing/selftests/arm64/fp/fp-ptrace.c +++ b/tools/testing/selftests/arm64/fp/fp-ptrace.c @@ -1096,6 +1096,9 @@ static bool sve_write_supported(struct test_config *config) if (config->sme_vl_in != config->sme_vl_expected) { return false; } + + if (!sve_supported()) + return false; } return true; From 2907609094e356428721d96e4ce86b723af4d062 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Mon, 18 Dec 2023 23:39:32 +0000 Subject: [PATCH 84/85] kselftest/arm64: Don't probe the current VL for unsupported vector types commit 9a802ddb2123e5adec394d35cd539cc0b15bc830 upstream. The vec-syscfg selftest verifies that setting the VL of the currently tested vector type does not disrupt the VL of the other vector type. To do this it records the current vector length for each type but neglects to guard this with a check for that vector type actually being supported. Add one, using a helper function which we also update all the other instances of this pattern. Signed-off-by: Mark Brown Link: https://lore.kernel.org/r/20231218-kselftest-arm64-vec-syscfg-rdvl-v1-1-0ac22d47e81f@kernel.org Signed-off-by: Will Deacon Signed-off-by: Bin Huang --- tools/testing/selftests/arm64/fp/vec-syscfg.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tools/testing/selftests/arm64/fp/vec-syscfg.c b/tools/testing/selftests/arm64/fp/vec-syscfg.c index 5f648b97a06f..ea9c7d47790f 100644 --- a/tools/testing/selftests/arm64/fp/vec-syscfg.c +++ b/tools/testing/selftests/arm64/fp/vec-syscfg.c @@ -66,6 +66,11 @@ static struct vec_data vec_data[] = { }, }; +static bool vec_type_supported(struct vec_data *data) +{ + return getauxval(data->hwcap_type) & data->hwcap; +} + static int stdio_read_integer(FILE *f, const char *what, int *val) { int n = 0; @@ -564,8 +569,11 @@ static void prctl_set_all_vqs(struct vec_data *data) return; } - for (i = 0; i < ARRAY_SIZE(vec_data); i++) + for (i = 0; i < ARRAY_SIZE(vec_data); i++) { + if (!vec_type_supported(&vec_data[i])) + continue; orig_vls[i] = vec_data[i].rdvl(); + } for (vq = SVE_VQ_MIN; vq <= SVE_VQ_MAX; vq++) { vl = sve_vl_from_vq(vq); @@ -594,7 +602,7 @@ static void prctl_set_all_vqs(struct vec_data *data) if (&vec_data[i] == data) continue; - if (!(getauxval(vec_data[i].hwcap_type) & vec_data[i].hwcap)) + if (!vec_type_supported(&vec_data[i])) continue; if (vec_data[i].rdvl() != orig_vls[i]) { @@ -765,7 +773,7 @@ int main(void) struct vec_data *data = &vec_data[i]; unsigned long supported; - supported = getauxval(data->hwcap_type) & data->hwcap; + supported = vec_type_supported(data); if (!supported) all_supported = false; From e9964f66595825dacf54d904e1284c527c94359c Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Mon, 9 Jun 2025 16:25:32 +0100 Subject: [PATCH 85/85] kselftest/arm64: Fix test for streaming FPSIMD write in sve-ptrace commit 94ab150010f430a36c72e2fe5b7da44a625a6ad5 upstream. Since f916dd32a943 ("arm64/fpsimd: ptrace: Mandate SVE payload for streaming-mode state") we do not support writing FPSIMD payload data when writing NT_ARM_SSVE but the sve-ptrace test has an explicit test for this being supported which was not updated to reflect the new behaviour. Fix the test to expect a failure when writing FPSIMD data to the streaming mode register set. Acked-by: Mark Rutland Signed-off-by: Mark Brown Link: https://lore.kernel.org/r/20250609-kselftest-arm64-ssve-fixups-v2-2-998fcfa6f240@kernel.org Signed-off-by: Catalin Marinas Signed-off-by: Bin Huang --- tools/testing/selftests/arm64/fp/sve-ptrace.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/arm64/fp/sve-ptrace.c b/tools/testing/selftests/arm64/fp/sve-ptrace.c index 408fb1c5c2f8..e47f96eea50d 100644 --- a/tools/testing/selftests/arm64/fp/sve-ptrace.c +++ b/tools/testing/selftests/arm64/fp/sve-ptrace.c @@ -300,8 +300,10 @@ static void ptrace_sve_fpsimd(pid_t child, const struct vec_type *type) p[j] = j; } + /* This should only succeed for SVE */ ret = set_sve(child, type, sve); - ksft_test_result(ret == 0, "%s FPSIMD set via SVE: %d\n", + ksft_test_result((type->regset == NT_ARM_SVE) == (ret == 0), + "%s FPSIMD set via SVE: %d\n", type->name, ret); if (ret) goto out;