From nobody Thu Oct 2 11:50:22 2025 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id 8D7DF2EBBB5 for ; Wed, 17 Sep 2025 11:08:54 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1758107335; cv=none; b=t2MSqf1yA2Q3dK1CE2tC9AEnuDx1loL4xgxjOx8jO0+Qev7nEE2YPhYwmoey2MO98T4qeX8KnSUIfaSG6QBwSb1NB2oYuu+jUB45p+cuyUDGVcaJkM9W5faxmxfgiAjOqkmsLybIR+qm+iC65FR4W4RwywT2pOcBSaAyiX0HPPA= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1758107335; c=relaxed/simple; bh=20QbCLjpe2fArK+DjDxhhuFdUUrwJ/AGa/wDAbICzTg=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=KX8var/DbA4/Kk3C3yH848WcppXacuDE7ryAkCZijIwrGc6dxd+0Ej3HdGz0/0YpSvKF4N8RBBBKFcBy51JNLSsacwAsm9RtkpofDD3dg8aRw1GA3E6++38/8EJu38YvYygvxkEjcH8qMngfrRbpwGbXL0KCbPCFtN3ke5fVKZA= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 99E1D2696; Wed, 17 Sep 2025 04:08:45 -0700 (PDT) Received: from e129823.cambridge.arm.com (e129823.arm.com [10.1.197.6]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id 905773F66E; Wed, 17 Sep 2025 04:08:51 -0700 (PDT) From: Yeoreum Yun To: catalin.marinas@arm.com, will@kernel.org, broonie@kernel.org, maz@kernel.org, oliver.upton@linux.dev, joey.gouly@arm.com, james.morse@arm.com, ardb@kernel.org, scott@os.amperecomputing.com, suzuki.poulose@arm.com, yuzenghui@huawei.com, mark.rutland@arm.com Cc: linux-arm-kernel@lists.infradead.org, kvmarm@lists.linux.dev, linux-kernel@vger.kernel.org, Yeoreum Yun Subject: [PATCH v8 1/5] arm64: cpufeature: add FEAT_LSUI Date: Wed, 17 Sep 2025 12:08:34 +0100 Message-Id: <20250917110838.917281-2-yeoreum.yun@arm.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20250917110838.917281-1-yeoreum.yun@arm.com> References: <20250917110838.917281-1-yeoreum.yun@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Since Armv9.6, FEAT_LSUI provides load/store instructions that allow privileged code to access user memory without clearing the PSTATE.PAN bit. Add CPU feature detection for FEAT_LSUI. Signed-off-by: Yeoreum Yun Reviewed-by: Catalin Marinas --- arch/arm64/kernel/cpufeature.c | 10 ++++++++++ arch/arm64/tools/cpucaps | 1 + 2 files changed, 11 insertions(+) diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index ef269a5a37e1..9f83d3fe941b 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -279,6 +279,7 @@ static const struct arm64_ftr_bits ftr_id_aa64isar2[] = =3D { =20 static const struct arm64_ftr_bits ftr_id_aa64isar3[] =3D { ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR3_E= L1_FPRCVT_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR3_EL= 1_LSUI_SHIFT, 4, ID_AA64ISAR3_EL1_LSUI_NI), ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR3_E= L1_FAMINMAX_SHIFT, 4, 0), ARM64_FTR_END, }; @@ -3156,6 +3157,15 @@ static const struct arm64_cpu_capabilities arm64_fea= tures[] =3D { .matches =3D has_cpuid_feature, ARM64_CPUID_FIELDS(ID_AA64PFR2_EL1, GCIE, IMP) }, +#ifdef CONFIG_AS_HAS_LSUI + { + .desc =3D "Unprivileged Load Store Instructions (LSUI)", + .capability =3D ARM64_HAS_LSUI, + .type =3D ARM64_CPUCAP_SYSTEM_FEATURE, + .matches =3D has_cpuid_feature, + ARM64_CPUID_FIELDS(ID_AA64ISAR3_EL1, LSUI, IMP) + }, +#endif {}, }; =20 diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index 9ff5cdbd2759..b728e5a2e05e 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -44,6 +44,7 @@ HAS_HCX HAS_LDAPR HAS_LPA2 HAS_LSE_ATOMICS +HAS_LSUI HAS_MOPS HAS_NESTED_VIRT HAS_BBML2_NOABORT --=20 LEVI:{C3F47F37-75D8-414A-A8BA-3980EC8A46D7} From nobody Thu Oct 2 11:50:22 2025 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id DF79C2F547F for ; Wed, 17 Sep 2025 11:08:56 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1758107338; cv=none; b=fK8fVxkM9Dk8HQf8a+8ZPMyb8M6dLvmne2dtFm5WLMX+P2GDPamAxLvLnlItjB5dJ8BfZUtIC7FR2f1XsLfiD+bvnluMcpy7sPjWXDGkcZSLZdYV5BFRDIoHnCeOV0wJ1XdO3jrmmo9rQ92atwC3ngK5ZruRMC0k9wTkdrELHNw= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1758107338; c=relaxed/simple; bh=EaXCQ2EJ2zVcQaIC9xmiLklVLUzPqC7aenGAqOdUpnc=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=IwiWZXWmrwBgfbLM05Rg0zcHU2r7nlyFK+FANqzcH72fo6uFTbAJBDJzM2/8BIl58TmddEsw3lDpj5jfKjB0x6/rQ9nLBGOQKzJ+UzGkrZ3G86cF9OdpV6ZT85eSiSfIOlnFyW2QQRHRUaEE1B+7CFfeXpktRwJ1gESrzr6NIPE= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 3331426A4; Wed, 17 Sep 2025 04:08:48 -0700 (PDT) Received: from e129823.cambridge.arm.com (e129823.arm.com [10.1.197.6]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id 20EF23F66E; Wed, 17 Sep 2025 04:08:54 -0700 (PDT) From: Yeoreum Yun To: catalin.marinas@arm.com, will@kernel.org, broonie@kernel.org, maz@kernel.org, oliver.upton@linux.dev, joey.gouly@arm.com, james.morse@arm.com, ardb@kernel.org, scott@os.amperecomputing.com, suzuki.poulose@arm.com, yuzenghui@huawei.com, mark.rutland@arm.com Cc: linux-arm-kernel@lists.infradead.org, kvmarm@lists.linux.dev, linux-kernel@vger.kernel.org, Yeoreum Yun Subject: [PATCH v8 2/5] KVM: arm64: expose FEAT_LSUI to guest Date: Wed, 17 Sep 2025 12:08:35 +0100 Message-Id: <20250917110838.917281-3-yeoreum.yun@arm.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20250917110838.917281-1-yeoreum.yun@arm.com> References: <20250917110838.917281-1-yeoreum.yun@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" expose FEAT_LSUI to guest. Signed-off-by: Yeoreum Yun Acked-by: Marc Zyngier Reviewed-by: Catalin Marinas --- arch/arm64/kvm/sys_regs.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index b29f72478a50..abdf19ae250e 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -1757,7 +1757,8 @@ static u64 __kvm_read_sanitised_id_reg(const struct k= vm_vcpu *vcpu, val &=3D ~ID_AA64ISAR2_EL1_WFxT; break; case SYS_ID_AA64ISAR3_EL1: - val &=3D ID_AA64ISAR3_EL1_FPRCVT | ID_AA64ISAR3_EL1_FAMINMAX; + val &=3D ID_AA64ISAR3_EL1_FPRCVT | ID_AA64ISAR3_EL1_FAMINMAX | + ID_AA64ISAR3_EL1_LSUI; break; case SYS_ID_AA64MMFR2_EL1: val &=3D ~ID_AA64MMFR2_EL1_CCIDX_MASK; @@ -3141,7 +3142,7 @@ static const struct sys_reg_desc sys_reg_descs[] =3D { ID_AA64ISAR2_EL1_APA3 | ID_AA64ISAR2_EL1_GPA3)), ID_WRITABLE(ID_AA64ISAR3_EL1, (ID_AA64ISAR3_EL1_FPRCVT | - ID_AA64ISAR3_EL1_FAMINMAX)), + ID_AA64ISAR3_EL1_FAMINMAX | ID_AA64ISAR3_EL1_LSUI)), ID_UNALLOCATED(6,4), ID_UNALLOCATED(6,5), ID_UNALLOCATED(6,6), --=20 LEVI:{C3F47F37-75D8-414A-A8BA-3980EC8A46D7} From nobody Thu Oct 2 11:50:22 2025 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id 57EB430BB9F for ; Wed, 17 Sep 2025 11:08:59 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1758107340; cv=none; b=YTDqUU0BpUc8+qg4nI/cLfymNgyeKd6t+CCkm8+855uByxCDauTKygWISia84pedG8m+97xMl1e5xh2LtfCHhmyCAjMdU6wjfaVQfOh5ai2zXneYw7iPU1Qzxv5u5qqj+mPJy/HHi+uXq76f5dY5eGteyvElLw/A5vsHNXGaDnU= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1758107340; c=relaxed/simple; bh=BhyM3yAonv/2/TCHGeyE8EMSG3cRlk4qFoT3wlB3Wk0=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=BQ1cuiXQxFTKQqgxpWpNg9en9hhAc2gg+VGwDYv9zA+c12Q3sWzVZoaS667FQFW6Vy09PzGakkSQGqOzz9EuTN92YkSqelJNyeF8yCj7siFLiAeZibOy7et5Yx052kGZ2n4JaXFlokYW9VpVfz1cgQalfnalN3xb0tOwiFEsqQM= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id BDFB2267F; Wed, 17 Sep 2025 04:08:50 -0700 (PDT) Received: from e129823.cambridge.arm.com (e129823.arm.com [10.1.197.6]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id AE77B3F66E; Wed, 17 Sep 2025 04:08:56 -0700 (PDT) From: Yeoreum Yun To: catalin.marinas@arm.com, will@kernel.org, broonie@kernel.org, maz@kernel.org, oliver.upton@linux.dev, joey.gouly@arm.com, james.morse@arm.com, ardb@kernel.org, scott@os.amperecomputing.com, suzuki.poulose@arm.com, yuzenghui@huawei.com, mark.rutland@arm.com Cc: linux-arm-kernel@lists.infradead.org, kvmarm@lists.linux.dev, linux-kernel@vger.kernel.org, Yeoreum Yun Subject: [PATCH v8 3/5] arm64: Kconfig: Detect toolchain support for LSUI Date: Wed, 17 Sep 2025 12:08:36 +0100 Message-Id: <20250917110838.917281-4-yeoreum.yun@arm.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20250917110838.917281-1-yeoreum.yun@arm.com> References: <20250917110838.917281-1-yeoreum.yun@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Since Armv9.6, FEAT_LSUI supplies the load/store instructions for previleged level to access to access user memory without clearing PSTATE.PAN bit. It's enough to add CONFIG_AS_HAS_LSUI only because the code for LSUI uses individual `.arch_extension` entries. Signed-off-by: Yeoreum Yun Reviewed-by: Catalin Marinas --- arch/arm64/Kconfig | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index e9bbfacc35a6..89a1a3771ed5 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -2239,6 +2239,11 @@ config ARM64_GCS =20 endmenu # "v9.4 architectural features" =20 +config AS_HAS_LSUI + def_bool $(as-instr,.arch_extension lsui) + help + Supported by LLVM 20+ and binutils 2.45+. + config ARM64_SVE bool "ARM Scalable Vector Extension support" default y --=20 LEVI:{C3F47F37-75D8-414A-A8BA-3980EC8A46D7} From nobody Thu Oct 2 11:50:22 2025 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id DC68C32B48E for ; Wed, 17 Sep 2025 11:09:01 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1758107343; cv=none; b=DZqc3NzfSn8jpdm/fb5hRQnoJgGZXRIDr6fXsRQwenZ8TN6faV1GAf9pAl2LwrFQ4W5w9f1gBjq26JfPevxddULB1EZVlLuxMIT6lObUUnSzUjHfgm37nvU8mJS1GHdJM/xKgysUEBlhcVePQKYhJd6CDLMbO3A79K2XFFJPkKI= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1758107343; c=relaxed/simple; bh=SscbY6KjcogaE502eJ5nQmaCURaWPLMy3N/x9vdSBX8=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=YcVuwzPZPMxuf5m3RODHjLBKI6D6/pWH5bk63LeZeAF0QpGl2rU2AyIF+bLZAkn8Wyndv7Q2gLj+zOL39yOafwURnGbJFAlkrQ00THL0VSmF9QMxwToh83BD58aYfSo4npIIH+QeUrRXuzC2/UM2JtTRV5lpBxccTggJjHbVUPU= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 4F3662696; Wed, 17 Sep 2025 04:08:53 -0700 (PDT) Received: from e129823.cambridge.arm.com (e129823.arm.com [10.1.197.6]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id 482BB3F66E; Wed, 17 Sep 2025 04:08:59 -0700 (PDT) From: Yeoreum Yun To: catalin.marinas@arm.com, will@kernel.org, broonie@kernel.org, maz@kernel.org, oliver.upton@linux.dev, joey.gouly@arm.com, james.morse@arm.com, ardb@kernel.org, scott@os.amperecomputing.com, suzuki.poulose@arm.com, yuzenghui@huawei.com, mark.rutland@arm.com Cc: linux-arm-kernel@lists.infradead.org, kvmarm@lists.linux.dev, linux-kernel@vger.kernel.org, Yeoreum Yun Subject: [PATCH v8 4/5] arm64: futex: refactor futex atomic operation Date: Wed, 17 Sep 2025 12:08:37 +0100 Message-Id: <20250917110838.917281-5-yeoreum.yun@arm.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20250917110838.917281-1-yeoreum.yun@arm.com> References: <20250917110838.917281-1-yeoreum.yun@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Refactor futex atomic operations using ll/sc method with clearing PSTATE.PAN to prepare to apply FEAT_LSUI on them. Signed-off-by: Yeoreum Yun Reviewed-by: Catalin Marinas --- arch/arm64/include/asm/futex.h | 128 +++++++++++++++++++++------------ 1 file changed, 82 insertions(+), 46 deletions(-) diff --git a/arch/arm64/include/asm/futex.h b/arch/arm64/include/asm/futex.h index bc06691d2062..f8cb674bdb3f 100644 --- a/arch/arm64/include/asm/futex.h +++ b/arch/arm64/include/asm/futex.h @@ -7,17 +7,21 @@ =20 #include #include +#include =20 #include =20 #define FUTEX_MAX_LOOPS 128 /* What's the largest number you can think of?= */ =20 -#define __futex_atomic_op(insn, ret, oldval, uaddr, tmp, oparg) \ -do { \ +#define LLSC_FUTEX_ATOMIC_OP(op, insn) \ +static __always_inline int \ +__llsc_futex_atomic_##op(int oparg, u32 __user *uaddr, int *oval) \ +{ \ unsigned int loops =3D FUTEX_MAX_LOOPS; \ + int ret, oldval, tmp; \ \ uaccess_enable_privileged(); \ - asm volatile( \ + asm volatile("// __llsc_futex_atomic_" #op "\n" \ " prfm pstl1strm, %2\n" \ "1: ldxr %w1, %2\n" \ insn "\n" \ @@ -35,45 +39,103 @@ do { \ : "r" (oparg), "Ir" (-EAGAIN) \ : "memory"); \ uaccess_disable_privileged(); \ -} while (0) + \ + if (!ret) \ + *oval =3D oldval; \ + \ + return ret; \ +} + +LLSC_FUTEX_ATOMIC_OP(add, "add %w3, %w1, %w5") +LLSC_FUTEX_ATOMIC_OP(or, "orr %w3, %w1, %w5") +LLSC_FUTEX_ATOMIC_OP(and, "and %w3, %w1, %w5") +LLSC_FUTEX_ATOMIC_OP(eor, "eor %w3, %w1, %w5") +LLSC_FUTEX_ATOMIC_OP(set, "mov %w3, %w5") + +static __always_inline int +__llsc_futex_cmpxchg(u32 __user *uaddr, u32 oldval, u32 newval, u32 *oval) +{ + int ret =3D 0; + unsigned int loops =3D FUTEX_MAX_LOOPS; + u32 val, tmp; + + uaccess_enable_privileged(); + asm volatile("//__llsc_futex_cmpxchg\n" +" prfm pstl1strm, %2\n" +"1: ldxr %w1, %2\n" +" eor %w3, %w1, %w5\n" +" cbnz %w3, 4f\n" +"2: stlxr %w3, %w6, %2\n" +" cbz %w3, 3f\n" +" sub %w4, %w4, %w3\n" +" cbnz %w4, 1b\n" +" mov %w0, %w7\n" +"3:\n" +" dmb ish\n" +"4:\n" + _ASM_EXTABLE_UACCESS_ERR(1b, 4b, %w0) + _ASM_EXTABLE_UACCESS_ERR(2b, 4b, %w0) + : "+r" (ret), "=3D&r" (val), "+Q" (*uaddr), "=3D&r" (tmp), "+r" (loops) + : "r" (oldval), "r" (newval), "Ir" (-EAGAIN) + : "memory"); + uaccess_disable_privileged(); + + if (!ret) + *oval =3D val; + + return ret; +} + +#define FUTEX_ATOMIC_OP(op) \ +static __always_inline int \ +__futex_atomic_##op(int oparg, u32 __user *uaddr, int *oval) \ +{ \ + return __llsc_futex_atomic_##op(oparg, uaddr, oval); \ +} + +FUTEX_ATOMIC_OP(add) +FUTEX_ATOMIC_OP(or) +FUTEX_ATOMIC_OP(and) +FUTEX_ATOMIC_OP(eor) +FUTEX_ATOMIC_OP(set) + +static __always_inline int +__futex_cmpxchg(u32 __user *uaddr, u32 oldval, u32 newval, u32 *oval) +{ + return __llsc_futex_cmpxchg(uaddr, oldval, newval, oval); +} =20 static inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *_uad= dr) { - int oldval =3D 0, ret, tmp; - u32 __user *uaddr =3D __uaccess_mask_ptr(_uaddr); + int ret; + u32 __user *uaddr; =20 if (!access_ok(_uaddr, sizeof(u32))) return -EFAULT; =20 + uaddr =3D __uaccess_mask_ptr(_uaddr); + switch (op) { case FUTEX_OP_SET: - __futex_atomic_op("mov %w3, %w5", - ret, oldval, uaddr, tmp, oparg); + ret =3D __futex_atomic_set(oparg, uaddr, oval); break; case FUTEX_OP_ADD: - __futex_atomic_op("add %w3, %w1, %w5", - ret, oldval, uaddr, tmp, oparg); + ret =3D __futex_atomic_add(oparg, uaddr, oval); break; case FUTEX_OP_OR: - __futex_atomic_op("orr %w3, %w1, %w5", - ret, oldval, uaddr, tmp, oparg); + ret =3D __futex_atomic_or(oparg, uaddr, oval); break; case FUTEX_OP_ANDN: - __futex_atomic_op("and %w3, %w1, %w5", - ret, oldval, uaddr, tmp, ~oparg); + ret =3D __futex_atomic_and(~oparg, uaddr, oval); break; case FUTEX_OP_XOR: - __futex_atomic_op("eor %w3, %w1, %w5", - ret, oldval, uaddr, tmp, oparg); + ret =3D __futex_atomic_eor(oparg, uaddr, oval); break; default: ret =3D -ENOSYS; } =20 - if (!ret) - *oval =3D oldval; - return ret; } =20 @@ -81,40 +143,14 @@ static inline int futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *_uaddr, u32 oldval, u32 newval) { - int ret =3D 0; - unsigned int loops =3D FUTEX_MAX_LOOPS; - u32 val, tmp; u32 __user *uaddr; =20 if (!access_ok(_uaddr, sizeof(u32))) return -EFAULT; =20 uaddr =3D __uaccess_mask_ptr(_uaddr); - uaccess_enable_privileged(); - asm volatile("// futex_atomic_cmpxchg_inatomic\n" -" prfm pstl1strm, %2\n" -"1: ldxr %w1, %2\n" -" sub %w3, %w1, %w5\n" -" cbnz %w3, 4f\n" -"2: stlxr %w3, %w6, %2\n" -" cbz %w3, 3f\n" -" sub %w4, %w4, %w3\n" -" cbnz %w4, 1b\n" -" mov %w0, %w7\n" -"3:\n" -" dmb ish\n" -"4:\n" - _ASM_EXTABLE_UACCESS_ERR(1b, 4b, %w0) - _ASM_EXTABLE_UACCESS_ERR(2b, 4b, %w0) - : "+r" (ret), "=3D&r" (val), "+Q" (*uaddr), "=3D&r" (tmp), "+r" (loops) - : "r" (oldval), "r" (newval), "Ir" (-EAGAIN) - : "memory"); - uaccess_disable_privileged(); - - if (!ret) - *uval =3D val; =20 - return ret; + return __futex_cmpxchg(uaddr, oldval, newval, uval); } =20 #endif /* __ASM_FUTEX_H */ --=20 LEVI:{C3F47F37-75D8-414A-A8BA-3980EC8A46D7} From nobody Thu Oct 2 11:50:22 2025 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id A244F2EAB89 for ; Wed, 17 Sep 2025 11:09:04 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1758107346; cv=none; b=SvRZqQdl6GUVVlY+uXY3yQwg1jPThwBJ0CuFfI0sTdlTE0eRShrVs0bQJmwd9xUuUcdtfqncqrqO/mRCCO/GWoA7KfIL7lmP8NFmEHG/wIkTIF4BzpgIJrCWkb10aBiLqOoAdNyIKmR8UZb1TzuXa++wssOcXO72zo1UjK2FwMo= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1758107346; c=relaxed/simple; bh=yG46QiIn2E8l+bpjPLStSV45GrTXN4pUWo7SrRu814Q=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=BGgbQHQxQdGXFBFQZrYNLO/+uRlgcexW06gM9tov4cWVzoQraj+KBD3xEPB/bB7N2G3GbCXGX4EFCVXI1jCOmJu+YAJkSj5A9mJ1k8L0NFo3/gVDu5ukQvYgJbOS4w5h5LFD3Z9siSFf7C+Bmcu47p0sOqy8A2oHCyBdDmurT8c= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id D7EF5267F; Wed, 17 Sep 2025 04:08:55 -0700 (PDT) Received: from e129823.cambridge.arm.com (e129823.arm.com [10.1.197.6]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id CD5EE3F66E; Wed, 17 Sep 2025 04:09:01 -0700 (PDT) From: Yeoreum Yun To: catalin.marinas@arm.com, will@kernel.org, broonie@kernel.org, maz@kernel.org, oliver.upton@linux.dev, joey.gouly@arm.com, james.morse@arm.com, ardb@kernel.org, scott@os.amperecomputing.com, suzuki.poulose@arm.com, yuzenghui@huawei.com, mark.rutland@arm.com Cc: linux-arm-kernel@lists.infradead.org, kvmarm@lists.linux.dev, linux-kernel@vger.kernel.org, Yeoreum Yun Subject: [PATCH v8 5/5] arm64: futex: support futex with FEAT_LSUI Date: Wed, 17 Sep 2025 12:08:38 +0100 Message-Id: <20250917110838.917281-6-yeoreum.yun@arm.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20250917110838.917281-1-yeoreum.yun@arm.com> References: <20250917110838.917281-1-yeoreum.yun@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Current futex atomic operations are implemented with ll/sc instructions and clearing PSTATE.PAN. Since Armv9.6, FEAT_LSUI supplies not only load/store instructions but also atomic operation for user memory access in kernel it doesn't need to clear PSTATE.PAN bit anymore. With theses instructions some of futex atomic operations don't need to be implmented with ldxr/stlxr pair instead can be implmented with one atomic operation supplied by FEAT_LSUI. However, some of futex atomic operation don't have matched instructuion i.e) eor or cmpxchg with word size. For those operation, uses cas{al}t to implement them. Signed-off-by: Yeoreum Yun --- arch/arm64/include/asm/futex.h | 136 ++++++++++++++++++++++++++++++++- 1 file changed, 135 insertions(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/futex.h b/arch/arm64/include/asm/futex.h index f8cb674bdb3f..683291700ff5 100644 --- a/arch/arm64/include/asm/futex.h +++ b/arch/arm64/include/asm/futex.h @@ -9,6 +9,8 @@ #include #include =20 +#include +#include #include =20 #define FUTEX_MAX_LOOPS 128 /* What's the largest number you can think of?= */ @@ -86,11 +88,143 @@ __llsc_futex_cmpxchg(u32 __user *uaddr, u32 oldval, u3= 2 newval, u32 *oval) return ret; } =20 +#ifdef CONFIG_AS_HAS_LSUI + +/* + * When the LSUI feature is present, the CPU also implements PAN, because + * FEAT_PAN has been mandatory since Armv8.1. Therefore, there is no need = to + * call uaccess_ttbr0_enable()/uaccess_ttbr0_disable() around each LSUI + * operation. + */ + +#define __LSUI_PREAMBLE ".arch_extension lsui\n" + +#define LSUI_FUTEX_ATOMIC_OP(op, asm_op, mb) \ +static __always_inline int \ +__lsui_futex_atomic_##op(int oparg, u32 __user *uaddr, int *oval) \ +{ \ + int ret =3D 0; \ + int oldval; \ + \ + asm volatile("// __lsui_futex_atomic_" #op "\n" \ + __LSUI_PREAMBLE \ +"1: " #asm_op #mb " %w3, %w2, %1\n" \ +"2:\n" \ + _ASM_EXTABLE_UACCESS_ERR(1b, 2b, %w0) \ + : "+r" (ret), "+Q" (*uaddr), "=3Dr" (oldval) \ + : "r" (oparg) \ + : "memory"); \ + \ + if (!ret) \ + *oval =3D oldval; \ + \ + return ret; \ +} + +LSUI_FUTEX_ATOMIC_OP(add, ldtadd, al) +LSUI_FUTEX_ATOMIC_OP(or, ldtset, al) +LSUI_FUTEX_ATOMIC_OP(andnot, ldtclr, al) +LSUI_FUTEX_ATOMIC_OP(set, swpt, al) + +static __always_inline int +__lsui_cmpxchg64(u64 __user *uaddr, u64 *oldval, u64 newval) +{ + int ret =3D 0; + + asm volatile("// __lsui_cmpxchg64\n" + __LSUI_PREAMBLE +"1: casalt %x2, %x3, %1\n" +"2:\n" + _ASM_EXTABLE_UACCESS_ERR(1b, 2b, %w0) + : "+r" (ret), "+Q" (*uaddr), "+r" (*oldval) + : "r" (newval) + : "memory"); + + return ret; +} + +static __always_inline int +__lsui_cmpxchg32(u32 __user *uaddr, u32 oldval, u32 newval, u32 *oval) +{ + u64 __user *uaddr_al; + u64 oval64, nval64, tmp; + static const u64 hi_mask =3D IS_ENABLED(CONFIG_CPU_LITTLE_ENDIAN) ? + GENMASK_U64(63, 32): GENMASK_U64(31, 0); + static const u8 hi_shift =3D IS_ENABLED(CONFIG_CPU_LITTLE_ENDIAN) ? 32 : = 0; + static const u8 lo_shift =3D IS_ENABLED(CONFIG_CPU_LITTLE_ENDIAN) ? 0 : 3= 2; + + uaddr_al =3D (u64 __user *) PTR_ALIGN_DOWN(uaddr, sizeof(u64)); + if (get_user(oval64, uaddr_al)) + return -EFAULT; + + if ((u32 __user *)uaddr_al !=3D uaddr) { + nval64 =3D ((oval64 & ~hi_mask) | ((u64)newval << hi_shift)); + oval64 =3D ((oval64 & ~hi_mask) | ((u64)oldval << hi_shift)); + } else { + nval64 =3D ((oval64 & hi_mask) | ((u64)newval << lo_shift)); + oval64 =3D ((oval64 & hi_mask) | ((u64)oldval << lo_shift)); + } + + tmp =3D oval64; + + if (__lsui_cmpxchg64(uaddr_al, &oval64, nval64)) + return -EFAULT; + + if (tmp !=3D oval64) + return -EAGAIN; + + *oval =3D oldval; + + return 0; +} + +static __always_inline int +__lsui_futex_atomic_and(int oparg, u32 __user *uaddr, int *oval) +{ + return __lsui_futex_atomic_andnot(~oparg, uaddr, oval); +} + +static __always_inline int +__lsui_futex_atomic_eor(int oparg, u32 __user *uaddr, int *oval) +{ + u32 oldval, newval; + + /* + * there are no ldteor/stteor instructions... + */ + if (get_user(oldval, uaddr)) + return -EFAULT; + + newval =3D oldval ^ oparg; + + return __lsui_cmpxchg32(uaddr, oldval, newval, oval); + +} + +static __always_inline int +__lsui_futex_cmpxchg(u32 __user *uaddr, u32 oldval, u32 newval, u32 *oval) +{ + return __lsui_cmpxchg32(uaddr, oldval, newval, oval); +} + +#define __lsui_llsc_body(op, ...) \ +({ \ + alternative_has_cap_likely(ARM64_HAS_LSUI) ? \ + __lsui_##op(__VA_ARGS__) : __llsc_##op(__VA_ARGS__); \ +}) + +#else /* CONFIG_AS_HAS_LSUI */ + +#define __lsui_llsc_body(op, ...) __llsc_##op(__VA_ARGS__) + +#endif /* CONFIG_AS_HAS_LSUI */ + + #define FUTEX_ATOMIC_OP(op) \ static __always_inline int \ __futex_atomic_##op(int oparg, u32 __user *uaddr, int *oval) \ { \ - return __llsc_futex_atomic_##op(oparg, uaddr, oval); \ + return __lsui_llsc_body(futex_atomic_##op, oparg, uaddr, oval); \ } =20 FUTEX_ATOMIC_OP(add) --=20 LEVI:{C3F47F37-75D8-414A-A8BA-3980EC8A46D7}