From nobody Thu Oct 2 13:03:53 2025 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id A244F2EAB89 for ; Wed, 17 Sep 2025 11:09:04 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1758107346; cv=none; b=SvRZqQdl6GUVVlY+uXY3yQwg1jPThwBJ0CuFfI0sTdlTE0eRShrVs0bQJmwd9xUuUcdtfqncqrqO/mRCCO/GWoA7KfIL7lmP8NFmEHG/wIkTIF4BzpgIJrCWkb10aBiLqOoAdNyIKmR8UZb1TzuXa++wssOcXO72zo1UjK2FwMo= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1758107346; c=relaxed/simple; bh=yG46QiIn2E8l+bpjPLStSV45GrTXN4pUWo7SrRu814Q=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=BGgbQHQxQdGXFBFQZrYNLO/+uRlgcexW06gM9tov4cWVzoQraj+KBD3xEPB/bB7N2G3GbCXGX4EFCVXI1jCOmJu+YAJkSj5A9mJ1k8L0NFo3/gVDu5ukQvYgJbOS4w5h5LFD3Z9siSFf7C+Bmcu47p0sOqy8A2oHCyBdDmurT8c= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id D7EF5267F; Wed, 17 Sep 2025 04:08:55 -0700 (PDT) Received: from e129823.cambridge.arm.com (e129823.arm.com [10.1.197.6]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id CD5EE3F66E; Wed, 17 Sep 2025 04:09:01 -0700 (PDT) From: Yeoreum Yun To: catalin.marinas@arm.com, will@kernel.org, broonie@kernel.org, maz@kernel.org, oliver.upton@linux.dev, joey.gouly@arm.com, james.morse@arm.com, ardb@kernel.org, scott@os.amperecomputing.com, suzuki.poulose@arm.com, yuzenghui@huawei.com, mark.rutland@arm.com Cc: linux-arm-kernel@lists.infradead.org, kvmarm@lists.linux.dev, linux-kernel@vger.kernel.org, Yeoreum Yun Subject: [PATCH v8 5/5] arm64: futex: support futex with FEAT_LSUI Date: Wed, 17 Sep 2025 12:08:38 +0100 Message-Id: <20250917110838.917281-6-yeoreum.yun@arm.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20250917110838.917281-1-yeoreum.yun@arm.com> References: <20250917110838.917281-1-yeoreum.yun@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Current futex atomic operations are implemented with ll/sc instructions and clearing PSTATE.PAN. Since Armv9.6, FEAT_LSUI supplies not only load/store instructions but also atomic operation for user memory access in kernel it doesn't need to clear PSTATE.PAN bit anymore. With theses instructions some of futex atomic operations don't need to be implmented with ldxr/stlxr pair instead can be implmented with one atomic operation supplied by FEAT_LSUI. However, some of futex atomic operation don't have matched instructuion i.e) eor or cmpxchg with word size. For those operation, uses cas{al}t to implement them. Signed-off-by: Yeoreum Yun --- arch/arm64/include/asm/futex.h | 136 ++++++++++++++++++++++++++++++++- 1 file changed, 135 insertions(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/futex.h b/arch/arm64/include/asm/futex.h index f8cb674bdb3f..683291700ff5 100644 --- a/arch/arm64/include/asm/futex.h +++ b/arch/arm64/include/asm/futex.h @@ -9,6 +9,8 @@ #include #include =20 +#include +#include #include =20 #define FUTEX_MAX_LOOPS 128 /* What's the largest number you can think of?= */ @@ -86,11 +88,143 @@ __llsc_futex_cmpxchg(u32 __user *uaddr, u32 oldval, u3= 2 newval, u32 *oval) return ret; } =20 +#ifdef CONFIG_AS_HAS_LSUI + +/* + * When the LSUI feature is present, the CPU also implements PAN, because + * FEAT_PAN has been mandatory since Armv8.1. Therefore, there is no need = to + * call uaccess_ttbr0_enable()/uaccess_ttbr0_disable() around each LSUI + * operation. + */ + +#define __LSUI_PREAMBLE ".arch_extension lsui\n" + +#define LSUI_FUTEX_ATOMIC_OP(op, asm_op, mb) \ +static __always_inline int \ +__lsui_futex_atomic_##op(int oparg, u32 __user *uaddr, int *oval) \ +{ \ + int ret =3D 0; \ + int oldval; \ + \ + asm volatile("// __lsui_futex_atomic_" #op "\n" \ + __LSUI_PREAMBLE \ +"1: " #asm_op #mb " %w3, %w2, %1\n" \ +"2:\n" \ + _ASM_EXTABLE_UACCESS_ERR(1b, 2b, %w0) \ + : "+r" (ret), "+Q" (*uaddr), "=3Dr" (oldval) \ + : "r" (oparg) \ + : "memory"); \ + \ + if (!ret) \ + *oval =3D oldval; \ + \ + return ret; \ +} + +LSUI_FUTEX_ATOMIC_OP(add, ldtadd, al) +LSUI_FUTEX_ATOMIC_OP(or, ldtset, al) +LSUI_FUTEX_ATOMIC_OP(andnot, ldtclr, al) +LSUI_FUTEX_ATOMIC_OP(set, swpt, al) + +static __always_inline int +__lsui_cmpxchg64(u64 __user *uaddr, u64 *oldval, u64 newval) +{ + int ret =3D 0; + + asm volatile("// __lsui_cmpxchg64\n" + __LSUI_PREAMBLE +"1: casalt %x2, %x3, %1\n" +"2:\n" + _ASM_EXTABLE_UACCESS_ERR(1b, 2b, %w0) + : "+r" (ret), "+Q" (*uaddr), "+r" (*oldval) + : "r" (newval) + : "memory"); + + return ret; +} + +static __always_inline int +__lsui_cmpxchg32(u32 __user *uaddr, u32 oldval, u32 newval, u32 *oval) +{ + u64 __user *uaddr_al; + u64 oval64, nval64, tmp; + static const u64 hi_mask =3D IS_ENABLED(CONFIG_CPU_LITTLE_ENDIAN) ? + GENMASK_U64(63, 32): GENMASK_U64(31, 0); + static const u8 hi_shift =3D IS_ENABLED(CONFIG_CPU_LITTLE_ENDIAN) ? 32 : = 0; + static const u8 lo_shift =3D IS_ENABLED(CONFIG_CPU_LITTLE_ENDIAN) ? 0 : 3= 2; + + uaddr_al =3D (u64 __user *) PTR_ALIGN_DOWN(uaddr, sizeof(u64)); + if (get_user(oval64, uaddr_al)) + return -EFAULT; + + if ((u32 __user *)uaddr_al !=3D uaddr) { + nval64 =3D ((oval64 & ~hi_mask) | ((u64)newval << hi_shift)); + oval64 =3D ((oval64 & ~hi_mask) | ((u64)oldval << hi_shift)); + } else { + nval64 =3D ((oval64 & hi_mask) | ((u64)newval << lo_shift)); + oval64 =3D ((oval64 & hi_mask) | ((u64)oldval << lo_shift)); + } + + tmp =3D oval64; + + if (__lsui_cmpxchg64(uaddr_al, &oval64, nval64)) + return -EFAULT; + + if (tmp !=3D oval64) + return -EAGAIN; + + *oval =3D oldval; + + return 0; +} + +static __always_inline int +__lsui_futex_atomic_and(int oparg, u32 __user *uaddr, int *oval) +{ + return __lsui_futex_atomic_andnot(~oparg, uaddr, oval); +} + +static __always_inline int +__lsui_futex_atomic_eor(int oparg, u32 __user *uaddr, int *oval) +{ + u32 oldval, newval; + + /* + * there are no ldteor/stteor instructions... + */ + if (get_user(oldval, uaddr)) + return -EFAULT; + + newval =3D oldval ^ oparg; + + return __lsui_cmpxchg32(uaddr, oldval, newval, oval); + +} + +static __always_inline int +__lsui_futex_cmpxchg(u32 __user *uaddr, u32 oldval, u32 newval, u32 *oval) +{ + return __lsui_cmpxchg32(uaddr, oldval, newval, oval); +} + +#define __lsui_llsc_body(op, ...) \ +({ \ + alternative_has_cap_likely(ARM64_HAS_LSUI) ? \ + __lsui_##op(__VA_ARGS__) : __llsc_##op(__VA_ARGS__); \ +}) + +#else /* CONFIG_AS_HAS_LSUI */ + +#define __lsui_llsc_body(op, ...) __llsc_##op(__VA_ARGS__) + +#endif /* CONFIG_AS_HAS_LSUI */ + + #define FUTEX_ATOMIC_OP(op) \ static __always_inline int \ __futex_atomic_##op(int oparg, u32 __user *uaddr, int *oval) \ { \ - return __llsc_futex_atomic_##op(oparg, uaddr, oval); \ + return __lsui_llsc_body(futex_atomic_##op, oparg, uaddr, oval); \ } =20 FUTEX_ATOMIC_OP(add) --=20 LEVI:{C3F47F37-75D8-414A-A8BA-3980EC8A46D7}