From nobody Fri May 8 01:39:16 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 1E76DC433FE for ; Sun, 15 May 2022 18:42:37 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S238400AbiEOSme (ORCPT ); Sun, 15 May 2022 14:42:34 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:35994 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S238372AbiEOSma (ORCPT ); Sun, 15 May 2022 14:42:30 -0400 Received: from mail-ej1-x62c.google.com (mail-ej1-x62c.google.com [IPv6:2a00:1450:4864:20::62c]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 2E38CDE7 for ; Sun, 15 May 2022 11:42:29 -0700 (PDT) Received: by mail-ej1-x62c.google.com with SMTP id l18so24846069ejc.7 for ; Sun, 15 May 2022 11:42:29 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20210112; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=KXuxsvvmzObesTCgGUdp3DjkR6F9yj2J25L79TJ+yVs=; b=Uj3BfnqK/PmvLMxm1jp32TG/3B40iMvsJHg7IfNSusjVrvdAp56m23eiIXkrAwCb4T iE+7bBSHW3vFA52sNt8UDCbohdsxkYhhKxHqUAyYCH2RSQQy+g81sH3WVeItnC0jy+43 5D/B+p95bcX3nfnTftSTxVC8b0DDVTlYfOdXXFLlNXB1+EZxCsnw2oDebhL/UjbYbpPH V5lSRHcKLZ8Yc8mh7Djf9uxLaYLIoeTrna0QQCA2+/wHH0h6TsuLO+N+owZtIh8C28N4 DUrD9wpSR4/SAAHNB1bSb2oHJg/bGME7xjcRmO223wZjaunRZE/dy2xw08N9Q4Xe3tPf svZA== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=KXuxsvvmzObesTCgGUdp3DjkR6F9yj2J25L79TJ+yVs=; b=mUvlLXZiAiKZvl74WJtvba0e2BhrpOljMCjqunL4J8Htrb9qOCLBDvbRmYIPtugF3N e1iC9U8aUivEC5b+L3LLe5I9A2iebYiDIEV06E6fTPnY8671u1LUijXXmk0wUQbaWrd6 k5EYGFUx6cQHj8C7LCiuyM6OH9+qtDqeNE1sWo1GAFp6KkhU1MzspIqRNn1Zqg+lvShr UFKcTk9syUIDSequfQ5z9P1fxHSmY6kozGf+DVmhfDB3yLFBzzg5JNSVu0UNlr0q1o4a czo9HHSDo98SSRBz3JizC+a6ufHcM4llABqFfsOJGdo6658HXJlPPJElTTz8RzedqpcG ZWNw== X-Gm-Message-State: AOAM530+TLDr6GtYjuA9pBpvb3hMJ3Ye5+x5kPbeGtM8x1YutGjNgXgg 80bfhEP7Bm/AixVrTUmoiVo= X-Google-Smtp-Source: ABdhPJwmEoJLpzsl3G+0/bDswbbrNULzSz7kliPeluMB6W1M4y5SVrDdF4tYBj3xh+ENkY7qZ4/7eA== X-Received: by 2002:a17:906:2989:b0:6f3:a215:8426 with SMTP id x9-20020a170906298900b006f3a2158426mr12613463eje.725.1652640147724; Sun, 15 May 2022 11:42:27 -0700 (PDT) Received: from localhost.localdomain (93-103-18-160.static.t-2.net. [93.103.18.160]) by smtp.gmail.com with ESMTPSA id hv7-20020a17090760c700b006f3ef214e62sm2861898ejc.200.2022.05.15.11.42.26 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Sun, 15 May 2022 11:42:27 -0700 (PDT) From: Uros Bizjak To: x86@kernel.org, linux-kernel@vger.kernel.org Cc: Uros Bizjak , Thomas Gleixner , Ingo Molnar , Borislav Petkov , Dave Hansen , "H. Peter Anvin" , Will Deacon , Peter Zijlstra , Boqun Feng , Mark Rutland , "Paul E. McKenney" , Marco Elver Subject: [PATCH v3 1/2] locking/atomic: Add generic try_cmpxchg64 support Date: Sun, 15 May 2022 20:42:03 +0200 Message-Id: <20220515184205.103089-2-ubizjak@gmail.com> X-Mailer: git-send-email 2.35.1 In-Reply-To: <20220515184205.103089-1-ubizjak@gmail.com> References: <20220515184205.103089-1-ubizjak@gmail.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" Add generic support for try_cmpxchg64{,_acquire,_release,_relaxed} and their falbacks involving cmpxchg64. Signed-off-by: Uros Bizjak Cc: Thomas Gleixner Cc: Ingo Molnar Cc: Borislav Petkov Cc: Dave Hansen Cc: "H. Peter Anvin" Cc: Will Deacon Cc: Peter Zijlstra Cc: Boqun Feng Cc: Mark Rutland Cc: "Paul E. McKenney" Cc: Marco Elver --- include/linux/atomic/atomic-arch-fallback.h | 72 ++++++++++++++++++++- include/linux/atomic/atomic-instrumented.h | 40 +++++++++++- scripts/atomic/gen-atomic-fallback.sh | 31 +++++---- scripts/atomic/gen-atomic-instrumented.sh | 2 +- 4 files changed, 129 insertions(+), 16 deletions(-) diff --git a/include/linux/atomic/atomic-arch-fallback.h b/include/linux/at= omic/atomic-arch-fallback.h index 6db58d180866..77bc5522e61c 100644 --- a/include/linux/atomic/atomic-arch-fallback.h +++ b/include/linux/atomic/atomic-arch-fallback.h @@ -147,6 +147,76 @@ =20 #endif /* arch_try_cmpxchg_relaxed */ =20 +#ifndef arch_try_cmpxchg64_relaxed +#ifdef arch_try_cmpxchg64 +#define arch_try_cmpxchg64_acquire arch_try_cmpxchg64 +#define arch_try_cmpxchg64_release arch_try_cmpxchg64 +#define arch_try_cmpxchg64_relaxed arch_try_cmpxchg64 +#endif /* arch_try_cmpxchg64 */ + +#ifndef arch_try_cmpxchg64 +#define arch_try_cmpxchg64(_ptr, _oldp, _new) \ +({ \ + typeof(*(_ptr)) *___op =3D (_oldp), ___o =3D *___op, ___r; \ + ___r =3D arch_cmpxchg64((_ptr), ___o, (_new)); \ + if (unlikely(___r !=3D ___o)) \ + *___op =3D ___r; \ + likely(___r =3D=3D ___o); \ +}) +#endif /* arch_try_cmpxchg64 */ + +#ifndef arch_try_cmpxchg64_acquire +#define arch_try_cmpxchg64_acquire(_ptr, _oldp, _new) \ +({ \ + typeof(*(_ptr)) *___op =3D (_oldp), ___o =3D *___op, ___r; \ + ___r =3D arch_cmpxchg64_acquire((_ptr), ___o, (_new)); \ + if (unlikely(___r !=3D ___o)) \ + *___op =3D ___r; \ + likely(___r =3D=3D ___o); \ +}) +#endif /* arch_try_cmpxchg64_acquire */ + +#ifndef arch_try_cmpxchg64_release +#define arch_try_cmpxchg64_release(_ptr, _oldp, _new) \ +({ \ + typeof(*(_ptr)) *___op =3D (_oldp), ___o =3D *___op, ___r; \ + ___r =3D arch_cmpxchg64_release((_ptr), ___o, (_new)); \ + if (unlikely(___r !=3D ___o)) \ + *___op =3D ___r; \ + likely(___r =3D=3D ___o); \ +}) +#endif /* arch_try_cmpxchg64_release */ + +#ifndef arch_try_cmpxchg64_relaxed +#define arch_try_cmpxchg64_relaxed(_ptr, _oldp, _new) \ +({ \ + typeof(*(_ptr)) *___op =3D (_oldp), ___o =3D *___op, ___r; \ + ___r =3D arch_cmpxchg64_relaxed((_ptr), ___o, (_new)); \ + if (unlikely(___r !=3D ___o)) \ + *___op =3D ___r; \ + likely(___r =3D=3D ___o); \ +}) +#endif /* arch_try_cmpxchg64_relaxed */ + +#else /* arch_try_cmpxchg64_relaxed */ + +#ifndef arch_try_cmpxchg64_acquire +#define arch_try_cmpxchg64_acquire(...) \ + __atomic_op_acquire(arch_try_cmpxchg64, __VA_ARGS__) +#endif + +#ifndef arch_try_cmpxchg64_release +#define arch_try_cmpxchg64_release(...) \ + __atomic_op_release(arch_try_cmpxchg64, __VA_ARGS__) +#endif + +#ifndef arch_try_cmpxchg64 +#define arch_try_cmpxchg64(...) \ + __atomic_op_fence(arch_try_cmpxchg64, __VA_ARGS__) +#endif + +#endif /* arch_try_cmpxchg64_relaxed */ + #ifndef arch_atomic_read_acquire static __always_inline int arch_atomic_read_acquire(const atomic_t *v) @@ -2386,4 +2456,4 @@ arch_atomic64_dec_if_positive(atomic64_t *v) #endif =20 #endif /* _LINUX_ATOMIC_FALLBACK_H */ -// 8e2cc06bc0d2c0967d2f8424762bd48555ee40ae +// b5e87bdd5ede61470c29f7a7e4de781af3770f09 diff --git a/include/linux/atomic/atomic-instrumented.h b/include/linux/ato= mic/atomic-instrumented.h index 5d69b143c28e..7a139ec030b0 100644 --- a/include/linux/atomic/atomic-instrumented.h +++ b/include/linux/atomic/atomic-instrumented.h @@ -2006,6 +2006,44 @@ atomic_long_dec_if_positive(atomic_long_t *v) arch_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \ }) =20 +#define try_cmpxchg64(ptr, oldp, ...) \ +({ \ + typeof(ptr) __ai_ptr =3D (ptr); \ + typeof(oldp) __ai_oldp =3D (oldp); \ + kcsan_mb(); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ + arch_try_cmpxchg64(__ai_ptr, __ai_oldp, __VA_ARGS__); \ +}) + +#define try_cmpxchg64_acquire(ptr, oldp, ...) \ +({ \ + typeof(ptr) __ai_ptr =3D (ptr); \ + typeof(oldp) __ai_oldp =3D (oldp); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ + arch_try_cmpxchg64_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \ +}) + +#define try_cmpxchg64_release(ptr, oldp, ...) \ +({ \ + typeof(ptr) __ai_ptr =3D (ptr); \ + typeof(oldp) __ai_oldp =3D (oldp); \ + kcsan_release(); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ + arch_try_cmpxchg64_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \ +}) + +#define try_cmpxchg64_relaxed(ptr, oldp, ...) \ +({ \ + typeof(ptr) __ai_ptr =3D (ptr); \ + typeof(oldp) __ai_oldp =3D (oldp); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ + arch_try_cmpxchg64_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \ +}) + #define cmpxchg_local(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr =3D (ptr); \ @@ -2045,4 +2083,4 @@ atomic_long_dec_if_positive(atomic_long_t *v) }) =20 #endif /* _LINUX_ATOMIC_INSTRUMENTED_H */ -// 87c974b93032afd42143613434d1a7788fa598f9 +// 764f741eb77a7ad565dc8d99ce2837d5542e8aee diff --git a/scripts/atomic/gen-atomic-fallback.sh b/scripts/atomic/gen-ato= mic-fallback.sh index 8e2da71f1d5f..3a07695e3c89 100755 --- a/scripts/atomic/gen-atomic-fallback.sh +++ b/scripts/atomic/gen-atomic-fallback.sh @@ -164,41 +164,44 @@ gen_xchg_fallbacks() =20 gen_try_cmpxchg_fallback() { + local cmpxchg=3D"$1"; shift; local order=3D"$1"; shift; =20 cat < X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 71F0DC433F5 for ; Sun, 15 May 2022 18:42:43 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S238415AbiEOSml (ORCPT ); Sun, 15 May 2022 14:42:41 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:36076 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S238384AbiEOSmc (ORCPT ); Sun, 15 May 2022 14:42:32 -0400 Received: from mail-ed1-x52d.google.com (mail-ed1-x52d.google.com [IPv6:2a00:1450:4864:20::52d]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 2288ADF15 for ; Sun, 15 May 2022 11:42:31 -0700 (PDT) Received: by mail-ed1-x52d.google.com with SMTP id y21so15769983edo.2 for ; Sun, 15 May 2022 11:42:31 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20210112; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=kf1qyJ7WcrGbAyk0+4ZCl1Hcx0Pw2x4ceEltokIn6Ec=; b=V+OE4PXTIHs3eExpcvYxcCgVE0v7IGBJuh3YlE13WPevogBZZfxhg1Igl7Y4+nH4mk 4sHrazj3HNHHianDRyfEucO/fVJIXTwxsXFDBVQ2XKrYV1rClmK6oXsB8C5uwi0Xhz4m GgeCnq0QQ9LneuNTbbgzd3aQCNSW9k/t8Nk1egm9jPQttvqvB7ByZlpU4D0K2VGY0noj cF7hWYP6DsFj8ntI426bXgGM/f3GKJLCrz0t0avo7qD5Gmsj/x41aE1E/cLosqRojKPh EhKuk1SWUt+FHkMmayhJP5rvcKGykKeh7Beoim1Pf2n8AoB7e0p3cd3VwAehCc4WJbul Ax/A== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=kf1qyJ7WcrGbAyk0+4ZCl1Hcx0Pw2x4ceEltokIn6Ec=; b=qmVQkPYfXPS/E9WHnsTeffjtP0YvJ1bjQ51KQxHQ0uAgmaAfkOGbCon7bz/Qai9aSB /NipMYKvVkEeVE1LEAFr8sAr0nGm1pJHzT9seTAtq1sm48khJLJE718pX3JoAIqxlac3 wgFRBGqM5d38IBtZDviVSpqz7mkc7KVRtmkQy+p5GfTmaAwkZcqJpQs3y7i9Ev+pSWys +VrNu6YxcQVnSUFzFWUmjTephgUd7PSsrq5K9IOwMxMWxjrN1yIh+z/twMFOBV1hcNTh VWtdw/Kmkuae25qZ8JN8DthShC781ft38SkIhoDDu4lmHm/veNl5iJlzMazTLAGyUl/Y R4rA== X-Gm-Message-State: AOAM530Nu2wEzZkJY6gKaC/BJquC1UMFT/PRTTruSThpDM6Raxf0S72R /Wh82kcbGWSS3kKFnA8BQnw= X-Google-Smtp-Source: ABdhPJy/R817RklJo9xBiaf3PVrccBMZmQvjIt08J7GAbWoKofO7M0ET9r2V7/mrtYjbDRwn69ypGw== X-Received: by 2002:a05:6402:296:b0:427:e497:29ef with SMTP id l22-20020a056402029600b00427e49729efmr9372588edv.399.1652640149715; Sun, 15 May 2022 11:42:29 -0700 (PDT) Received: from localhost.localdomain (93-103-18-160.static.t-2.net. [93.103.18.160]) by smtp.gmail.com with ESMTPSA id hv7-20020a17090760c700b006f3ef214e62sm2861898ejc.200.2022.05.15.11.42.28 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Sun, 15 May 2022 11:42:29 -0700 (PDT) From: Uros Bizjak To: x86@kernel.org, linux-kernel@vger.kernel.org Cc: Uros Bizjak , Thomas Gleixner , Ingo Molnar , Borislav Petkov , Dave Hansen , "H. Peter Anvin" , Will Deacon , Peter Zijlstra , Boqun Feng , Mark Rutland , "Paul E. McKenney" , Marco Elver Subject: [PATCH v3 2/2] locking/atomic/x86: Introduce arch_try_cmpxchg64 Date: Sun, 15 May 2022 20:42:04 +0200 Message-Id: <20220515184205.103089-3-ubizjak@gmail.com> X-Mailer: git-send-email 2.35.1 In-Reply-To: <20220515184205.103089-1-ubizjak@gmail.com> References: <20220515184205.103089-1-ubizjak@gmail.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" Introduce arch_try_cmpxchg64 for 64-bit and 32-bit targets to improve code using cmpxchg64. On 64-bit targets, the generated assembly improves from: ab: 89 c8 mov %ecx,%eax ad: 48 89 4c 24 60 mov %rcx,0x60(%rsp) b2: 83 e0 fd and $0xfffffffd,%eax b5: 89 54 24 64 mov %edx,0x64(%rsp) b9: 88 44 24 60 mov %al,0x60(%rsp) bd: 48 89 c8 mov %rcx,%rax c0: c6 44 24 62 f2 movb $0xf2,0x62(%rsp) c5: 48 8b 74 24 60 mov 0x60(%rsp),%rsi ca: f0 49 0f b1 34 24 lock cmpxchg %rsi,(%r12) d0: 48 39 c1 cmp %rax,%rcx d3: 75 cf jne a4 to: b3: 89 c2 mov %eax,%edx b5: 48 89 44 24 60 mov %rax,0x60(%rsp) ba: 83 e2 fd and $0xfffffffd,%edx bd: 89 4c 24 64 mov %ecx,0x64(%rsp) c1: 88 54 24 60 mov %dl,0x60(%rsp) c5: c6 44 24 62 f2 movb $0xf2,0x62(%rsp) ca: 48 8b 54 24 60 mov 0x60(%rsp),%rdx cf: f0 48 0f b1 13 lock cmpxchg %rdx,(%rbx) d4: 75 d5 jne ab where a move and a compare after cmpxchg is saved. The improvements for 32-bit targets are even more noticeable, because dual-word compare after cmpxchg8b gets eliminated. Signed-off-by: Uros Bizjak Cc: Thomas Gleixner Cc: Ingo Molnar Cc: Borislav Petkov Cc: Dave Hansen Cc: "H. Peter Anvin" Cc: Will Deacon Cc: Peter Zijlstra Cc: Boqun Feng Cc: Mark Rutland Cc: "Paul E. McKenney" Cc: Marco Elver --- arch/x86/include/asm/cmpxchg_32.h | 21 +++++++++++++++++++++ arch/x86/include/asm/cmpxchg_64.h | 6 ++++++ 2 files changed, 27 insertions(+) diff --git a/arch/x86/include/asm/cmpxchg_32.h b/arch/x86/include/asm/cmpxc= hg_32.h index 0a7fe0321613..215f5a65790f 100644 --- a/arch/x86/include/asm/cmpxchg_32.h +++ b/arch/x86/include/asm/cmpxchg_32.h @@ -42,6 +42,9 @@ static inline void set_64bit(volatile u64 *ptr, u64 value) #define arch_cmpxchg64_local(ptr, o, n) \ ((__typeof__(*(ptr)))__cmpxchg64_local((ptr), (unsigned long long)(o), \ (unsigned long long)(n))) +#define arch_try_cmpxchg64(ptr, po, n) \ + __try_cmpxchg64((ptr), (unsigned long long *)(po), \ + (unsigned long long)(n)) #endif =20 static inline u64 __cmpxchg64(volatile u64 *ptr, u64 old, u64 new) @@ -70,6 +73,24 @@ static inline u64 __cmpxchg64_local(volatile u64 *ptr, u= 64 old, u64 new) return prev; } =20 +static inline bool __try_cmpxchg64(volatile u64 *ptr, u64 *pold, u64 new) +{ + bool success; + u64 old =3D *pold; + asm volatile(LOCK_PREFIX "cmpxchg8b %[ptr]" + CC_SET(z) + : CC_OUT(z) (success), + [ptr] "+m" (*ptr), + "+A" (old) + : "b" ((u32)new), + "c" ((u32)(new >> 32)) + : "memory"); + + if (unlikely(!success)) + *pold =3D old; + return success; +} + #ifndef CONFIG_X86_CMPXCHG64 /* * Building a kernel capable running on 80386 and 80486. It may be necessa= ry diff --git a/arch/x86/include/asm/cmpxchg_64.h b/arch/x86/include/asm/cmpxc= hg_64.h index 072e5459fe2f..250187ac8248 100644 --- a/arch/x86/include/asm/cmpxchg_64.h +++ b/arch/x86/include/asm/cmpxchg_64.h @@ -19,6 +19,12 @@ static inline void set_64bit(volatile u64 *ptr, u64 val) arch_cmpxchg_local((ptr), (o), (n)); \ }) =20 +#define arch_try_cmpxchg64(ptr, po, n) \ +({ \ + BUILD_BUG_ON(sizeof(*(ptr)) !=3D 8); \ + arch_try_cmpxchg((ptr), (po), (n)); \ +}) + #define system_has_cmpxchg_double() boot_cpu_has(X86_FEATURE_CX16) =20 #endif /* _ASM_X86_CMPXCHG_64_H */ --=20 2.35.1