From nobody Sat Feb 7 13:41:31 2026 Received: from mail-ej1-f51.google.com (mail-ej1-f51.google.com [209.85.218.51]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id B41061272AB for ; Tue, 9 Apr 2024 10:05:09 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=209.85.218.51 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1712657111; cv=none; b=jyYBLN8Hi6EhXmAYsMZG/yFVimmFdsJp4fZylabLBRdlwWbWGQTUWWd7Z3wm289maVWlrRgaRm7zsDsyuC2xJ3Gm2wzG/te8EQQXCQnlxjcFvmaRqMOZQkbp+qNZ7bDpQwwfmtTmORVgS9GV/D+3YdWWkWctJIw9mn8nEdNL8ek= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1712657111; c=relaxed/simple; bh=Otkz9VxKv1AmngInYT0fy2HPyWY/dnaWtJxwZzJ8HVo=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=MphixTJQWnMiTx6BISTH5m95ZnOR5Gk0935L/39nQfItMwRBm1ddwsf00Sp+9zN9LcEqTMorjDcnyu3DycPOEzgbcBf68r9+8OwMVjR+dTpm0uzvsyGwIHsg0OuVvpTnloUJEkdkBINJEIK33omNtrfcGtGUGp7j7mwE25/i974= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=gmail.com; spf=pass smtp.mailfrom=gmail.com; dkim=pass (2048-bit key) header.d=gmail.com header.i=@gmail.com header.b=m+8CqoNL; arc=none smtp.client-ip=209.85.218.51 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=gmail.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=gmail.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=gmail.com header.i=@gmail.com header.b="m+8CqoNL" Received: by mail-ej1-f51.google.com with SMTP id a640c23a62f3a-a450bedffdfso758466266b.3 for ; Tue, 09 Apr 2024 03:05:09 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20230601; t=1712657108; x=1713261908; darn=vger.kernel.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=oAGoeR5uWamfDBsJ3kwTlmEZZs8wXAmwviK0FpBjRBA=; b=m+8CqoNLgsHh4knfz4w6azklxs4BBd/8d7xKDdcgEQWBfjgCHOuBK5kLz/21nlVCqo Qi38/qrv7KCMNzHGiQTjwbuTCD7kksfg/+V4P+RvJhm7P2yayTldQNvmMWILw7XNdeH1 srZFsXM9E51Pfj7N6pJHq5tMZIRi6RSts91jGpibwOLxm6O33uAIqFUBYvyqFmyHAY/z 8xXWNKbEaYHVkVRI/BYxxbzmmro50k7PBiBFObtSavMx2tHRN/K/kA/+xBu77ixR95Qf wgy3hZ5hpJjn2kI0a6TkHggepjpcEc9M/jukdzHz4ZcwW2sF8bGo9Yt1w06lSu8fDBaO msew== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1712657108; x=1713261908; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=oAGoeR5uWamfDBsJ3kwTlmEZZs8wXAmwviK0FpBjRBA=; b=B7FdOJVIhP7efAqCFvCI66a5tjMLw7ck89WqGB1ZsBlBi93TjKqKnESbUiqh7pB37U QVHSmcgYUwbV/GzPgDHrLoyuSzXcO3UJQIicMyuxDLMRngx9Ev1M9SavdpMVRgiUtbvg UkDWO9rymyoT0SZ85YrBregMlwjTT4SjiO1BN2LC/BvUMf6YnxoTQWEakgsxrAu4rD/X BM4naCSR4ges8JUu445P95T2Oh5mQMlvH18J6ThLWGxjyn3+rMm7EmhBiiFP9E/+gXFa eT+lisWdHEa8eNqj1RkV97bgKE6cbkm7qvOWS0hll2m9rgw7iVKF6SojLHmwFh5X6nQa 5b+A== X-Forwarded-Encrypted: i=1; AJvYcCUKT56Sl/9+qDtPA+JbB3xSu+R24KI50zFayKP5mqlG3JQnutEaidhDMzZR+r4BnfPBvpMRjDQsoLIqSofi6jXCs8ZZoiquJlKi9jOe X-Gm-Message-State: AOJu0Ywutw4h9kLhrwoT0ENfaRHsQhCeIVgBM0Zi6ofS/Gd+urN8urto t3aqGKi2B23D045bt5P/vJCcfh3h4CaTOVVu1K43Dopjns1/fOhv X-Google-Smtp-Source: AGHT+IG8Nl6vjVWOGS3WZO62fvd41W0VNuc+nkbgDwo1+KvpOPZZhfSRDaKJaeHTxttw0I9PiPrZHQ== X-Received: by 2002:a17:907:80a:b0:a51:f63d:84d0 with SMTP id wv10-20020a170907080a00b00a51f63d84d0mr1086765ejb.64.1712657107776; Tue, 09 Apr 2024 03:05:07 -0700 (PDT) Received: from fedora.iskraemeco.si ([193.77.86.250]) by smtp.gmail.com with ESMTPSA id qs1-20020a170906458100b00a4e6626ae21sm5496681ejc.0.2024.04.09.03.05.07 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Tue, 09 Apr 2024 03:05:07 -0700 (PDT) From: Uros Bizjak To: x86@kernel.org, linux-kernel@vger.kernel.org Cc: Uros Bizjak , Thomas Gleixner , Ingo Molnar , Borislav Petkov , Dave Hansen , "H. Peter Anvin" , Peter Zijlstra Subject: [PATCH 1/6] locking/atomic/x86: Introduce arch_atomic64_try_cmpxchg to x86_32 Date: Tue, 9 Apr 2024 12:03:52 +0200 Message-ID: <20240409100503.274629-2-ubizjak@gmail.com> X-Mailer: git-send-email 2.44.0 In-Reply-To: <20240409100503.274629-1-ubizjak@gmail.com> References: <20240409100503.274629-1-ubizjak@gmail.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Introduce arch_atomic64_try_cmpxchg for 32-bit targets to use optimized target specific implementation instead of a generic one. This implementation eliminates dual-word compare after cmpxchg8b instruction and improves generated asm code from: 2273: f0 0f c7 0f lock cmpxchg8b (%edi) 2277: 8b 74 24 2c mov 0x2c(%esp),%esi 227b: 89 d3 mov %edx,%ebx 227d: 89 c2 mov %eax,%edx 227f: 89 5c 24 10 mov %ebx,0x10(%esp) 2283: 8b 7c 24 30 mov 0x30(%esp),%edi 2287: 89 44 24 1c mov %eax,0x1c(%esp) 228b: 31 f2 xor %esi,%edx 228d: 89 d0 mov %edx,%eax 228f: 89 da mov %ebx,%edx 2291: 31 fa xor %edi,%edx 2293: 09 d0 or %edx,%eax 2295: 0f 85 a5 00 00 00 jne 2340 <...> to: 2270: f0 0f c7 0f lock cmpxchg8b (%edi) 2274: 0f 85 a6 00 00 00 jne 2320 <...> Signed-off-by: Uros Bizjak Cc: Thomas Gleixner Cc: Ingo Molnar Cc: Borislav Petkov Cc: Dave Hansen Cc: "H. Peter Anvin" Cc: Peter Zijlstra --- arch/x86/include/asm/atomic64_32.h | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/arch/x86/include/asm/atomic64_32.h b/arch/x86/include/asm/atom= ic64_32.h index d510405e4e1d..11e817dab44a 100644 --- a/arch/x86/include/asm/atomic64_32.h +++ b/arch/x86/include/asm/atomic64_32.h @@ -61,12 +61,18 @@ ATOMIC64_DECL(add_unless); #undef __ATOMIC64_DECL #undef ATOMIC64_EXPORT =20 -static __always_inline s64 arch_atomic64_cmpxchg(atomic64_t *v, s64 o, s64= n) +static __always_inline s64 arch_atomic64_cmpxchg(atomic64_t *v, s64 old, s= 64 new) { - return arch_cmpxchg64(&v->counter, o, n); + return arch_cmpxchg64(&v->counter, old, new); } #define arch_atomic64_cmpxchg arch_atomic64_cmpxchg =20 +static __always_inline bool arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *= old, s64 new) +{ + return arch_try_cmpxchg64(&v->counter, old, new); +} +#define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg + static __always_inline s64 arch_atomic64_xchg(atomic64_t *v, s64 n) { s64 o; --=20 2.44.0 From nobody Sat Feb 7 13:41:31 2026 Received: from mail-ed1-f47.google.com (mail-ed1-f47.google.com [209.85.208.47]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 9E1118173B for ; Tue, 9 Apr 2024 10:05:11 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=209.85.208.47 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1712657113; cv=none; b=tOilzj3V/WMNTckbPqqyCuO45BBSn9O2jquoVZJpngguJCg3LyJNWOr3X+MbTr5ZQa38LZ8sOrsAONG5cLix8pd+GPy2qHeI7QSDvkolticLIOJ969y9EE9lu+CRJ0jdezg8CbOAceCxy/HBLcQdp/wofs7E12jKskoI+Gd3Z4M= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1712657113; c=relaxed/simple; bh=/Nb0syoqXrLtiMCAaNUxknKwFQ8hydCC1BIHa4Sg4cw=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=gf9kNY2yrXSpZpcN4s8dQ59FveX4qqbAkDT3r6TMeV4t3IoL3kUXu7ZEAOjc6URBTVJU9bh87zdSrmIL3vCcDQCtLcZArTL13C9+5EIUqg+J072ftDPRSV9n8uelnTyWZkJyEa6rOhIkIg531fb/bXebLjDe4DNU7QQUD31UmCM= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=gmail.com; spf=pass smtp.mailfrom=gmail.com; dkim=pass (2048-bit key) header.d=gmail.com header.i=@gmail.com header.b=HLSCgrRX; arc=none smtp.client-ip=209.85.208.47 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=gmail.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=gmail.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=gmail.com header.i=@gmail.com header.b="HLSCgrRX" Received: by mail-ed1-f47.google.com with SMTP id 4fb4d7f45d1cf-56e6282dd72so2287572a12.3 for ; Tue, 09 Apr 2024 03:05:11 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20230601; t=1712657110; x=1713261910; darn=vger.kernel.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=4JAl2q7pGidcLS0wjbtFM6lZo9cgP3O7vC63sSwKbiE=; b=HLSCgrRXtL1hIEWKXr21MfMw29kTtEHXLvuwDhTfXyr+wwPX/q31lLV/2PJ+dAQX5F Srfd/evLdmEMyZDBCbOS8vXlkODT2ozpjJSb77uNGUfJfxH93xAX4hsOOipFphyAWRHV WcWpLQ0BNPPo12EXNLpDYyyri1XCo9e0yhjuslue7GQHfWZA+5UB0jUh6gsWmc8Mqkf1 I4I0xD0dLevbOeffFQBEV93C9pTB0vqjJtyrf3tTkLro/stdIXrtCMwblmJ57V7pA83l 2HnUlQyjY98XeYebwvO2BjFeGqPsA/PZV3+2SMydx78Hyj+k+Hb3G51DWb811+Od4puY 45yA== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1712657110; x=1713261910; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=4JAl2q7pGidcLS0wjbtFM6lZo9cgP3O7vC63sSwKbiE=; b=Zz5a6qZ+SOfUnr9nHuNvg0EpWarOn2H4QiXhfQM9V+x5IKadkBe8o40RpxWJsIN+Gc cK5JtV7Zi6dtlP1Cv4pIHXEeON7PNRXs/TdDJ470kCMnjFOEjb1C0QpxyeahyScxQCln aWOwXjaHzQl12tA1kCX8FrtbzPpzPOF7gyUV1GbWBo6MBqeaOmb2hzdp9kR8T1nfeGuv UPrXS7xoFg4ZSLfAjGiWWadHWAlKpLZzPdNrrXzm+/Zca4ZLm2n/WuFhS3HgOaDBOMq8 Kgq9nZVzORvnH92QwppUlbZOzA/c9YtK7hAp5bd+o6mC7Mgclr4dcj3aZmf0pkgL04HG b0sA== X-Forwarded-Encrypted: i=1; AJvYcCXFbIxkTJnQr8UsZCbeOOkETYfJjb0EDkNd4gNNQyy458KTyp6BFbmkFOjyTjbo0D6L6QdbdWtFcw3ejzPVUHft7OJg2CKqbkLuB/fg X-Gm-Message-State: AOJu0YzLY8zWfizfN+Yjtkwl93Eb7VRWB3q6LMSH28k6JwWHmLSPKInP e3DyE7FLIcC/eKD7mkq9AkC5zo18+HJ2DVqKceDeVo16yEECCfXl X-Google-Smtp-Source: AGHT+IHKyj6j3YIALqLzvD/zqxAze4Rcqk7nzqdEyINEKyYh9fT9ZqY6Cq+bHIiHeEm4EtqxwV3QGw== X-Received: by 2002:a17:906:558d:b0:a51:a09c:16a5 with SMTP id y13-20020a170906558d00b00a51a09c16a5mr6174552ejp.23.1712657109732; Tue, 09 Apr 2024 03:05:09 -0700 (PDT) Received: from fedora.iskraemeco.si ([193.77.86.250]) by smtp.gmail.com with ESMTPSA id qs1-20020a170906458100b00a4e6626ae21sm5496681ejc.0.2024.04.09.03.05.07 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Tue, 09 Apr 2024 03:05:08 -0700 (PDT) From: Uros Bizjak To: x86@kernel.org, linux-kernel@vger.kernel.org Cc: Uros Bizjak , Thomas Gleixner , Ingo Molnar , Borislav Petkov , Dave Hansen , "H. Peter Anvin" , Peter Zijlstra Subject: [PATCH 2/6] locking/atomic/x86: Rewrite x86_32 arch_atomic64_{,fetch}_{and,or,xor}() functions Date: Tue, 9 Apr 2024 12:03:53 +0200 Message-ID: <20240409100503.274629-3-ubizjak@gmail.com> X-Mailer: git-send-email 2.44.0 In-Reply-To: <20240409100503.274629-1-ubizjak@gmail.com> References: <20240409100503.274629-1-ubizjak@gmail.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Rewrite x86_32 arch_atomic64_{,fetch}_{and,or,xor}() functions to use arch_atomic64_try_cmpxchg. This implementation avoids one extra trip through the cmpxchg loop. The value preload before the cmpxchg loop does not need to be atomic, but should use READ_ONCE to prevent compiler from merging, refetching or reordering the read. The generated code improves from: 1917d5: 31 c9 xor %ecx,%ecx 1917d7: 31 db xor %ebx,%ebx 1917d9: 89 4c 24 3c mov %ecx,0x3c(%esp) 1917dd: 8b 74 24 24 mov 0x24(%esp),%esi 1917e1: 89 c8 mov %ecx,%eax 1917e3: 89 5c 24 34 mov %ebx,0x34(%esp) 1917e7: 8b 7c 24 28 mov 0x28(%esp),%edi 1917eb: 21 ce and %ecx,%esi 1917ed: 89 74 24 4c mov %esi,0x4c(%esp) 1917f1: 21 df and %ebx,%edi 1917f3: 89 de mov %ebx,%esi 1917f5: 89 7c 24 50 mov %edi,0x50(%esp) 1917f9: 8b 54 24 4c mov 0x4c(%esp),%edx 1917fd: 8b 7c 24 2c mov 0x2c(%esp),%edi 191801: 8b 4c 24 50 mov 0x50(%esp),%ecx 191805: 89 d3 mov %edx,%ebx 191807: 89 f2 mov %esi,%edx 191809: f0 0f c7 0f lock cmpxchg8b (%edi) 19180d: 89 c1 mov %eax,%ecx 19180f: 8b 74 24 34 mov 0x34(%esp),%esi 191813: 89 d3 mov %edx,%ebx 191815: 89 44 24 4c mov %eax,0x4c(%esp) 191819: 8b 44 24 3c mov 0x3c(%esp),%eax 19181d: 89 df mov %ebx,%edi 19181f: 89 54 24 44 mov %edx,0x44(%esp) 191823: 89 ca mov %ecx,%edx 191825: 31 de xor %ebx,%esi 191827: 31 c8 xor %ecx,%eax 191829: 09 f0 or %esi,%eax 19182b: 75 ac jne 1917d9 <...> to: 1912ba: 8b 06 mov (%esi),%eax 1912bc: 8b 56 04 mov 0x4(%esi),%edx 1912bf: 89 44 24 3c mov %eax,0x3c(%esp) 1912c3: 89 c1 mov %eax,%ecx 1912c5: 23 4c 24 34 and 0x34(%esp),%ecx 1912c9: 89 d3 mov %edx,%ebx 1912cb: 23 5c 24 38 and 0x38(%esp),%ebx 1912cf: 89 54 24 40 mov %edx,0x40(%esp) 1912d3: 89 4c 24 2c mov %ecx,0x2c(%esp) 1912d7: 89 5c 24 30 mov %ebx,0x30(%esp) 1912db: 8b 5c 24 2c mov 0x2c(%esp),%ebx 1912df: 8b 4c 24 30 mov 0x30(%esp),%ecx 1912e3: f0 0f c7 0e lock cmpxchg8b (%esi) 1912e7: 0f 85 f3 02 00 00 jne 1915e0 <...> Signed-off-by: Uros Bizjak Cc: Thomas Gleixner Cc: Ingo Molnar Cc: Borislav Petkov Cc: Dave Hansen Cc: "H. Peter Anvin" Cc: Peter Zijlstra --- arch/x86/include/asm/atomic64_32.h | 44 ++++++++++++------------------ 1 file changed, 18 insertions(+), 26 deletions(-) diff --git a/arch/x86/include/asm/atomic64_32.h b/arch/x86/include/asm/atom= ic64_32.h index 11e817dab44a..84affd7a5d1c 100644 --- a/arch/x86/include/asm/atomic64_32.h +++ b/arch/x86/include/asm/atomic64_32.h @@ -201,69 +201,61 @@ static __always_inline s64 arch_atomic64_dec_if_posit= ive(atomic64_t *v) =20 static __always_inline void arch_atomic64_and(s64 i, atomic64_t *v) { - s64 old, c =3D 0; + s64 val =3D __READ_ONCE(v->counter); =20 - while ((old =3D arch_atomic64_cmpxchg(v, c, c & i)) !=3D c) - c =3D old; + do { } while (!arch_atomic64_try_cmpxchg(v, &val, val & i)); } =20 static __always_inline s64 arch_atomic64_fetch_and(s64 i, atomic64_t *v) { - s64 old, c =3D 0; + s64 val =3D __READ_ONCE(v->counter); =20 - while ((old =3D arch_atomic64_cmpxchg(v, c, c & i)) !=3D c) - c =3D old; + do { } while (!arch_atomic64_try_cmpxchg(v, &val, val & i)); =20 - return old; + return val; } #define arch_atomic64_fetch_and arch_atomic64_fetch_and =20 static __always_inline void arch_atomic64_or(s64 i, atomic64_t *v) { - s64 old, c =3D 0; + s64 val =3D __READ_ONCE(v->counter); =20 - while ((old =3D arch_atomic64_cmpxchg(v, c, c | i)) !=3D c) - c =3D old; + do { } while (!arch_atomic64_try_cmpxchg(v, &val, val | i)); } =20 static __always_inline s64 arch_atomic64_fetch_or(s64 i, atomic64_t *v) { - s64 old, c =3D 0; + s64 val =3D __READ_ONCE(v->counter); =20 - while ((old =3D arch_atomic64_cmpxchg(v, c, c | i)) !=3D c) - c =3D old; + do { } while (!arch_atomic64_try_cmpxchg(v, &val, val | i)); =20 - return old; + return val; } #define arch_atomic64_fetch_or arch_atomic64_fetch_or =20 static __always_inline void arch_atomic64_xor(s64 i, atomic64_t *v) { - s64 old, c =3D 0; + s64 val =3D __READ_ONCE(v->counter); =20 - while ((old =3D arch_atomic64_cmpxchg(v, c, c ^ i)) !=3D c) - c =3D old; + do { } while (!arch_atomic64_try_cmpxchg(v, &val, val ^ i)); } =20 static __always_inline s64 arch_atomic64_fetch_xor(s64 i, atomic64_t *v) { - s64 old, c =3D 0; + s64 val =3D __READ_ONCE(v->counter); =20 - while ((old =3D arch_atomic64_cmpxchg(v, c, c ^ i)) !=3D c) - c =3D old; + do { } while (!arch_atomic64_try_cmpxchg(v, &val, val ^ i)); =20 - return old; + return val; } #define arch_atomic64_fetch_xor arch_atomic64_fetch_xor =20 static __always_inline s64 arch_atomic64_fetch_add(s64 i, atomic64_t *v) { - s64 old, c =3D 0; + s64 val =3D __READ_ONCE(v->counter); =20 - while ((old =3D arch_atomic64_cmpxchg(v, c, c + i)) !=3D c) - c =3D old; - - return old; + do { } while (!arch_atomic64_try_cmpxchg(v, &val, val + i)); + return val; } #define arch_atomic64_fetch_add arch_atomic64_fetch_add =20 --=20 2.44.0 From nobody Sat Feb 7 13:41:31 2026 Received: from mail-lj1-f170.google.com (mail-lj1-f170.google.com [209.85.208.170]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id A293482865 for ; Tue, 9 Apr 2024 10:05:13 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=209.85.208.170 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1712657115; cv=none; b=ew6C4/NiOmra8mst3xeDWPc02DXMAKND9mXuZCs2+bUzxsWgryTZDU4/ocXg1euxEs+1F7aCapl1qHmJ805++9/enQ2M1jasbnqOsdPicmdlZqu267q8/Xu+m7AcndhzTREdszmnFSrH+ktJ18Htp7HubPiUGn4k2JD2zo9g0Mw= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1712657115; c=relaxed/simple; bh=ugA08uXqoEwEn8GuKopyp1EqHtpqnUKd/PxDao6nDlI=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=jj2WhN7Esqah4F8kNR2xNCtzos1xw3IQDKEXrLlqXY+ZxVU/EsSthNWZtSWyV+afepLekc9JJSF2B9zw0aEHCOaT8Y2sa0nce7G9BFYSjWeQnTNHHxwq/4I9pPq/hWxXvRO8ejYdcZiSV4eNh47xQj24SFpOfRHTmAr3LZnjtT0= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=gmail.com; spf=pass smtp.mailfrom=gmail.com; dkim=pass (2048-bit key) header.d=gmail.com header.i=@gmail.com header.b=QQtTZ1jY; arc=none smtp.client-ip=209.85.208.170 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=gmail.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=gmail.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=gmail.com header.i=@gmail.com header.b="QQtTZ1jY" Received: by mail-lj1-f170.google.com with SMTP id 38308e7fff4ca-2d82713f473so92622761fa.3 for ; Tue, 09 Apr 2024 03:05:13 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20230601; t=1712657112; x=1713261912; darn=vger.kernel.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=hew6IrPXucrn4SI3v5T6ztETy+bvxdIyXI1Ro/bitzY=; b=QQtTZ1jYd3Dx0XGXbriuS03LiCZi47ythiq4Sda7uHQQ5nrCnXNqyn5vlL2xKn/Hx7 sm7E4qXj68SyCw12ote1EO9wt5Y0pQkJqbmx1wjSXKO8ChpQ38ovgMa4wL0FmBJ1gtJN HaXh04uiq31kCmPwRAV4UQZAd5BvEE03PNEzCgUtznmHedVtbypZPSdtRGElLMa+CK1r uX4n88NnUg3iIRUigN10ZwpLGCZYU+5rELi7WupWR3Z7bglnojlUkaLSv8BWgCzOzCdH sBYyNrlVnG+Aioy7icIYD0HHPeGJPshqsGzB6/PCcPw2gwNseJnNa7VRejYmLc0s5pux OIpg== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1712657112; x=1713261912; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=hew6IrPXucrn4SI3v5T6ztETy+bvxdIyXI1Ro/bitzY=; b=bTOzL/wGfZaBUQBv722+GhpY8IIaPa8Shws51Gvw4IpugKRqhC919T5V0wsLichnHK UEhjsEb+1QZyDtUUwkiFISgqMDCs0XizxtBkreve5EDM+sc3ThXWYPUpePVZuPpgtV9d XYsgYCQsDEnneP3TAVmZP+Mb5jLYRJygMxzqgT2Fe29zg8AFodg8bEzBx8kbkEZsRsBf on7krtVr3/A7POEiQBl5rRfJDP31XNycLWwLKaAMUkl3Yf+M8JodZUqXw0CX5wQAYcUj REiyDkczN+NQxg3KKYX1t56unjZTV+a82j10fK31L2PqDVA7uWyz8MpJ30Ux74q5ImBW Q6uw== X-Forwarded-Encrypted: i=1; AJvYcCWJRXKYn95SzoVZmSiaA7Rdga2jvmlhg4xRYEkPz0ZP7+5oDHrBveRVKz0TtTvxn22UgjyQ2wD8QBBROes//zTMTS20h1LRsb9cxdqh X-Gm-Message-State: AOJu0YyTntkoK2KtXtukTw+2fit0H5e9FFsxABdByoT5eZGusvJmGrX7 2eQ28TvUlkbNSNr0mRK2csNtWsuzZF60E+6+DQjSFi4SoNlShNVn X-Google-Smtp-Source: AGHT+IF/CiW5guZCr3itYqmfESeOATzJU7bpKqf1oFcMCzYNzUgGvRPQ6FqKK7h0BUTghiHIvY5nKg== X-Received: by 2002:a2e:2e11:0:b0:2d4:5370:5e8a with SMTP id u17-20020a2e2e11000000b002d453705e8amr9333404lju.22.1712657111401; Tue, 09 Apr 2024 03:05:11 -0700 (PDT) Received: from fedora.iskraemeco.si ([193.77.86.250]) by smtp.gmail.com with ESMTPSA id qs1-20020a170906458100b00a4e6626ae21sm5496681ejc.0.2024.04.09.03.05.09 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Tue, 09 Apr 2024 03:05:10 -0700 (PDT) From: Uros Bizjak To: x86@kernel.org, linux-kernel@vger.kernel.org Cc: Uros Bizjak , Thomas Gleixner , Ingo Molnar , Borislav Petkov , Dave Hansen , "H. Peter Anvin" , Peter Zijlstra Subject: [PATCH 3/6] locking/atomic/x86: Use READ_ONCE before atomic{,64}_try_cmpxchg loops Date: Tue, 9 Apr 2024 12:03:54 +0200 Message-ID: <20240409100503.274629-4-ubizjak@gmail.com> X-Mailer: git-send-email 2.44.0 In-Reply-To: <20240409100503.274629-1-ubizjak@gmail.com> References: <20240409100503.274629-1-ubizjak@gmail.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" The value preload before the cmpxchg loop does not need to be atomic, but should use READ_ONCE to prevent compiler from merging, refetching or reordering the read. This patch unifies arch_atomic{,64}_{,fetch}_{and,or,xor}() macros between x86_32 and x86_64 targets. No functional changes intended. Signed-off-by: Uros Bizjak Cc: Thomas Gleixner Cc: Ingo Molnar Cc: Borislav Petkov Cc: Dave Hansen Cc: "H. Peter Anvin" Cc: Peter Zijlstra --- arch/x86/include/asm/atomic.h | 8 ++++---- arch/x86/include/asm/atomic64_64.h | 20 ++++++++++---------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/arch/x86/include/asm/atomic.h b/arch/x86/include/asm/atomic.h index 55a55ec04350..b166da21ee98 100644 --- a/arch/x86/include/asm/atomic.h +++ b/arch/x86/include/asm/atomic.h @@ -20,7 +20,7 @@ static __always_inline int arch_atomic_read(const atomic_= t *v) * Note for KASAN: we deliberately don't use READ_ONCE_NOCHECK() here, * it's non-inlined function that increases binary size and stack usage. */ - return __READ_ONCE((v)->counter); + return __READ_ONCE(v->counter); } =20 static __always_inline void arch_atomic_set(atomic_t *v, int i) @@ -132,7 +132,7 @@ static __always_inline void arch_atomic_and(int i, atom= ic_t *v) =20 static __always_inline int arch_atomic_fetch_and(int i, atomic_t *v) { - int val =3D arch_atomic_read(v); + int val =3D __READ_ONCE(v->counter); =20 do { } while (!arch_atomic_try_cmpxchg(v, &val, val & i)); =20 @@ -150,7 +150,7 @@ static __always_inline void arch_atomic_or(int i, atomi= c_t *v) =20 static __always_inline int arch_atomic_fetch_or(int i, atomic_t *v) { - int val =3D arch_atomic_read(v); + int val =3D __READ_ONCE(v->counter); =20 do { } while (!arch_atomic_try_cmpxchg(v, &val, val | i)); =20 @@ -168,7 +168,7 @@ static __always_inline void arch_atomic_xor(int i, atom= ic_t *v) =20 static __always_inline int arch_atomic_fetch_xor(int i, atomic_t *v) { - int val =3D arch_atomic_read(v); + int val =3D __READ_ONCE(v->counter); =20 do { } while (!arch_atomic_try_cmpxchg(v, &val, val ^ i)); =20 diff --git a/arch/x86/include/asm/atomic64_64.h b/arch/x86/include/asm/atom= ic64_64.h index 3165c0feedf7..e7b12a48fecb 100644 --- a/arch/x86/include/asm/atomic64_64.h +++ b/arch/x86/include/asm/atomic64_64.h @@ -12,7 +12,7 @@ =20 static __always_inline s64 arch_atomic64_read(const atomic64_t *v) { - return __READ_ONCE((v)->counter); + return __READ_ONCE(v->counter); } =20 static __always_inline void arch_atomic64_set(atomic64_t *v, s64 i) @@ -126,10 +126,10 @@ static __always_inline void arch_atomic64_and(s64 i, = atomic64_t *v) =20 static __always_inline s64 arch_atomic64_fetch_and(s64 i, atomic64_t *v) { - s64 val =3D arch_atomic64_read(v); + s64 val =3D __READ_ONCE(v->counter); + + do { } while (!arch_atomic64_try_cmpxchg(v, &val, val & i)); =20 - do { - } while (!arch_atomic64_try_cmpxchg(v, &val, val & i)); return val; } #define arch_atomic64_fetch_and arch_atomic64_fetch_and @@ -144,10 +144,10 @@ static __always_inline void arch_atomic64_or(s64 i, a= tomic64_t *v) =20 static __always_inline s64 arch_atomic64_fetch_or(s64 i, atomic64_t *v) { - s64 val =3D arch_atomic64_read(v); + s64 val =3D __READ_ONCE(v->counter); + + do { } while (!arch_atomic64_try_cmpxchg(v, &val, val | i)); =20 - do { - } while (!arch_atomic64_try_cmpxchg(v, &val, val | i)); return val; } #define arch_atomic64_fetch_or arch_atomic64_fetch_or @@ -162,10 +162,10 @@ static __always_inline void arch_atomic64_xor(s64 i, = atomic64_t *v) =20 static __always_inline s64 arch_atomic64_fetch_xor(s64 i, atomic64_t *v) { - s64 val =3D arch_atomic64_read(v); + s64 val =3D __READ_ONCE(v->counter); + + do { } while (!arch_atomic64_try_cmpxchg(v, &val, val ^ i)); =20 - do { - } while (!arch_atomic64_try_cmpxchg(v, &val, val ^ i)); return val; } #define arch_atomic64_fetch_xor arch_atomic64_fetch_xor --=20 2.44.0 From nobody Sat Feb 7 13:41:31 2026 Received: from mail-ed1-f51.google.com (mail-ed1-f51.google.com [209.85.208.51]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 34B3F86AC8 for ; Tue, 9 Apr 2024 10:05:14 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=209.85.208.51 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1712657117; cv=none; b=C6RdZxvdFY4MdNIvf3qe6disakToHY1NdchInsO87vRjfpm/gAmRkXWVo3xdUpH3to9enyBLKmoORn+eR0v4lAM26fwxTLs2wIp6grCEH9UpLht2BmAHMBWI9ZtBr+/F0j5nGCVaxNWhpiJtw7kpaNp0FZLShj46y+m18ZX47bQ= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1712657117; c=relaxed/simple; bh=+yDCeMsqfwL99gW6SvVlG/j6+165sentyxdZBx5xZYg=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=MRSr7YiiH1uZON4MfCjt7l4AJ1eRD7VHuPAQCDJIRWMEacc29ghuQiRkN0tpqOqxYGr7/kb1vqHDprRvPCVvJgi7xsNcvRcOKhVJS5BOWSC013/6pOtmJSf6HDJ8uW61bYtQVbFv4McrrcJrtlPVqxwMvtVIdWCHZvdZ3OtQpVQ= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=gmail.com; spf=pass smtp.mailfrom=gmail.com; dkim=pass (2048-bit key) header.d=gmail.com header.i=@gmail.com header.b=cmAaHBaR; arc=none smtp.client-ip=209.85.208.51 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=gmail.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=gmail.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=gmail.com header.i=@gmail.com header.b="cmAaHBaR" Received: by mail-ed1-f51.google.com with SMTP id 4fb4d7f45d1cf-56845954ffeso7293865a12.2 for ; Tue, 09 Apr 2024 03:05:14 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20230601; t=1712657113; x=1713261913; darn=vger.kernel.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=yDyjDITii34VI9lPR5N4ADo94eAWkADYksZtESuAH2M=; b=cmAaHBaRBau+V4wS41q9e1Me4Z1ncnlzOwLEr6gtoAToynmhJDJBm1LZTmQ+G4N8Q4 FwhiZyTnL3PHq7kVaaQMv08wtI8hB7/4aCkF91qY19rleWgZntwVITlWNUQVA2ZR5zwU 1iowEKO90dlFLYwVza1h1fCTxX5GRyGdCixLvHwCZVR4LHoOp3QqG52SXfVG8+E57b9W HELwpgXZwGJljk+Toe0tksKoIzf/VRiRWoxxVxuZ0jOoJc7qq5PpvN14SPyv4hmZ7fwL FNSw2gXOi/jsBZ88nTQgkVa7WPNbrsJctZHE0xjkEhz6wUBZ0qv0mSg9oHk0I3iBoDoP tofQ== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1712657113; x=1713261913; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=yDyjDITii34VI9lPR5N4ADo94eAWkADYksZtESuAH2M=; b=aHX37WNmyjWyq/kJjBvonwCk8R2MuRslRB5g2kmXgKg52la/qmzeqN6F56U0UeXOfK X5pMT8AITpIlixNI1SzzDLPur9pdTDWeiCoss+e6PIrQjWBGATHXWMMYYbJeUYnS1tQK l8jO3BUNOR8RpFCnj6w/puqaYbRGrYEAyCLuVgkwEJ9zks92FUGKe/YCquImnp5K+3AZ rNYB6CHypMEyMq5ceyyRTfXy2sPzzEKIzekR1QuIBJENY9UE1fQKtVEyWNXkRmSjHMKj dX6hcf9XhCNsnZ5JR3zW7DgoS4KjCvAA/oLfaCIlKVjefIlK0nSPRdCkIcuuD0YEx6/o 4LIg== X-Forwarded-Encrypted: i=1; AJvYcCWsGCd71fVIgvdCXrtlBg4QHrTayutX3EQJRw8A/5LYDI7mclN+5ZWcBtyhDScTdu3C4gXAir1m0VU8A2ulP+wLO1m1TaoZHxt+YHVY X-Gm-Message-State: AOJu0YxB9ejEG8mfxGhGK+1OfsNyfSJqIVYXsHNFH4kJbyJ7tdqfqkbC 8UES1NX6GkGxeCDfoIdpoQhwOplajM2Z0iYQGjIXFl/0h+Q9BDnL X-Google-Smtp-Source: AGHT+IFTksx4G/lsKVJ4YE2wR0cuLbVAbVcz3Q/Es4sHPwEpArtufxPn41aChi9fHjXL3E1KWRa3gQ== X-Received: by 2002:a17:906:694f:b0:a51:b228:9282 with SMTP id c15-20020a170906694f00b00a51b2289282mr5619021ejs.44.1712657113240; Tue, 09 Apr 2024 03:05:13 -0700 (PDT) Received: from fedora.iskraemeco.si ([193.77.86.250]) by smtp.gmail.com with ESMTPSA id qs1-20020a170906458100b00a4e6626ae21sm5496681ejc.0.2024.04.09.03.05.11 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Tue, 09 Apr 2024 03:05:12 -0700 (PDT) From: Uros Bizjak To: x86@kernel.org, linux-kernel@vger.kernel.org Cc: Uros Bizjak , Thomas Gleixner , Ingo Molnar , Borislav Petkov , Dave Hansen , "H. Peter Anvin" , Peter Zijlstra Subject: [PATCH 4/6] locking/atomic/x86: Merge x86_32 and x86_64 arch_atomic64_fetch_{and,or,xor}() functions Date: Tue, 9 Apr 2024 12:03:55 +0200 Message-ID: <20240409100503.274629-5-ubizjak@gmail.com> X-Mailer: git-send-email 2.44.0 In-Reply-To: <20240409100503.274629-1-ubizjak@gmail.com> References: <20240409100503.274629-1-ubizjak@gmail.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Move the same definitions of arch_atomic64_fetch_{and,or,xor}() from x86/include/asm/atomic64_32.h and x86/include/asm/atomic64_64.h to the common place in arch/x86/include/asm/atomic.h No functional changes intended. Signed-off-by: Uros Bizjak Cc: Thomas Gleixner Cc: Ingo Molnar Cc: Borislav Petkov Cc: Dave Hansen Cc: "H. Peter Anvin" Cc: Peter Zijlstra --- arch/x86/include/asm/atomic.h | 30 ++++++++++++++++++++++++++++++ arch/x86/include/asm/atomic64_32.h | 30 ------------------------------ arch/x86/include/asm/atomic64_64.h | 30 ------------------------------ 3 files changed, 30 insertions(+), 60 deletions(-) diff --git a/arch/x86/include/asm/atomic.h b/arch/x86/include/asm/atomic.h index b166da21ee98..b2e44de36934 100644 --- a/arch/x86/include/asm/atomic.h +++ b/arch/x86/include/asm/atomic.h @@ -182,4 +182,34 @@ static __always_inline int arch_atomic_fetch_xor(int i= , atomic_t *v) # include #endif =20 +static __always_inline s64 arch_atomic64_fetch_and(s64 i, atomic64_t *v) +{ + s64 val =3D __READ_ONCE(v->counter); + + do { } while (!arch_atomic64_try_cmpxchg(v, &val, val & i)); + + return val; +} +#define arch_atomic64_fetch_and arch_atomic64_fetch_and + +static __always_inline s64 arch_atomic64_fetch_or(s64 i, atomic64_t *v) +{ + s64 val =3D __READ_ONCE(v->counter); + + do { } while (!arch_atomic64_try_cmpxchg(v, &val, val | i)); + + return val; +} +#define arch_atomic64_fetch_or arch_atomic64_fetch_or + +static __always_inline s64 arch_atomic64_fetch_xor(s64 i, atomic64_t *v) +{ + s64 val =3D __READ_ONCE(v->counter); + + do { } while (!arch_atomic64_try_cmpxchg(v, &val, val ^ i)); + + return val; +} +#define arch_atomic64_fetch_xor arch_atomic64_fetch_xor + #endif /* _ASM_X86_ATOMIC_H */ diff --git a/arch/x86/include/asm/atomic64_32.h b/arch/x86/include/asm/atom= ic64_32.h index 84affd7a5d1c..4f79198da98e 100644 --- a/arch/x86/include/asm/atomic64_32.h +++ b/arch/x86/include/asm/atomic64_32.h @@ -206,16 +206,6 @@ static __always_inline void arch_atomic64_and(s64 i, a= tomic64_t *v) do { } while (!arch_atomic64_try_cmpxchg(v, &val, val & i)); } =20 -static __always_inline s64 arch_atomic64_fetch_and(s64 i, atomic64_t *v) -{ - s64 val =3D __READ_ONCE(v->counter); - - do { } while (!arch_atomic64_try_cmpxchg(v, &val, val & i)); - - return val; -} -#define arch_atomic64_fetch_and arch_atomic64_fetch_and - static __always_inline void arch_atomic64_or(s64 i, atomic64_t *v) { s64 val =3D __READ_ONCE(v->counter); @@ -223,16 +213,6 @@ static __always_inline void arch_atomic64_or(s64 i, at= omic64_t *v) do { } while (!arch_atomic64_try_cmpxchg(v, &val, val | i)); } =20 -static __always_inline s64 arch_atomic64_fetch_or(s64 i, atomic64_t *v) -{ - s64 val =3D __READ_ONCE(v->counter); - - do { } while (!arch_atomic64_try_cmpxchg(v, &val, val | i)); - - return val; -} -#define arch_atomic64_fetch_or arch_atomic64_fetch_or - static __always_inline void arch_atomic64_xor(s64 i, atomic64_t *v) { s64 val =3D __READ_ONCE(v->counter); @@ -240,16 +220,6 @@ static __always_inline void arch_atomic64_xor(s64 i, a= tomic64_t *v) do { } while (!arch_atomic64_try_cmpxchg(v, &val, val ^ i)); } =20 -static __always_inline s64 arch_atomic64_fetch_xor(s64 i, atomic64_t *v) -{ - s64 val =3D __READ_ONCE(v->counter); - - do { } while (!arch_atomic64_try_cmpxchg(v, &val, val ^ i)); - - return val; -} -#define arch_atomic64_fetch_xor arch_atomic64_fetch_xor - static __always_inline s64 arch_atomic64_fetch_add(s64 i, atomic64_t *v) { s64 val =3D __READ_ONCE(v->counter); diff --git a/arch/x86/include/asm/atomic64_64.h b/arch/x86/include/asm/atom= ic64_64.h index e7b12a48fecb..b2c9974ba971 100644 --- a/arch/x86/include/asm/atomic64_64.h +++ b/arch/x86/include/asm/atomic64_64.h @@ -124,16 +124,6 @@ static __always_inline void arch_atomic64_and(s64 i, a= tomic64_t *v) : "memory"); } =20 -static __always_inline s64 arch_atomic64_fetch_and(s64 i, atomic64_t *v) -{ - s64 val =3D __READ_ONCE(v->counter); - - do { } while (!arch_atomic64_try_cmpxchg(v, &val, val & i)); - - return val; -} -#define arch_atomic64_fetch_and arch_atomic64_fetch_and - static __always_inline void arch_atomic64_or(s64 i, atomic64_t *v) { asm volatile(LOCK_PREFIX "orq %1,%0" @@ -142,16 +132,6 @@ static __always_inline void arch_atomic64_or(s64 i, at= omic64_t *v) : "memory"); } =20 -static __always_inline s64 arch_atomic64_fetch_or(s64 i, atomic64_t *v) -{ - s64 val =3D __READ_ONCE(v->counter); - - do { } while (!arch_atomic64_try_cmpxchg(v, &val, val | i)); - - return val; -} -#define arch_atomic64_fetch_or arch_atomic64_fetch_or - static __always_inline void arch_atomic64_xor(s64 i, atomic64_t *v) { asm volatile(LOCK_PREFIX "xorq %1,%0" @@ -160,14 +140,4 @@ static __always_inline void arch_atomic64_xor(s64 i, a= tomic64_t *v) : "memory"); } =20 -static __always_inline s64 arch_atomic64_fetch_xor(s64 i, atomic64_t *v) -{ - s64 val =3D __READ_ONCE(v->counter); - - do { } while (!arch_atomic64_try_cmpxchg(v, &val, val ^ i)); - - return val; -} -#define arch_atomic64_fetch_xor arch_atomic64_fetch_xor - #endif /* _ASM_X86_ATOMIC64_64_H */ --=20 2.44.0 From nobody Sat Feb 7 13:41:31 2026 Received: from mail-lf1-f52.google.com (mail-lf1-f52.google.com [209.85.167.52]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id ECABF127B78 for ; Tue, 9 Apr 2024 10:05:16 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=209.85.167.52 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1712657118; cv=none; b=tYIY78dZoPNBW764uFb4Q9Qe00toowxUM7838rqyk8BaI+7imzFar9pb6fB9jRYLDIUt6vdujoo//IR6vpf7oAlbNg+o5AVAxacl3F5qSB5f+nuYjbKgW/e1lQolYE6R/u4Q4HmfnwLwndme5eaXw43AJBB1C6q7dKX1MFhHDg4= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1712657118; c=relaxed/simple; bh=+Fa+VjFTLuwGROrkdO13qvGvvADDnYTOxp3ABbZPnww=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=AE01dQB1sOM4evLBKr09ssv8VLDhGZo+ClZwcPWTNxn67sQaDAWoXFX8Gy5YIN1pkDTX8Qic52qg/iiXltXYTmhcMuRAYXUcfUiAMKapQSdmL5P+BkdtLZ2NijOq+nTwjNcnXLL+DjbczxfKa8ZRaSgNUBm5AFMqq1Yz3eVWneE= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=gmail.com; spf=pass smtp.mailfrom=gmail.com; dkim=pass (2048-bit key) header.d=gmail.com header.i=@gmail.com header.b=hBD1Dh6X; arc=none smtp.client-ip=209.85.167.52 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=gmail.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=gmail.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=gmail.com header.i=@gmail.com header.b="hBD1Dh6X" Received: by mail-lf1-f52.google.com with SMTP id 2adb3069b0e04-51381021af1so8730386e87.0 for ; Tue, 09 Apr 2024 03:05:16 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20230601; t=1712657115; x=1713261915; darn=vger.kernel.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=OjP8OuPSYU8pIC+kj3/su0ypXF+SYBCAH3lD5yqhYyM=; b=hBD1Dh6XFYrNwTj0zZI3kDf2r1xbMRYbziDOIlYhlzJuSq+GDxyIzHCOjUQbryEXri JTPfBr7UCWMnUsKhwohDwJpcaQiQ00VV8mFtVGMlExCFL2wGEsros5bN2qvjDKtDYj4F lZuleocWltHQzAbuJdoM3KeM/qmbfhhvzZUR1TX5gAtMaNWeoimDwfMeeyY8r9+PSrg1 fNm63Duz/Xi/rgoDsvZKb0fHm0Ax0N1v/w243yW0m5i4uNagzrmahMmOep+XIFXKEHOW EXCfjoFrBQYTGO7CYPtIKkV9/ztskN0xOkwvEKgXZVdZDKPU5kHKbwv7VjZmQuVYDzAX Ox5A== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1712657115; x=1713261915; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=OjP8OuPSYU8pIC+kj3/su0ypXF+SYBCAH3lD5yqhYyM=; b=bxVk/BanPaTxwvLQww4Byv2+jwyeGlLaAKNa9/vqAuyKghaGTRLTwAEfWkAuiGu2rW 8V8ky35uS+R5JbP+MKJFhiYcUfAWDvOf7MkZzj+E6xQJKzAbpIdjIAl5EWvajagK66au eAwAm2xT/Q2Ua6Kah9/MygWbsn8fW+tpN8Wd+qit1Yqz8sJCnpxBdJuNX0piLVgJpx0e O/A2SfUWLd5h4oiAuyhNfc50fVogG1XaxJOJkITrq8s8ilEwH+HI1uzPgB7oGDEWtp5v Xwmy15PRqkgiddAkWYAJc0w65+ldPFvwGc0737Ra3UVzw0QgbA667oQtqhkEeEogqnRA FxKw== X-Forwarded-Encrypted: i=1; AJvYcCWRWKc/4NH5D4+wEYL0c8VkZc/tjCuPRXPUX9yY/CUtiYeGW35mOHwdPGT+EVRabmlG9axGF/dCn8f7GMeqSG2SnGWLsv5XPai6bMMR X-Gm-Message-State: AOJu0YzPtZKXzMCkyGl9sn2ZqWhqOq5Pj7iutriLdKEbg9Ate0EmQqoD lwV9sO97LghrOcLRH4O8AwTqNpiEj60GCCPlxv2jCkl4v4C1QlxP X-Google-Smtp-Source: AGHT+IEZDqWek1R8pb5BSd4GD7BsxX4mL7A81DVxst0hWKSLUUuvUDWNrQX52A4wuEmiKVhoe11XUw== X-Received: by 2002:ac2:5302:0:b0:513:c223:f0e4 with SMTP id c2-20020ac25302000000b00513c223f0e4mr10321029lfh.10.1712657114965; Tue, 09 Apr 2024 03:05:14 -0700 (PDT) Received: from fedora.iskraemeco.si ([193.77.86.250]) by smtp.gmail.com with ESMTPSA id qs1-20020a170906458100b00a4e6626ae21sm5496681ejc.0.2024.04.09.03.05.13 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Tue, 09 Apr 2024 03:05:13 -0700 (PDT) From: Uros Bizjak To: x86@kernel.org, linux-kernel@vger.kernel.org Cc: Uros Bizjak , Thomas Gleixner , Ingo Molnar , Borislav Petkov , Dave Hansen , "H. Peter Anvin" , Peter Zijlstra Subject: [PATCH 5/6] locking/atomic/x86: Define arch_atomic_sub() family using arch_atomic_add() functions Date: Tue, 9 Apr 2024 12:03:56 +0200 Message-ID: <20240409100503.274629-6-ubizjak@gmail.com> X-Mailer: git-send-email 2.44.0 In-Reply-To: <20240409100503.274629-1-ubizjak@gmail.com> References: <20240409100503.274629-1-ubizjak@gmail.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" There is no need to implement arch_atomic_sub() family of inline functions, corresponding macros can be directly implemented using arch_atomic_add() inlines with negated argument. No functional changes intended. Signed-off-by: Uros Bizjak Cc: Thomas Gleixner Cc: Ingo Molnar Cc: Borislav Petkov Cc: Dave Hansen Cc: "H. Peter Anvin" Cc: Peter Zijlstra --- arch/x86/include/asm/atomic.h | 12 ++---------- arch/x86/include/asm/atomic64_32.h | 23 ++++++++++++----------- arch/x86/include/asm/atomic64_64.h | 12 ++---------- 3 files changed, 16 insertions(+), 31 deletions(-) diff --git a/arch/x86/include/asm/atomic.h b/arch/x86/include/asm/atomic.h index b2e44de36934..77526257dacf 100644 --- a/arch/x86/include/asm/atomic.h +++ b/arch/x86/include/asm/atomic.h @@ -86,11 +86,7 @@ static __always_inline int arch_atomic_add_return(int i,= atomic_t *v) } #define arch_atomic_add_return arch_atomic_add_return =20 -static __always_inline int arch_atomic_sub_return(int i, atomic_t *v) -{ - return arch_atomic_add_return(-i, v); -} -#define arch_atomic_sub_return arch_atomic_sub_return +#define arch_atomic_sub_return(i, v) arch_atomic_add_return(-(i), v) =20 static __always_inline int arch_atomic_fetch_add(int i, atomic_t *v) { @@ -98,11 +94,7 @@ static __always_inline int arch_atomic_fetch_add(int i, = atomic_t *v) } #define arch_atomic_fetch_add arch_atomic_fetch_add =20 -static __always_inline int arch_atomic_fetch_sub(int i, atomic_t *v) -{ - return xadd(&v->counter, -i); -} -#define arch_atomic_fetch_sub arch_atomic_fetch_sub +#define arch_atomic_fetch_sub(i, v) arch_atomic_fetch_add(-(i), v) =20 static __always_inline int arch_atomic_cmpxchg(atomic_t *v, int old, int n= ew) { diff --git a/arch/x86/include/asm/atomic64_32.h b/arch/x86/include/asm/atom= ic64_32.h index 4f79198da98e..862448db1207 100644 --- a/arch/x86/include/asm/atomic64_32.h +++ b/arch/x86/include/asm/atomic64_32.h @@ -199,6 +199,18 @@ static __always_inline s64 arch_atomic64_dec_if_positi= ve(atomic64_t *v) #undef alternative_atomic64 #undef __alternative_atomic64 =20 +static __always_inline s64 arch_atomic64_fetch_add(s64 i, atomic64_t *v) +{ + s64 val =3D __READ_ONCE(v->counter); + + do { } while (!arch_atomic64_try_cmpxchg(v, &val, val + i)); + + return val; +} +#define arch_atomic64_fetch_add arch_atomic64_fetch_add + +#define arch_atomic64_fetch_sub(i, v) arch_atomic64_fetch_add(-(i), v) + static __always_inline void arch_atomic64_and(s64 i, atomic64_t *v) { s64 val =3D __READ_ONCE(v->counter); @@ -220,15 +232,4 @@ static __always_inline void arch_atomic64_xor(s64 i, a= tomic64_t *v) do { } while (!arch_atomic64_try_cmpxchg(v, &val, val ^ i)); } =20 -static __always_inline s64 arch_atomic64_fetch_add(s64 i, atomic64_t *v) -{ - s64 val =3D __READ_ONCE(v->counter); - - do { } while (!arch_atomic64_try_cmpxchg(v, &val, val + i)); - return val; -} -#define arch_atomic64_fetch_add arch_atomic64_fetch_add - -#define arch_atomic64_fetch_sub(i, v) arch_atomic64_fetch_add(-(i), (v)) - #endif /* _ASM_X86_ATOMIC64_32_H */ diff --git a/arch/x86/include/asm/atomic64_64.h b/arch/x86/include/asm/atom= ic64_64.h index b2c9974ba971..a96a4b9acfcb 100644 --- a/arch/x86/include/asm/atomic64_64.h +++ b/arch/x86/include/asm/atomic64_64.h @@ -80,11 +80,7 @@ static __always_inline s64 arch_atomic64_add_return(s64 = i, atomic64_t *v) } #define arch_atomic64_add_return arch_atomic64_add_return =20 -static __always_inline s64 arch_atomic64_sub_return(s64 i, atomic64_t *v) -{ - return arch_atomic64_add_return(-i, v); -} -#define arch_atomic64_sub_return arch_atomic64_sub_return +#define arch_atomic64_sub_return(i, v) arch_atomic64_add_return(-(i), v) =20 static __always_inline s64 arch_atomic64_fetch_add(s64 i, atomic64_t *v) { @@ -92,11 +88,7 @@ static __always_inline s64 arch_atomic64_fetch_add(s64 i= , atomic64_t *v) } #define arch_atomic64_fetch_add arch_atomic64_fetch_add =20 -static __always_inline s64 arch_atomic64_fetch_sub(s64 i, atomic64_t *v) -{ - return xadd(&v->counter, -i); -} -#define arch_atomic64_fetch_sub arch_atomic64_fetch_sub +#define arch_atomic64_fetch_sub(i, v) arch_atomic64_fetch_add(-(i), v) =20 static __always_inline s64 arch_atomic64_cmpxchg(atomic64_t *v, s64 old, s= 64 new) { --=20 2.44.0 From nobody Sat Feb 7 13:41:31 2026 Received: from mail-ej1-f43.google.com (mail-ej1-f43.google.com [209.85.218.43]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 6D578129A7B for ; Tue, 9 Apr 2024 10:05:18 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=209.85.218.43 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1712657120; cv=none; b=o9I+C0NJg37FDxDIrdqJvkoIlfeO/wh7JLI47si4IF2Ewt70WUYi9ezUudc5HUNjmdOTvrTdPkgY+tAB/lzcI4Oel7y/OEUFBoeennmk+fAduH5/7IqC9/twx/71mWDL5YH33Pvl9CPSD0b++2IKLEwVeo2b5XZLeNQaAxo6Y9s= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1712657120; c=relaxed/simple; bh=+IJA9+/LkDO3Qh/z4ZzTDQ69ze3J+3wDzxu13Sj1clQ=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=QykGzBAqdV6cYzvbgGolW358DBpdUjVzGLCj7HgpMuVSbfBXYLd4+TqdhAH4kAtcwSFF6OKa1eQoHHAtbbWB04GU8akjycUZe29D5MwcJgtrKGYK1CuSbr9waf+DA9Yqhw0bWysOBQEJRAFgmNdtNkN2Z1zCF0CK6dzoRM67UX8= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=gmail.com; spf=pass smtp.mailfrom=gmail.com; dkim=pass (2048-bit key) header.d=gmail.com header.i=@gmail.com header.b=bC6p1iHW; arc=none smtp.client-ip=209.85.218.43 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=gmail.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=gmail.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=gmail.com header.i=@gmail.com header.b="bC6p1iHW" Received: by mail-ej1-f43.google.com with SMTP id a640c23a62f3a-a5200202c1bso26184666b.0 for ; Tue, 09 Apr 2024 03:05:18 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20230601; t=1712657117; x=1713261917; darn=vger.kernel.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=lyHz56u/iO9/FgsI4q0cP6V/rorpyyesh7+7+zlmKI8=; b=bC6p1iHW8rZgqBzJSrEDMfsBA8aTO0wbFgbYcGVyKUpUB4JNc/tKHr2S8OdXpgabDz 2DMYFlMzl25nixcs6I64S795bRhd5dhCO2cuYQVQXKHuGN6ZDSNrdez3VQf+PXqcTzpI r2iF9EzOAMSHbUB61JpiLH48+Fn/ee0pp34JuSEp8NipWxg7YS/N60ZWLSvvFQVu+vVE GUYszspOjp9Ar/f0j4pQV9gQdOtcFhX7zLWuDdUljn/fQA3VdkgghI6lwKsIIn6phpiG 8PijcE2aRcLEjmxGl4wA2l5LtSlMrEPBHDY0Wk5WcLKVkoajljBHjb87Qwt4NNLHx0Ep 5x0w== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1712657117; x=1713261917; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=lyHz56u/iO9/FgsI4q0cP6V/rorpyyesh7+7+zlmKI8=; b=qAxiTrTRMziYMLl4vF6x9+3ftV7pTJ5FP/pqSNPX6Cpj5RQgLMPbwXHr49tf4UMQWD WCpB+NUOrh3FcSXZtR+SYx4dXeBljV9TRoTcLqfYGRkp0etzto/2+GlRSVLbGyqKiqbs xGKJn1biRgHedCUzWW7+0jvtKi28NbAoLQhREyHVwoiXBnUMcFY+3XM+q4OMgVBZ2ayx VrdfJBEA31KMh+jOooArfUUCsn17shSNfw6/T+avI+UsE31ZSDZRviZWG49udNL9OabD 3VhHdK8kglUfjwzAAMa9vjqoCfin7J+RuhCjMat5fbD28M8FG3tv9FPXt5mxA/3Tf9Km Z/9w== X-Forwarded-Encrypted: i=1; AJvYcCWefhON1LfgyPf+vo98S3HXbwUUo45X0BK99CIL4LcBolKAIeE/YqgLI9wGkWcVVsj06HEB1dlEmEub51WG+ZKWqpdKYcHreK12SoKc X-Gm-Message-State: AOJu0YwGJ6efaJq3NQ0f+qaO4uXrdL6WdlOvyKnyl1xtWeZk8jh+4LVI KR7uxmJo6rQshTxz7SGLTVwOGG9yRioD8PLWtLrh/V9yOJNH9iq4 X-Google-Smtp-Source: AGHT+IH4uESLV606lHisWITKlS8BD5JNspBkRmUwN0p7WVc2Hamq61jeR9aVU0Fl23GolotFpGbE+A== X-Received: by 2002:a17:907:7ea4:b0:a4e:5676:417d with SMTP id qb36-20020a1709077ea400b00a4e5676417dmr12020143ejc.58.1712657116669; Tue, 09 Apr 2024 03:05:16 -0700 (PDT) Received: from fedora.iskraemeco.si ([193.77.86.250]) by smtp.gmail.com with ESMTPSA id qs1-20020a170906458100b00a4e6626ae21sm5496681ejc.0.2024.04.09.03.05.15 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Tue, 09 Apr 2024 03:05:15 -0700 (PDT) From: Uros Bizjak To: x86@kernel.org, linux-kernel@vger.kernel.org Cc: Uros Bizjak , Thomas Gleixner , Ingo Molnar , Borislav Petkov , Dave Hansen , "H. Peter Anvin" , Peter Zijlstra Subject: [PATCH 6/6] locking/atomic/x86: Reorder a couple of arch_atomic64 functions Date: Tue, 9 Apr 2024 12:03:57 +0200 Message-ID: <20240409100503.274629-7-ubizjak@gmail.com> X-Mailer: git-send-email 2.44.0 In-Reply-To: <20240409100503.274629-1-ubizjak@gmail.com> References: <20240409100503.274629-1-ubizjak@gmail.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Reorder a couple of arch_atomic64 functions in arch/x86/include/asm/atomic64_32.h to better match their sequence of declarations between x86_32 and x86_64. No functional changes intended. Signed-off-by: Uros Bizjak Cc: Thomas Gleixner Cc: Ingo Molnar Cc: Borislav Petkov Cc: Dave Hansen Cc: "H. Peter Anvin" Cc: Peter Zijlstra --- arch/x86/include/asm/atomic64_32.h | 46 +++++++++++++++--------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/arch/x86/include/asm/atomic64_32.h b/arch/x86/include/asm/atom= ic64_32.h index 862448db1207..3864d82a9339 100644 --- a/arch/x86/include/asm/atomic64_32.h +++ b/arch/x86/include/asm/atomic64_32.h @@ -12,7 +12,7 @@ typedef struct { s64 __aligned(8) counter; } atomic64_t; =20 -#define ATOMIC64_INIT(val) { (val) } +#define ATOMIC64_INIT(i) { (i) } =20 #define __ATOMIC64_DECL(sym) void atomic64_##sym(atomic64_t *, ...) #ifndef ATOMIC64_EXPORT @@ -61,17 +61,21 @@ ATOMIC64_DECL(add_unless); #undef __ATOMIC64_DECL #undef ATOMIC64_EXPORT =20 -static __always_inline s64 arch_atomic64_cmpxchg(atomic64_t *v, s64 old, s= 64 new) +static __always_inline s64 arch_atomic64_read(const atomic64_t *v) { - return arch_cmpxchg64(&v->counter, old, new); + s64 r; + alternative_atomic64(read, "=3D&A" (r), "c" (v) : "memory"); + return r; } -#define arch_atomic64_cmpxchg arch_atomic64_cmpxchg =20 -static __always_inline bool arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *= old, s64 new) +static __always_inline void arch_atomic64_set(atomic64_t *v, s64 i) { - return arch_try_cmpxchg64(&v->counter, old, new); + unsigned high =3D (unsigned)(i >> 32); + unsigned low =3D (unsigned)i; + alternative_atomic64(set, /* no output */, + "S" (v), "b" (low), "c" (high) + : "eax", "edx", "memory"); } -#define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg =20 static __always_inline s64 arch_atomic64_xchg(atomic64_t *v, s64 n) { @@ -85,22 +89,6 @@ static __always_inline s64 arch_atomic64_xchg(atomic64_t= *v, s64 n) } #define arch_atomic64_xchg arch_atomic64_xchg =20 -static __always_inline void arch_atomic64_set(atomic64_t *v, s64 i) -{ - unsigned high =3D (unsigned)(i >> 32); - unsigned low =3D (unsigned)i; - alternative_atomic64(set, /* no output */, - "S" (v), "b" (low), "c" (high) - : "eax", "edx", "memory"); -} - -static __always_inline s64 arch_atomic64_read(const atomic64_t *v) -{ - s64 r; - alternative_atomic64(read, "=3D&A" (r), "c" (v) : "memory"); - return r; -} - static __always_inline s64 arch_atomic64_add_return(s64 i, atomic64_t *v) { alternative_atomic64(add_return, @@ -199,6 +187,18 @@ static __always_inline s64 arch_atomic64_dec_if_positi= ve(atomic64_t *v) #undef alternative_atomic64 #undef __alternative_atomic64 =20 +static __always_inline s64 arch_atomic64_cmpxchg(atomic64_t *v, s64 old, s= 64 new) +{ + return arch_cmpxchg64(&v->counter, old, new); +} +#define arch_atomic64_cmpxchg arch_atomic64_cmpxchg + +static __always_inline bool arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *= old, s64 new) +{ + return arch_try_cmpxchg64(&v->counter, old, new); +} +#define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg + static __always_inline s64 arch_atomic64_fetch_add(s64 i, atomic64_t *v) { s64 val =3D __READ_ONCE(v->counter); --=20 2.44.0