From nobody Wed Jan 7 03:30:13 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 833EBE748E9 for ; Sun, 1 Oct 2023 13:16:56 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S234997AbjJANQp (ORCPT ); Sun, 1 Oct 2023 09:16:45 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:34596 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S229716AbjJANQk (ORCPT ); Sun, 1 Oct 2023 09:16:40 -0400 Received: from mail-ej1-x635.google.com (mail-ej1-x635.google.com [IPv6:2a00:1450:4864:20::635]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id F3A9EB8 for ; Sun, 1 Oct 2023 06:16:37 -0700 (PDT) Received: by mail-ej1-x635.google.com with SMTP id a640c23a62f3a-9a58dbd5daeso2167839066b.2 for ; Sun, 01 Oct 2023 06:16:37 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20230601; t=1696166196; x=1696770996; darn=vger.kernel.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=qamY6WyojZk0Zj2T/3tRAPkmbEH4lz9al7O5t6kqV54=; b=H1G09yVBQ44Actd6QwTMkGczEtqfw2iknnYzQm3L7FmyD0M7zYpOAg8ucLQQI1OijJ mzIn0FmoQI5E7PzwMS9x38U/zy6unKyErjqSIL5VdM28TVvesTduHsOwgL+ST+DMCzJj +bTaFvnJi6H85DuAUDZ5JJg7TbS95cmL8YBXGHR728xu/lqar5yuwAkHxbiYh5mf7qGI rgSEohvmHh2T12gxcCcTnp3f97sofV8iseeuIi3ddHIWCB6T+1GctxL/eJuhpPmnlfz1 gegRSJKN3+kUYa2vuLrFtJddXK7rb5Gv7/PhCkKLYnXPcA/eZKtSFsOAiBCmvXXYBJ8z mOZQ== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1696166196; x=1696770996; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=qamY6WyojZk0Zj2T/3tRAPkmbEH4lz9al7O5t6kqV54=; b=u8Hs4TI42BfxrS9C23TkeK/HAU/WYLFocgUZvRTBM4oQM9Jag3ysx7jjQ+Eejj7cIw pe3ELXBkzQd9jCoAWBYBEUqJkGLFpMMdySW14nFqI+9icncd8TcRbO3DPDsjrvEkBYji vh4n71kariPbKavKAIDki4f+uQDgQRjMbjKNW8n0ggefGYiew03T5YXvI4I0tRVNM6hN nBIAcvYxQtRLotejIsXOo1K0WjiwsMuMrxcnH+71VD5KGXPO//Ub52WhOGD3y3X9pUus FpZjnL1jdB/PPp/saxgZvdXojPwgwIPTeqshJxBRNQFZ2OyNC5YF2oaIhMUmGLUY5SgG IUpA== X-Gm-Message-State: AOJu0YxGPMJ3Jm+3UX205KTvTvhI8F3PwS7L0tug+Le6PeXZntOQOKJl Z0jZbsTgauyktvabQyYRhDHQrn1HOvjcQU+Q X-Google-Smtp-Source: AGHT+IHRWZurJereLL48r/V2TUF1m5AcurnyEwrKYdgueDjw8zZWEiuBO0v7J8Q+dP/7Kfo3Ce+5ww== X-Received: by 2002:a17:907:770e:b0:9ae:738b:86d0 with SMTP id kw14-20020a170907770e00b009ae738b86d0mr8701612ejc.66.1696166196167; Sun, 01 Oct 2023 06:16:36 -0700 (PDT) Received: from localhost.localdomain ([46.248.82.114]) by smtp.gmail.com with ESMTPSA id lv23-20020a170906bc9700b0099d798a6bb5sm15504272ejb.67.2023.10.01.06.16.35 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Sun, 01 Oct 2023 06:16:35 -0700 (PDT) From: Uros Bizjak To: x86@kernel.org, linux-kernel@vger.kernel.org Cc: Uros Bizjak , Andy Lutomirski , Ingo Molnar , Nadav Amit , Brian Gerst , Denys Vlasenko , "H . Peter Anvin" , Linus Torvalds , Peter Zijlstra , Thomas Gleixner , Borislav Petkov , Josh Poimboeuf Subject: [RFC PATCH 1/4] x86/percpu: Update arch/x86/include/asm/percpu.h to the current tip Date: Sun, 1 Oct 2023 15:14:33 +0200 Message-ID: <20231001131620.112484-2-ubizjak@gmail.com> X-Mailer: git-send-email 2.41.0 In-Reply-To: <20231001131620.112484-1-ubizjak@gmail.com> References: <20231001131620.112484-1-ubizjak@gmail.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" This is just a convenient patch that brings current mainline version of arch/x86/include/asm/percpu.h to the version in the current tip tree. Cc: Andy Lutomirski Cc: Ingo Molnar Cc: Nadav Amit Cc: Brian Gerst Cc: Denys Vlasenko Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Borislav Petkov Cc: Josh Poimboeuf Signed-off-by: Uros Bizjak --- arch/x86/include/asm/percpu.h | 110 ++++++++++++++++++++++++++++++++-- 1 file changed, 104 insertions(+), 6 deletions(-) diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h index 34734d730463..20624b80f890 100644 --- a/arch/x86/include/asm/percpu.h +++ b/arch/x86/include/asm/percpu.h @@ -210,6 +210,25 @@ do { \ (typeof(_var))(unsigned long) pco_old__; \ }) =20 +#define percpu_try_cmpxchg_op(size, qual, _var, _ovalp, _nval) \ +({ \ + bool success; \ + __pcpu_type_##size *pco_oval__ =3D (__pcpu_type_##size *)(_ovalp); \ + __pcpu_type_##size pco_old__ =3D *pco_oval__; \ + __pcpu_type_##size pco_new__ =3D __pcpu_cast_##size(_nval); \ + asm qual (__pcpu_op2_##size("cmpxchg", "%[nval]", \ + __percpu_arg([var])) \ + CC_SET(z) \ + : CC_OUT(z) (success), \ + [oval] "+a" (pco_old__), \ + [var] "+m" (_var) \ + : [nval] __pcpu_reg_##size(, pco_new__) \ + : "memory"); \ + if (unlikely(!success)) \ + *pco_oval__ =3D pco_old__; \ + likely(success); \ +}) + #if defined(CONFIG_X86_32) && !defined(CONFIG_UML) #define percpu_cmpxchg64_op(size, qual, _var, _oval, _nval) \ ({ \ @@ -223,26 +242,63 @@ do { \ old__.var =3D _oval; \ new__.var =3D _nval; \ \ - asm qual (ALTERNATIVE("leal %P[var], %%esi; call this_cpu_cmpxchg8b_emu",= \ + asm qual (ALTERNATIVE("call this_cpu_cmpxchg8b_emu", \ "cmpxchg8b " __percpu_arg([var]), X86_FEATURE_CX8) \ : [var] "+m" (_var), \ "+a" (old__.low), \ "+d" (old__.high) \ : "b" (new__.low), \ - "c" (new__.high) \ - : "memory", "esi"); \ + "c" (new__.high), \ + "S" (&(_var)) \ + : "memory"); \ \ old__.var; \ }) =20 #define raw_cpu_cmpxchg64(pcp, oval, nval) percpu_cmpxchg64_op(8, = , pcp, oval, nval) #define this_cpu_cmpxchg64(pcp, oval, nval) percpu_cmpxchg64_op(8, volatil= e, pcp, oval, nval) + +#define percpu_try_cmpxchg64_op(size, qual, _var, _ovalp, _nval) \ +({ \ + bool success; \ + u64 *_oval =3D (u64 *)(_ovalp); \ + union { \ + u64 var; \ + struct { \ + u32 low, high; \ + }; \ + } old__, new__; \ + \ + old__.var =3D *_oval; \ + new__.var =3D _nval; \ + \ + asm qual (ALTERNATIVE("call this_cpu_cmpxchg8b_emu", \ + "cmpxchg8b " __percpu_arg([var]), X86_FEATURE_CX8) \ + CC_SET(z) \ + : CC_OUT(z) (success), \ + [var] "+m" (_var), \ + "+a" (old__.low), \ + "+d" (old__.high) \ + : "b" (new__.low), \ + "c" (new__.high), \ + "S" (&(_var)) \ + : "memory"); \ + if (unlikely(!success)) \ + *_oval =3D old__.var; \ + likely(success); \ +}) + +#define raw_cpu_try_cmpxchg64(pcp, ovalp, nval) percpu_try_cmpxchg64_op(8= , , pcp, ovalp, nval) +#define this_cpu_try_cmpxchg64(pcp, ovalp, nval) percpu_try_cmpxchg64_op(8= , volatile, pcp, ovalp, nval) #endif =20 #ifdef CONFIG_X86_64 #define raw_cpu_cmpxchg64(pcp, oval, nval) percpu_cmpxchg_op(8, , = pcp, oval, nval); #define this_cpu_cmpxchg64(pcp, oval, nval) percpu_cmpxchg_op(8, volatile,= pcp, oval, nval); =20 +#define raw_cpu_try_cmpxchg64(pcp, ovalp, nval) percpu_try_cmpxchg_op(8, = , pcp, ovalp, nval); +#define this_cpu_try_cmpxchg64(pcp, ovalp, nval) percpu_try_cmpxchg_op(8, = volatile, pcp, ovalp, nval); + #define percpu_cmpxchg128_op(size, qual, _var, _oval, _nval) \ ({ \ union { \ @@ -255,20 +311,54 @@ do { \ old__.var =3D _oval; \ new__.var =3D _nval; \ \ - asm qual (ALTERNATIVE("leaq %P[var], %%rsi; call this_cpu_cmpxchg16b_emu"= , \ + asm qual (ALTERNATIVE("call this_cpu_cmpxchg16b_emu", \ "cmpxchg16b " __percpu_arg([var]), X86_FEATURE_CX16) \ : [var] "+m" (_var), \ "+a" (old__.low), \ "+d" (old__.high) \ : "b" (new__.low), \ - "c" (new__.high) \ - : "memory", "rsi"); \ + "c" (new__.high), \ + "S" (&(_var)) \ + : "memory"); \ \ old__.var; \ }) =20 #define raw_cpu_cmpxchg128(pcp, oval, nval) percpu_cmpxchg128_op(16, = , pcp, oval, nval) #define this_cpu_cmpxchg128(pcp, oval, nval) percpu_cmpxchg128_op(16, vola= tile, pcp, oval, nval) + +#define percpu_try_cmpxchg128_op(size, qual, _var, _ovalp, _nval) \ +({ \ + bool success; \ + u128 *_oval =3D (u128 *)(_ovalp); \ + union { \ + u128 var; \ + struct { \ + u64 low, high; \ + }; \ + } old__, new__; \ + \ + old__.var =3D *_oval; \ + new__.var =3D _nval; \ + \ + asm qual (ALTERNATIVE("call this_cpu_cmpxchg16b_emu", \ + "cmpxchg16b " __percpu_arg([var]), X86_FEATURE_CX16) \ + CC_SET(z) \ + : CC_OUT(z) (success), \ + [var] "+m" (_var), \ + "+a" (old__.low), \ + "+d" (old__.high) \ + : "b" (new__.low), \ + "c" (new__.high), \ + "S" (&(_var)) \ + : "memory"); \ + if (unlikely(!success)) \ + *_oval =3D old__.var; \ + likely(success); \ +}) + +#define raw_cpu_try_cmpxchg128(pcp, ovalp, nval) percpu_try_cmpxchg128_op(= 16, , pcp, ovalp, nval) +#define this_cpu_try_cmpxchg128(pcp, ovalp, nval) percpu_try_cmpxchg128_op= (16, volatile, pcp, ovalp, nval) #endif =20 /* @@ -343,6 +433,9 @@ do { \ #define raw_cpu_cmpxchg_1(pcp, oval, nval) percpu_cmpxchg_op(1, , pcp, ova= l, nval) #define raw_cpu_cmpxchg_2(pcp, oval, nval) percpu_cmpxchg_op(2, , pcp, ova= l, nval) #define raw_cpu_cmpxchg_4(pcp, oval, nval) percpu_cmpxchg_op(4, , pcp, ova= l, nval) +#define raw_cpu_try_cmpxchg_1(pcp, ovalp, nval) percpu_try_cmpxchg_op(1, ,= pcp, ovalp, nval) +#define raw_cpu_try_cmpxchg_2(pcp, ovalp, nval) percpu_try_cmpxchg_op(2, ,= pcp, ovalp, nval) +#define raw_cpu_try_cmpxchg_4(pcp, ovalp, nval) percpu_try_cmpxchg_op(4, ,= pcp, ovalp, nval) =20 #define this_cpu_add_return_1(pcp, val) percpu_add_return_op(1, volatile,= pcp, val) #define this_cpu_add_return_2(pcp, val) percpu_add_return_op(2, volatile,= pcp, val) @@ -350,6 +443,9 @@ do { \ #define this_cpu_cmpxchg_1(pcp, oval, nval) percpu_cmpxchg_op(1, volatile,= pcp, oval, nval) #define this_cpu_cmpxchg_2(pcp, oval, nval) percpu_cmpxchg_op(2, volatile,= pcp, oval, nval) #define this_cpu_cmpxchg_4(pcp, oval, nval) percpu_cmpxchg_op(4, volatile,= pcp, oval, nval) +#define this_cpu_try_cmpxchg_1(pcp, ovalp, nval) percpu_try_cmpxchg_op(1, = volatile, pcp, ovalp, nval) +#define this_cpu_try_cmpxchg_2(pcp, ovalp, nval) percpu_try_cmpxchg_op(2, = volatile, pcp, ovalp, nval) +#define this_cpu_try_cmpxchg_4(pcp, ovalp, nval) percpu_try_cmpxchg_op(4, = volatile, pcp, ovalp, nval) =20 /* * Per cpu atomic 64 bit operations are only available under 64 bit. @@ -364,6 +460,7 @@ do { \ #define raw_cpu_add_return_8(pcp, val) percpu_add_return_op(8, , pcp, val) #define raw_cpu_xchg_8(pcp, nval) raw_percpu_xchg_op(pcp, nval) #define raw_cpu_cmpxchg_8(pcp, oval, nval) percpu_cmpxchg_op(8, , pcp, ova= l, nval) +#define raw_cpu_try_cmpxchg_8(pcp, ovalp, nval) percpu_try_cmpxchg_op(8, ,= pcp, ovalp, nval) =20 #define this_cpu_read_8(pcp) percpu_from_op(8, volatile, "mov", pcp) #define this_cpu_write_8(pcp, val) percpu_to_op(8, volatile, "mov", (pcp)= , val) @@ -373,6 +470,7 @@ do { \ #define this_cpu_add_return_8(pcp, val) percpu_add_return_op(8, volatile,= pcp, val) #define this_cpu_xchg_8(pcp, nval) percpu_xchg_op(8, volatile, pcp, nval) #define this_cpu_cmpxchg_8(pcp, oval, nval) percpu_cmpxchg_op(8, volatile,= pcp, oval, nval) +#define this_cpu_try_cmpxchg_8(pcp, ovalp, nval) percpu_try_cmpxchg_op(8, = volatile, pcp, ovalp, nval) #endif =20 static __always_inline bool x86_this_cpu_constant_test_bit(unsigned int nr, --=20 2.41.0 From nobody Wed Jan 7 03:30:13 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id C658EE748F5 for ; Sun, 1 Oct 2023 13:16:56 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S235006AbjJANQt (ORCPT ); Sun, 1 Oct 2023 09:16:49 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:34600 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S234984AbjJANQm (ORCPT ); Sun, 1 Oct 2023 09:16:42 -0400 Received: from mail-ej1-x634.google.com (mail-ej1-x634.google.com [IPv6:2a00:1450:4864:20::634]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id CA6DFBA for ; Sun, 1 Oct 2023 06:16:38 -0700 (PDT) Received: by mail-ej1-x634.google.com with SMTP id a640c23a62f3a-9a9f139cd94so2053738866b.2 for ; Sun, 01 Oct 2023 06:16:38 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20230601; t=1696166197; x=1696770997; darn=vger.kernel.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=GJ0tDwQlRD6Ds2+VaCyIxp6VxUBqGS+A19FnDIU/GbU=; b=i2u356GhIprffm6YXVr6jVWf8UFEOvllXrTckigMg2VwOjqbjJgxATnqTQlCbiJ43k 0Gi36VnVce1JPbccduU/eSnU777IkWpGksaVvvV2qrx1GcvFPtB/wS1bNO6n4e5o0fbT 1bXvCpyxcQIJhYW/qH0ex2WkSBfHxhU8M0V5F8TpbfQ7aQ8GuouCCLs1xsmp9PorYAHi RYIoYDZ53fNr1fF4F9j5xoqSKNZl9OTqZXxeKycJX8tw1lP2umYwrJfmaJ9ca/uTIx03 qeOo4h7IC1rA/Bj/sIg8peXIumL04uQyPdvpWM/ZBtJGMPFGf7qMSmD+LpXIL1QwcqRj hxQg== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1696166197; x=1696770997; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=GJ0tDwQlRD6Ds2+VaCyIxp6VxUBqGS+A19FnDIU/GbU=; b=TkV6bsdP50/uVp5o+1VDiTeqAw/b3CXiae5Z6AJRfde0eX/46nqwHhoR3FnoBJ/OMo A6W1yPjSG/EEnfyZauEmAG2P3x320ALUBx/4EZXSs7jkYSQoEnblMm+FeIcIHrzYOBCY 57OulQJWSugm3oC/7XvwiNk2OhF01HtLdG1AkrDM8KrzSaVH0mQCDB5hqbZV/r1cwuVv Da985bCRf0O1Z6bG8ujy5jvfIQa871NIYfq/Va/bbX6AFzsGMQXOb8R01H8HCctO+X5U qaBfzO6YR43NrCwpm9iT1E9bZ/x7Fndlf8mNENBU1JK2b0IwRjgs9fVpZMqm7/W3lOdj BU/w== X-Gm-Message-State: AOJu0YyoPBSBh1UuuL1Ij9D+TdX38Ec8UymlQ8AJ1Av70D3sNJ992adD ku+/nnFTtAr8GZuGPkp9f5k= X-Google-Smtp-Source: AGHT+IH+iG2cmTsm7yj58wvz1hW5RV3lsVS1dxDNFVwQS+dO+T7ENDzoRDzdClCgFS6guYkEuEAWSg== X-Received: by 2002:a17:906:210a:b0:9b2:b149:b818 with SMTP id 10-20020a170906210a00b009b2b149b818mr8574770ejt.70.1696166197218; Sun, 01 Oct 2023 06:16:37 -0700 (PDT) Received: from localhost.localdomain ([46.248.82.114]) by smtp.gmail.com with ESMTPSA id lv23-20020a170906bc9700b0099d798a6bb5sm15504272ejb.67.2023.10.01.06.16.36 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Sun, 01 Oct 2023 06:16:36 -0700 (PDT) From: Uros Bizjak To: x86@kernel.org, linux-kernel@vger.kernel.org Cc: Uros Bizjak , Andy Lutomirski , Ingo Molnar , Nadav Amit , Brian Gerst , Denys Vlasenko , "H . Peter Anvin" , Linus Torvalds , Peter Zijlstra , Thomas Gleixner , Borislav Petkov , Josh Poimboeuf Subject: [RFC PATCH 2/4] x86/percpu: Detect compiler support for named address spaces Date: Sun, 1 Oct 2023 15:14:34 +0200 Message-ID: <20231001131620.112484-3-ubizjak@gmail.com> X-Mailer: git-send-email 2.41.0 In-Reply-To: <20231001131620.112484-1-ubizjak@gmail.com> References: <20231001131620.112484-1-ubizjak@gmail.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" Detect compiler support for named address spaces by trying to compile a small testcase that includes __seg_gs and __seg_gs keywords. Reflect successful compilation in CC_HAS_NAMED_AS and set USE_X86_SEG_SUPPORT to signal when segment qualifiers should be used. Cc: Andy Lutomirski Cc: Ingo Molnar Cc: Nadav Amit Cc: Brian Gerst Cc: Denys Vlasenko Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Borislav Petkov Cc: Josh Poimboeuf Signed-off-by: Uros Bizjak --- arch/x86/Kconfig | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 66bfabae8814..f66fa5bf1f78 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -2388,6 +2388,13 @@ source "kernel/livepatch/Kconfig" =20 endmenu =20 +config CC_HAS_NAMED_AS + def_bool $(success,echo 'int __seg_fs fs; int __seg_gs gs;' | $(CC) -x c = - -c -o /dev/null) + +config USE_X86_SEG_SUPPORT + def_bool y + depends on CC_HAS_NAMED_AS && SMP + config CC_HAS_SLS def_bool $(cc-option,-mharden-sls=3Dall) =20 --=20 2.41.0 From nobody Wed Jan 7 03:30:13 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id B5FE4E748F6 for ; Sun, 1 Oct 2023 13:16:56 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S235020AbjJANQx (ORCPT ); Sun, 1 Oct 2023 09:16:53 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:34610 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S234956AbjJANQm (ORCPT ); Sun, 1 Oct 2023 09:16:42 -0400 Received: from mail-ej1-x629.google.com (mail-ej1-x629.google.com [IPv6:2a00:1450:4864:20::629]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id ED0B8B7 for ; Sun, 1 Oct 2023 06:16:39 -0700 (PDT) Received: by mail-ej1-x629.google.com with SMTP id a640c23a62f3a-9a9f139cd94so2053740766b.2 for ; Sun, 01 Oct 2023 06:16:39 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20230601; t=1696166198; x=1696770998; darn=vger.kernel.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=rS7jRVcQ6NtK3qXxw1G/jurrddIxGHuAxg9jtCM/lIk=; b=m2mupvdZAqrvIuno5IiJqu1g9qD8GzP3o9vEponxLqFKoBzLJhUj8Dfi5v10h+Ntt5 exI3hYsjznyqHqqhaUnGm7+Biw0SqJe2yzmFVC7cRKWF1cWL3XYAXkiGhewFamRRpMFy piUg/w62f5u3ZiCDeZfYoQWk43F9Y4sdY3Gc3F9Fbz46eOPxGmYXQ19MILM7MLPPNCyx P7iPvN6iUNA620abUPTAZ/lq2Q7zEQ1ha9pQhwsF1Ao4dkWGeTMKrdYW+VskvwIh/cTx 57qxnHrj5zLA770JixGj/e+t5uJeLVEZKk/qc90c6ds0WGp86GqP+fVpD6M1nhTRXHfk E1wA== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1696166198; x=1696770998; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=rS7jRVcQ6NtK3qXxw1G/jurrddIxGHuAxg9jtCM/lIk=; b=sioa1j/ZyV5IFIsCgWUbx/7uIn0yKQkaGl0escLgNC8JC6BIA+ka4MEcBtEK44GATm CrwiH7utTBPhfpk4jSbyKehQ8VXV61RKRRFLr9YuuSdhAz3WH4nblCTv2d1Jnh+mzr19 r7MIiTTl5Elpz6/8GO/YR9w8K5k4pDOlpES9Szw9fXe9hAIQysZ0GngccUCzwxEG2BEu kn0r5iGwA5fVxf/VQRksml1togmWEaTgsc9KZnfVLG2+VPFb6rIy2sfglKrCS2Ad7cRX tb7JTsWRaXuecF27bOri7gWxfHVyUv6GFGRBhgbVBN8RNAIRWnvCM4ScdKYW1HPbG890 UosQ== X-Gm-Message-State: AOJu0YwVGSSxzoSw4byu9sAz8kf9wwqUmQhtZJdGr8q+pAfPvjMskg/W J2R5N+/0Pm1YWM0HlX1d044= X-Google-Smtp-Source: AGHT+IFqb1idNfPnJMnfnXkq8sZ2ptjGTnbjcG0pTq6iBMVgE2OAoL+xBuTRFQyjqaWDlJrFLy6INw== X-Received: by 2002:a17:906:5397:b0:9ae:4f13:e581 with SMTP id g23-20020a170906539700b009ae4f13e581mr8391092ejo.59.1696166198188; Sun, 01 Oct 2023 06:16:38 -0700 (PDT) Received: from localhost.localdomain ([46.248.82.114]) by smtp.gmail.com with ESMTPSA id lv23-20020a170906bc9700b0099d798a6bb5sm15504272ejb.67.2023.10.01.06.16.37 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Sun, 01 Oct 2023 06:16:37 -0700 (PDT) From: Uros Bizjak To: x86@kernel.org, linux-kernel@vger.kernel.org Cc: Uros Bizjak , Andy Lutomirski , Ingo Molnar , Nadav Amit , Brian Gerst , Denys Vlasenko , "H . Peter Anvin" , Linus Torvalds , Peter Zijlstra , Thomas Gleixner , Borislav Petkov , Josh Poimboeuf Subject: [RFC PATCH 3/4] x86/percpu: Use compiler segment prefix qualifier Date: Sun, 1 Oct 2023 15:14:35 +0200 Message-ID: <20231001131620.112484-4-ubizjak@gmail.com> X-Mailer: git-send-email 2.41.0 In-Reply-To: <20231001131620.112484-1-ubizjak@gmail.com> References: <20231001131620.112484-1-ubizjak@gmail.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" From: Nadav Amit Using a segment prefix qualifier is cleaner than using a segment prefix in the inline assembly, and provides the compiler with more information, telling it that __seg_gs:[addr] is different than [addr] when it analyzes data dependencies. It also enables various optimizations that will be implemented in the next patches. Use segment prefix qualifiers when they are supported. Unfortunately, gcc does not provide a way to remove segment qualifiers, which is needed to use typeof() to create local instances of the per-cpu variable. For this reason, do not use the segment qualifier for per-cpu variables, and do casting using the segment qualifier instead. Uros: Improve compiler support detection and update the patch to the current mainline. Cc: Andy Lutomirski Cc: Ingo Molnar Cc: Nadav Amit Cc: Brian Gerst Cc: Denys Vlasenko Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Borislav Petkov Cc: Josh Poimboeuf Signed-off-by: Nadav Amit Signed-off-by: Uros Bizjak --- arch/x86/include/asm/percpu.h | 68 +++++++++++++++++++++++----------- arch/x86/include/asm/preempt.h | 2 +- 2 files changed, 47 insertions(+), 23 deletions(-) diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h index 20624b80f890..da451202a1b9 100644 --- a/arch/x86/include/asm/percpu.h +++ b/arch/x86/include/asm/percpu.h @@ -28,26 +28,50 @@ #include =20 #ifdef CONFIG_SMP + +#ifdef CONFIG_CC_HAS_NAMED_AS + +#ifdef CONFIG_X86_64 +#define __percpu_seg_override __seg_gs +#else +#define __percpu_seg_override __seg_fs +#endif + +#define __percpu_prefix "" + +#else /* CONFIG_CC_HAS_NAMED_AS */ + +#define __percpu_seg_override #define __percpu_prefix "%%"__stringify(__percpu_seg)":" + +#endif /* CONFIG_CC_HAS_NAMED_AS */ + +#define __force_percpu_prefix "%%"__stringify(__percpu_seg)":" #define __my_cpu_offset this_cpu_read(this_cpu_off) =20 /* * Compared to the generic __my_cpu_offset version, the following * saves one instruction and avoids clobbering a temp register. */ -#define arch_raw_cpu_ptr(ptr) \ -({ \ - unsigned long tcp_ptr__; \ - asm ("add " __percpu_arg(1) ", %0" \ - : "=3Dr" (tcp_ptr__) \ - : "m" (this_cpu_off), "0" (ptr)); \ - (typeof(*(ptr)) __kernel __force *)tcp_ptr__; \ +#define arch_raw_cpu_ptr(ptr) \ +({ \ + unsigned long tcp_ptr__; \ + asm ("add " __percpu_arg(1) ", %0" \ + : "=3Dr" (tcp_ptr__) \ + : "m" (__my_cpu_var(this_cpu_off)), "0" (ptr)); \ + (typeof(*(ptr)) __kernel __force *)tcp_ptr__; \ }) -#else +#else /* CONFIG_SMP */ +#define __percpu_seg_override #define __percpu_prefix "" -#endif +#define __force_percpu_prefix "" +#endif /* CONFIG_SMP */ =20 +#define __my_cpu_type(var) typeof(var) __percpu_seg_override +#define __my_cpu_ptr(ptr) (__my_cpu_type(*ptr) *)(uintptr_t)(ptr) +#define __my_cpu_var(var) (*__my_cpu_ptr(&var)) #define __percpu_arg(x) __percpu_prefix "%" #x +#define __force_percpu_arg(x) __force_percpu_prefix "%" #x =20 /* * Initialized pointers to per-cpu variables needed for the boot @@ -107,14 +131,14 @@ do { \ (void)pto_tmp__; \ } \ asm qual(__pcpu_op2_##size(op, "%[val]", __percpu_arg([var])) \ - : [var] "+m" (_var) \ + : [var] "+m" (__my_cpu_var(_var)) \ : [val] __pcpu_reg_imm_##size(pto_val__)); \ } while (0) =20 #define percpu_unary_op(size, qual, op, _var) \ ({ \ asm qual (__pcpu_op1_##size(op, __percpu_arg([var])) \ - : [var] "+m" (_var)); \ + : [var] "+m" (__my_cpu_var(_var))); \ }) =20 /* @@ -144,14 +168,14 @@ do { \ __pcpu_type_##size pfo_val__; \ asm qual (__pcpu_op2_##size(op, __percpu_arg([var]), "%[val]") \ : [val] __pcpu_reg_##size("=3D", pfo_val__) \ - : [var] "m" (_var)); \ + : [var] "m" (__my_cpu_var(_var))); \ (typeof(_var))(unsigned long) pfo_val__; \ }) =20 #define percpu_stable_op(size, op, _var) \ ({ \ __pcpu_type_##size pfo_val__; \ - asm(__pcpu_op2_##size(op, __percpu_arg(P[var]), "%[val]") \ + asm(__pcpu_op2_##size(op, __force_percpu_arg(P[var]), "%[val]") \ : [val] __pcpu_reg_##size("=3D", pfo_val__) \ : [var] "p" (&(_var))); \ (typeof(_var))(unsigned long) pfo_val__; \ @@ -166,7 +190,7 @@ do { \ asm qual (__pcpu_op2_##size("xadd", "%[tmp]", \ __percpu_arg([var])) \ : [tmp] __pcpu_reg_##size("+", paro_tmp__), \ - [var] "+m" (_var) \ + [var] "+m" (__my_cpu_var(_var)) \ : : "memory"); \ (typeof(_var))(unsigned long) (paro_tmp__ + _val); \ }) @@ -187,7 +211,7 @@ do { \ __percpu_arg([var])) \ "\n\tjnz 1b" \ : [oval] "=3D&a" (pxo_old__), \ - [var] "+m" (_var) \ + [var] "+m" (__my_cpu_var(_var)) \ : [nval] __pcpu_reg_##size(, pxo_new__) \ : "memory"); \ (typeof(_var))(unsigned long) pxo_old__; \ @@ -204,7 +228,7 @@ do { \ asm qual (__pcpu_op2_##size("cmpxchg", "%[nval]", \ __percpu_arg([var])) \ : [oval] "+a" (pco_old__), \ - [var] "+m" (_var) \ + [var] "+m" (__my_cpu_var(_var)) \ : [nval] __pcpu_reg_##size(, pco_new__) \ : "memory"); \ (typeof(_var))(unsigned long) pco_old__; \ @@ -221,7 +245,7 @@ do { \ CC_SET(z) \ : CC_OUT(z) (success), \ [oval] "+a" (pco_old__), \ - [var] "+m" (_var) \ + [var] "+m" (__my_cpu_var(_var)) \ : [nval] __pcpu_reg_##size(, pco_new__) \ : "memory"); \ if (unlikely(!success)) \ @@ -244,7 +268,7 @@ do { \ \ asm qual (ALTERNATIVE("call this_cpu_cmpxchg8b_emu", \ "cmpxchg8b " __percpu_arg([var]), X86_FEATURE_CX8) \ - : [var] "+m" (_var), \ + : [var] "+m" (__my_cpu_var(_var)), \ "+a" (old__.low), \ "+d" (old__.high) \ : "b" (new__.low), \ @@ -276,7 +300,7 @@ do { \ "cmpxchg8b " __percpu_arg([var]), X86_FEATURE_CX8) \ CC_SET(z) \ : CC_OUT(z) (success), \ - [var] "+m" (_var), \ + [var] "+m" (__my_cpu_var(_var)), \ "+a" (old__.low), \ "+d" (old__.high) \ : "b" (new__.low), \ @@ -313,7 +337,7 @@ do { \ \ asm qual (ALTERNATIVE("call this_cpu_cmpxchg16b_emu", \ "cmpxchg16b " __percpu_arg([var]), X86_FEATURE_CX16) \ - : [var] "+m" (_var), \ + : [var] "+m" (__my_cpu_var(_var)), \ "+a" (old__.low), \ "+d" (old__.high) \ : "b" (new__.low), \ @@ -345,7 +369,7 @@ do { \ "cmpxchg16b " __percpu_arg([var]), X86_FEATURE_CX16) \ CC_SET(z) \ : CC_OUT(z) (success), \ - [var] "+m" (_var), \ + [var] "+m" (__my_cpu_var(_var)), \ "+a" (old__.low), \ "+d" (old__.high) \ : "b" (new__.low), \ @@ -494,7 +518,7 @@ static inline bool x86_this_cpu_variable_test_bit(int n= r, asm volatile("btl "__percpu_arg(2)",%1" CC_SET(c) : CC_OUT(c) (oldbit) - : "m" (*(unsigned long __percpu *)addr), "Ir" (nr)); + : "m" (*__my_cpu_ptr((unsigned long __percpu *)(addr))), "Ir" (nr)); =20 return oldbit; } diff --git a/arch/x86/include/asm/preempt.h b/arch/x86/include/asm/preempt.h index 2d13f25b1bd8..e25b95e7cf82 100644 --- a/arch/x86/include/asm/preempt.h +++ b/arch/x86/include/asm/preempt.h @@ -92,7 +92,7 @@ static __always_inline void __preempt_count_sub(int val) */ static __always_inline bool __preempt_count_dec_and_test(void) { - return GEN_UNARY_RMWcc("decl", pcpu_hot.preempt_count, e, + return GEN_UNARY_RMWcc("decl", __my_cpu_var(pcpu_hot.preempt_count), e, __percpu_arg([var])); } =20 --=20 2.41.0 From nobody Wed Jan 7 03:30:13 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 9C096E748ED for ; Sun, 1 Oct 2023 13:16:56 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S234992AbjJANQ4 (ORCPT ); Sun, 1 Oct 2023 09:16:56 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:34612 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S234993AbjJANQn (ORCPT ); Sun, 1 Oct 2023 09:16:43 -0400 Received: from mail-ej1-x62b.google.com (mail-ej1-x62b.google.com [IPv6:2a00:1450:4864:20::62b]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 02E96B8 for ; Sun, 1 Oct 2023 06:16:41 -0700 (PDT) Received: by mail-ej1-x62b.google.com with SMTP id a640c23a62f3a-99de884ad25so2168723266b.3 for ; Sun, 01 Oct 2023 06:16:40 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20230601; t=1696166199; x=1696770999; darn=vger.kernel.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=jISPkvnxHNZ9YOvTMmU4+FmuVTZn4CUvBaU3KXe/pp4=; b=WCGZ4EKLUjEMsGp1rm2NTOlXQi63Rd1R1h+BEmN9sp8ajE80gpZX26xUNEFVpuhQ4L EFVNSbkDETmBxOKuIJDjv/ax9YVi5xuODDzhc1JWwIa3XqFiq+RswySyfzIP4t2vm9aL IyFroKnlLDWSmuol+YkdS8eOPnjSAAiCp3IGf7cKqqSx+Ruf2F7K0aSS9IrW5LUCNTke BEqTkHfaueFZcWtIHOJlvyeqrQgYp6KbFWsUVoJPQV8GFEE/8rprq9dXZRn8cKV02Spd tDTpdbTSXyneZ1hKgH/bPDPHjr4w5NM877i4rSibHSFyaEablIuT3Zs5F0aXsFz0kXwx IFBQ== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1696166199; x=1696770999; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=jISPkvnxHNZ9YOvTMmU4+FmuVTZn4CUvBaU3KXe/pp4=; b=fYYq9Mvpzy/kuVY4P/uH4QJxvbnWFBBo+2lI0yHVoBHKpIV9XFm3nyBVlPH4Jd4FZR 1lN1Ub0+GcevTIHkzUZZGkCNd95JiQrmyhlADJevz1gvn9CYIzxaCH2vM9Uc1yif5TK9 iKGxf1ENXnpF+0GZ7PeMWArvv9Bd0uWQNnElsqHxW4L7tRZAqe0pQo9HUKnT4Z06h2Ph EMeB3HItVRDwnig7uzYUnYDdjr5jVRT4mTOMqZ28WLKxPprPWULpLqgNnCR9ZMu4dUDb Ops2GZilslYBwiEDxc+7INODXQQTq3ycLGZrHFakWRsqF0PI0+/Q5mVRxnWnSf6VZ+vP +d3Q== X-Gm-Message-State: AOJu0YxmQpbk9I84DeG+I2eiKoHEJCLTq8TKqhtqnBQyowNzhNWGfuav uQANtjwqFgt74ts/utJ7/vk= X-Google-Smtp-Source: AGHT+IHqZFgfFLGRezJorS7YXOWe2HwziQUZEK/PuXaRrXoBQYoUKEUf80TYdu9SSW3tQhMeEGWVZQ== X-Received: by 2002:a17:906:58c6:b0:9b2:6ce3:bdf3 with SMTP id e6-20020a17090658c600b009b26ce3bdf3mr10268002ejs.54.1696166199117; Sun, 01 Oct 2023 06:16:39 -0700 (PDT) Received: from localhost.localdomain ([46.248.82.114]) by smtp.gmail.com with ESMTPSA id lv23-20020a170906bc9700b0099d798a6bb5sm15504272ejb.67.2023.10.01.06.16.38 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Sun, 01 Oct 2023 06:16:38 -0700 (PDT) From: Uros Bizjak To: x86@kernel.org, linux-kernel@vger.kernel.org Cc: Uros Bizjak , Andy Lutomirski , Ingo Molnar , Nadav Amit , Brian Gerst , Denys Vlasenko , "H . Peter Anvin" , Linus Torvalds , Peter Zijlstra , Thomas Gleixner , Borislav Petkov , Josh Poimboeuf Subject: [RFC PATCH 4/4] x86/percpu: Use C for percpu read/write accessors Date: Sun, 1 Oct 2023 15:14:36 +0200 Message-ID: <20231001131620.112484-5-ubizjak@gmail.com> X-Mailer: git-send-email 2.41.0 In-Reply-To: <20231001131620.112484-1-ubizjak@gmail.com> References: <20231001131620.112484-1-ubizjak@gmail.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" The percpu code mostly uses inline assembly. Using segment qualifiers allows to use C code instead, which enables the compiler to perform various optimizations (e.g. propagation of memory arguments). Convert percpu read and write accessors to C code, so the memory argument can be propagated to the instruction that uses this argument. Some examples of propagations: a) into sign/zero extensions: 110b54: 65 0f b6 05 00 00 00 movzbl %gs:0x0(%rip),%eax 11ab90: 65 0f b6 15 00 00 00 movzbl %gs:0x0(%rip),%edx 14484a: 65 0f b7 35 00 00 00 movzwl %gs:0x0(%rip),%esi 1a08a9: 65 0f b6 43 78 movzbl %gs:0x78(%rbx),%eax 1a08f9: 65 0f b6 43 78 movzbl %gs:0x78(%rbx),%eax 4ab29a: 65 48 63 15 00 00 00 movslq %gs:0x0(%rip),%rdx 4be128: 65 4c 63 25 00 00 00 movslq %gs:0x0(%rip),%r12 547468: 65 48 63 1f movslq %gs:(%rdi),%rbx 5474e7: 65 48 63 0a movslq %gs:(%rdx),%rcx 54d05d: 65 48 63 0d 00 00 00 movslq %gs:0x0(%rip),%rcx b) into compares: b40804: 65 f7 05 00 00 00 00 testl $0xf0000,%gs:0x0(%rip) b487e8: 65 f7 05 00 00 00 00 testl $0xf0000,%gs:0x0(%rip) b6f14c: 65 f6 05 00 00 00 00 testb $0x1,%gs:0x0(%rip) bac1b8: 65 f6 05 00 00 00 00 testb $0x1,%gs:0x0(%rip) df2244: 65 f7 05 00 00 00 00 testl $0xff00,%gs:0x0(%rip) 9a7517: 65 80 3d 00 00 00 00 cmpb $0x0,%gs:0x0(%rip) b282ba: 65 44 3b 35 00 00 00 cmp %gs:0x0(%rip),%r14d b48f61: 65 66 83 3d 00 00 00 cmpw $0x8,%gs:0x0(%rip) b493fe: 65 80 38 00 cmpb $0x0,%gs:(%rax) b73867: 65 66 83 3d 00 00 00 cmpw $0x8,%gs:0x0(%rip) c) into other insns: 65ec02: 65 0f 44 15 00 00 00 cmove %gs:0x0(%rip),%edx 6c98ac: 65 0f 44 15 00 00 00 cmove %gs:0x0(%rip),%edx 9aafaf: 65 0f 44 15 00 00 00 cmove %gs:0x0(%rip),%edx b45868: 65 0f 48 35 00 00 00 cmovs %gs:0x0(%rip),%esi d276f8: 65 0f 44 15 00 00 00 cmove %gs:0x0(%rip),%edx The above propagations result in the following code size improvements for current mainline kernel (with the default config), compiled with gcc (GCC) 12.3.1 20230508 (Red Hat 12.3.1-1) text data bss dec hex filename 25508862 4386540 808388 30703790 1d480ae vmlinux-vanilla.o 25500922 4386532 808388 30695842 1d461a2 vmlinux-new.o The conversion of other read-modify-write instructions does not bring us any benefits, the compiler has some problems when constructing RMW instructions from the generic code and easily misses some opportunities. Cc: Andy Lutomirski Cc: Ingo Molnar Cc: Nadav Amit Cc: Brian Gerst Cc: Denys Vlasenko Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Borislav Petkov Cc: Josh Poimboeuf Co-developed-by: Nadav Amit Signed-off-by: Nadav Amit Signed-off-by: Uros Bizjak --- arch/x86/include/asm/percpu.h | 65 +++++++++++++++++++++++++++++------ 1 file changed, 54 insertions(+), 11 deletions(-) diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h index da451202a1b9..60ea7755c0fe 100644 --- a/arch/x86/include/asm/percpu.h +++ b/arch/x86/include/asm/percpu.h @@ -400,13 +400,66 @@ do { \ #define this_cpu_read_stable_8(pcp) percpu_stable_op(8, "mov", pcp) #define this_cpu_read_stable(pcp) __pcpu_size_call_return(this_cpu_read_st= able_, pcp) =20 +#ifdef CONFIG_USE_X86_SEG_SUPPORT + +#define __raw_cpu_read(qual, pcp) \ +({ \ + *(qual __my_cpu_type(pcp) *)__my_cpu_ptr(&(pcp)); \ +}) + +#define __raw_cpu_write(qual, pcp, val) \ +do { \ + *(qual __my_cpu_type(pcp) *)__my_cpu_ptr(&(pcp)) =3D (val); \ +} while (0) + +#define raw_cpu_read_1(pcp) __raw_cpu_read(, pcp) +#define raw_cpu_read_2(pcp) __raw_cpu_read(, pcp) +#define raw_cpu_read_4(pcp) __raw_cpu_read(, pcp) +#define raw_cpu_write_1(pcp, val) __raw_cpu_write(, pcp, val) +#define raw_cpu_write_2(pcp, val) __raw_cpu_write(, pcp, val) +#define raw_cpu_write_4(pcp, val) __raw_cpu_write(, pcp, val) + +#define this_cpu_read_1(pcp) __raw_cpu_read(volatile, pcp) +#define this_cpu_read_2(pcp) __raw_cpu_read(volatile, pcp) +#define this_cpu_read_4(pcp) __raw_cpu_read(volatile, pcp) +#define this_cpu_write_1(pcp, val) __raw_cpu_write(volatile, pcp, val) +#define this_cpu_write_2(pcp, val) __raw_cpu_write(volatile, pcp, val) +#define this_cpu_write_4(pcp, val) __raw_cpu_write(volatile, pcp, val) + +#ifdef CONFIG_X86_64 +#define raw_cpu_read_8(pcp) __raw_cpu_read(, pcp) +#define raw_cpu_write_8(pcp, val) __raw_cpu_write(, pcp, val) + +#define this_cpu_read_8(pcp) __raw_cpu_read(volatile, pcp) +#define this_cpu_write_8(pcp, val) __raw_cpu_write(volatile, pcp, val) +#endif + +#else /* CONFIG_USE_X86_SEG_SUPPORT */ + #define raw_cpu_read_1(pcp) percpu_from_op(1, , "mov", pcp) #define raw_cpu_read_2(pcp) percpu_from_op(2, , "mov", pcp) #define raw_cpu_read_4(pcp) percpu_from_op(4, , "mov", pcp) - #define raw_cpu_write_1(pcp, val) percpu_to_op(1, , "mov", (pcp), val) #define raw_cpu_write_2(pcp, val) percpu_to_op(2, , "mov", (pcp), val) #define raw_cpu_write_4(pcp, val) percpu_to_op(4, , "mov", (pcp), val) + +#define this_cpu_read_1(pcp) percpu_from_op(1, volatile, "mov", pcp) +#define this_cpu_read_2(pcp) percpu_from_op(2, volatile, "mov", pcp) +#define this_cpu_read_4(pcp) percpu_from_op(4, volatile, "mov", pcp) +#define this_cpu_write_1(pcp, val) percpu_to_op(1, volatile, "mov", (pcp),= val) +#define this_cpu_write_2(pcp, val) percpu_to_op(2, volatile, "mov", (pcp),= val) +#define this_cpu_write_4(pcp, val) percpu_to_op(4, volatile, "mov", (pcp),= val) + +#ifdef CONFIG_X86_64 +#define raw_cpu_read_8(pcp) percpu_from_op(8, , "mov", pcp) +#define raw_cpu_write_8(pcp, val) percpu_to_op(8, , "mov", (pcp), val) + +#define this_cpu_read_8(pcp) percpu_from_op(8, volatile, "mov", pcp) +#define this_cpu_write_8(pcp, val) percpu_to_op(8, volatile, "mov", (pcp),= val) +#endif + +#endif /* CONFIG_USE_X86_SEG_SUPPORT */ + #define raw_cpu_add_1(pcp, val) percpu_add_op(1, , (pcp), val) #define raw_cpu_add_2(pcp, val) percpu_add_op(2, , (pcp), val) #define raw_cpu_add_4(pcp, val) percpu_add_op(4, , (pcp), val) @@ -432,12 +485,6 @@ do { \ #define raw_cpu_xchg_2(pcp, val) raw_percpu_xchg_op(pcp, val) #define raw_cpu_xchg_4(pcp, val) raw_percpu_xchg_op(pcp, val) =20 -#define this_cpu_read_1(pcp) percpu_from_op(1, volatile, "mov", pcp) -#define this_cpu_read_2(pcp) percpu_from_op(2, volatile, "mov", pcp) -#define this_cpu_read_4(pcp) percpu_from_op(4, volatile, "mov", pcp) -#define this_cpu_write_1(pcp, val) percpu_to_op(1, volatile, "mov", (pcp),= val) -#define this_cpu_write_2(pcp, val) percpu_to_op(2, volatile, "mov", (pcp),= val) -#define this_cpu_write_4(pcp, val) percpu_to_op(4, volatile, "mov", (pcp),= val) #define this_cpu_add_1(pcp, val) percpu_add_op(1, volatile, (pcp), val) #define this_cpu_add_2(pcp, val) percpu_add_op(2, volatile, (pcp), val) #define this_cpu_add_4(pcp, val) percpu_add_op(4, volatile, (pcp), val) @@ -476,8 +523,6 @@ do { \ * 32 bit must fall back to generic operations. */ #ifdef CONFIG_X86_64 -#define raw_cpu_read_8(pcp) percpu_from_op(8, , "mov", pcp) -#define raw_cpu_write_8(pcp, val) percpu_to_op(8, , "mov", (pcp), val) #define raw_cpu_add_8(pcp, val) percpu_add_op(8, , (pcp), val) #define raw_cpu_and_8(pcp, val) percpu_to_op(8, , "and", (pcp), val) #define raw_cpu_or_8(pcp, val) percpu_to_op(8, , "or", (pcp), val) @@ -486,8 +531,6 @@ do { \ #define raw_cpu_cmpxchg_8(pcp, oval, nval) percpu_cmpxchg_op(8, , pcp, ova= l, nval) #define raw_cpu_try_cmpxchg_8(pcp, ovalp, nval) percpu_try_cmpxchg_op(8, ,= pcp, ovalp, nval) =20 -#define this_cpu_read_8(pcp) percpu_from_op(8, volatile, "mov", pcp) -#define this_cpu_write_8(pcp, val) percpu_to_op(8, volatile, "mov", (pcp)= , val) #define this_cpu_add_8(pcp, val) percpu_add_op(8, volatile, (pcp), val) #define this_cpu_and_8(pcp, val) percpu_to_op(8, volatile, "and", (pcp), = val) #define this_cpu_or_8(pcp, val) percpu_to_op(8, volatile, "or", (pcp), v= al) --=20 2.41.0