From nobody Sun Jun 14 21:38:11 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 0352AC4332F for ; Wed, 11 May 2022 07:27:12 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S242724AbiEKH1K (ORCPT ); Wed, 11 May 2022 03:27:10 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:51658 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S242910AbiEKH1F (ORCPT ); Wed, 11 May 2022 03:27:05 -0400 Received: from mail-pj1-x1035.google.com (mail-pj1-x1035.google.com [IPv6:2607:f8b0:4864:20::1035]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id BCC6E3BF90 for ; Wed, 11 May 2022 00:27:03 -0700 (PDT) Received: by mail-pj1-x1035.google.com with SMTP id qe3-20020a17090b4f8300b001dc24e4da73so3215329pjb.1 for ; Wed, 11 May 2022 00:27:03 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20210112; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=cjE84FymrEK7bxD5XFoGhOeZrQEQrNAiUbfRS8xIEcU=; b=klbQJoWUooPPrElfPbe90DkbxVaCHOjQgHPdPkWZ9EL5GUtvOVe6OiRplW16LY4ZY5 k8MgfzvLE5cVP4mK9AdasxgHCTaqOUw/FvRPM7VpBFMUygJKJVnPWeCx+sqTx4DSyR/j 8A8/xgSnuVKWSqsp9caBimzpCSkf+IaINHQiUbMs+a67mGe/+A1Re38gsvN6Ds5ojn7A W8P3Pgz+HnapBe7IoINiyzlspoyskQng/glCIzer8vIwIOM0o0E7P+pRHSvgqMCJpbZx 9bliRghw2MYr/GLeB+QvWR0C5QQKxPbBXCsbarHNF+aJj100F9bCh7YiAKauwvOL4SW5 uqwQ== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=cjE84FymrEK7bxD5XFoGhOeZrQEQrNAiUbfRS8xIEcU=; b=Gm7kXrcQnUaqiDX4HJ9YUOHf2hQXpyZCgSOyuhy/lV9z/F/jYV9OfgyWzJNbcne9W4 C/25YA6VVSTHO1Wxvt+b4LUtwKrD8uC/9rpzkkdjmFRfFEW1FwGt9HmIrzmzD39pXa7f IzAtSAvp834a1szSCCxCvXtHVk1/thzD8Yosbrl41bmLcUsPuj6uFSwLJXwMDNdq0M10 FNuwZ/Nm9L7N++qyjDrztSLufC1qmMQnY5ObhdWpf0VixnQYqBPokANMTTGKxP0Pgr3I rDLM0txXDbCZIagJfnXqn33u6OVyjYm6Qht80CLk2/L51AkLNRSH4JYzwvtGF0FcmVvO FHjw== X-Gm-Message-State: AOAM530UIb8r8d4l6iuMLNMR7CZWL5yM06a6DnWqM3vP/g4geRRfPQQ7 X8icROLK/CewIzjNF7SOsbdbqdD6gik= X-Google-Smtp-Source: ABdhPJymHzix4/M1wBKJtK96BCPSuwsqFL3j3N0YLQI2kCeEPCOJtMNL++Qz+8/4vKgW26FlWsvwPQ== X-Received: by 2002:a17:90b:3a86:b0:1dc:2343:2429 with SMTP id om6-20020a17090b3a8600b001dc23432429mr3918001pjb.206.1652254022944; Wed, 11 May 2022 00:27:02 -0700 (PDT) Received: from localhost ([47.251.4.198]) by smtp.gmail.com with ESMTPSA id rm10-20020a17090b3eca00b001cd4989fee1sm3519825pjb.45.2022.05.11.00.27.01 (version=TLS1_2 cipher=ECDHE-ECDSA-AES128-GCM-SHA256 bits=128/128); Wed, 11 May 2022 00:27:02 -0700 (PDT) From: Lai Jiangshan To: linux-kernel@vger.kernel.org Cc: Borislav Petkov , Peter Zijlstra , Josh Poimboeuf , Andy Lutomirski , Thomas Gleixner , x86@kernel.org, Lai Jiangshan , Ingo Molnar , Dave Hansen , "H. Peter Anvin" , Kuppuswamy Sathyanarayanan , Joerg Roedel , "Kirill A. Shutemov" , Miguel Ojeda , Kees Cook , Nathan Chancellor , Andrew Morton , Alexei Starovoitov , Marco Elver , Hao Luo , Nick Desaulniers , Rasmus Villemoes Subject: [PATCH 1/7] x86/entry: Introduce __entry_text for entry code written in C Date: Wed, 11 May 2022 15:27:41 +0800 Message-Id: <20220511072747.3960-2-jiangshanlai@gmail.com> X-Mailer: git-send-email 2.19.1.6.gb485710b In-Reply-To: <20220511072747.3960-1-jiangshanlai@gmail.com> References: <20220511072747.3960-1-jiangshanlai@gmail.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" From: Lai Jiangshan Some entry code will be implemented in C files. Introduce __entry_text to set them in .entry.text section. The new __entry_text disables instrumentation like noinstr, so __noinstr_section() is added for noinstr and the new __entry_text. Note, entry code can not access to %gs before the %gs base is switched to kernel %gs base, so stack protector can not be used on the C entry code. But __entry_text doesn't disable stack protector since some compilers might not support function level granular attribute to disable stack protector. It will be disabled in C file level. Cc: Borislav Petkov Reviewed-by: Miguel Ojeda Reviewed-by: Kees Cook Suggested-by: Nick Desaulniers Suggested-by: Peter Zijlstra Signed-off-by: Lai Jiangshan --- arch/x86/include/asm/idtentry.h | 3 +++ include/linux/compiler_types.h | 8 +++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/arch/x86/include/asm/idtentry.h b/arch/x86/include/asm/idtentr= y.h index 72184b0b2219..acc4c99f801c 100644 --- a/arch/x86/include/asm/idtentry.h +++ b/arch/x86/include/asm/idtentry.h @@ -13,6 +13,9 @@ =20 #include =20 +/* Entry code written in C. */ +#define __entry_text __noinstr_section(".entry.text") + /** * DECLARE_IDTENTRY - Declare functions for simple IDT entry points * No error code pushed by hardware diff --git a/include/linux/compiler_types.h b/include/linux/compiler_types.h index 1c2c33ae1b37..8c7e81efe9bf 100644 --- a/include/linux/compiler_types.h +++ b/include/linux/compiler_types.h @@ -225,9 +225,11 @@ struct ftrace_likely_data { #endif =20 /* Section for code which can't be instrumented at all */ -#define noinstr \ - noinline notrace __attribute((__section__(".noinstr.text"))) \ - __no_kcsan __no_sanitize_address __no_profile __no_sanitize_coverage +#define __noinstr_section(section) \ + noinline notrace __section(section) __no_profile \ + __no_kcsan __no_sanitize_address __no_sanitize_coverage + +#define noinstr __noinstr_section(".noinstr.text") =20 #endif /* __KERNEL__ */ =20 --=20 2.19.1.6.gb485710b From nobody Sun Jun 14 21:38:11 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 23E78C433EF for ; Wed, 11 May 2022 07:27:31 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S242960AbiEKH11 (ORCPT ); Wed, 11 May 2022 03:27:27 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:52468 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S242940AbiEKH1O (ORCPT ); Wed, 11 May 2022 03:27:14 -0400 Received: from mail-pj1-x1032.google.com (mail-pj1-x1032.google.com [IPv6:2607:f8b0:4864:20::1032]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 6EB783CA5A for ; Wed, 11 May 2022 00:27:12 -0700 (PDT) Received: by mail-pj1-x1032.google.com with SMTP id cq17-20020a17090af99100b001dc0386cd8fso1287779pjb.5 for ; Wed, 11 May 2022 00:27:12 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20210112; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=EK03M+puSu84y4LA2tpR63LGf6pYpcixQrSXDU5WY9M=; b=XAqq6Vy677zzwsOLR9heaBfHSiYJe2gOzNQQ8+f8Jzs3ePCqpPjy3DXDNGUg/D3E+y 8AkX4EnxmxrMT9D5Rb9TTqaNrb97ZTLl+4pDgOvrYEes8fqaLI7LjIUdjnFXMOHuXRGJ d8ANWQyWBnbZk7uEGOZnIVHP+/yqwIfFwIvS7vtm0lbGNPWgYb/uaOj1EIPWi3cTEmrc 0B5BQs6Ky91uwVf81cfPp6wSXFCzUUBxbnt/4mfG2z519sABIlo04X2OA3iXGchmCANl T7FDPM1OazmIwU3GBHv2pRV7EowhxsW7ybaneHES2aGhu69zuZR3KWVzNhPDVXKU294z Bgdg== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=EK03M+puSu84y4LA2tpR63LGf6pYpcixQrSXDU5WY9M=; b=pSQQb/og9xo4UD09oDUB7pHSUt940iSjjrrNsJFqjT9/W+Jcy2A3UdGLS7Tw0n577O /j4iz+6h1dIrz7RH6LaFAsOHnG2C2OEC93EMnmi9twkrX9cFJv25ETSwlCUrjjvvcR2l swqkHjKIL+9lFbhdP7dFOpfN2srmeVbgkbeZVR+JkrYGVbC54AEMhtadRM4fRfuuRyLK 8zZgtP9hlSWnJol/CPQPEU9NvGsYPreBekVnVAl16MsOayTHZNCZuYPSkBujUa/gubeq hivFD84hsHA6KQkdjwcr+65uue0408iM2omhz6SiehXArai/soWbrG05vgI44wgkRGaI wt/g== X-Gm-Message-State: AOAM5319AXJc7YdXitYLrSBBTFtL6q0WF9BWGDWkVEgrWYMgng+dFr2r oGqubkZdeieXVBNA++RjiE/P4hnBSJw= X-Google-Smtp-Source: ABdhPJy+S8UjM/NrSA0lu4NnbybJQGh/8pHGRrtf0KqkDZcoHERomd3uPoA8dO0uqFfyRkozbrPpcw== X-Received: by 2002:a17:902:d58b:b0:15f:16f9:abe4 with SMTP id k11-20020a170902d58b00b0015f16f9abe4mr11352905plh.83.1652254031691; Wed, 11 May 2022 00:27:11 -0700 (PDT) Received: from localhost ([47.251.4.198]) by smtp.gmail.com with ESMTPSA id r13-20020a170902ea4d00b0015e8d4eb1e5sm962306plg.47.2022.05.11.00.27.10 (version=TLS1_2 cipher=ECDHE-ECDSA-AES128-GCM-SHA256 bits=128/128); Wed, 11 May 2022 00:27:11 -0700 (PDT) From: Lai Jiangshan To: linux-kernel@vger.kernel.org Cc: Borislav Petkov , Peter Zijlstra , Josh Poimboeuf , Andy Lutomirski , Thomas Gleixner , x86@kernel.org, Lai Jiangshan , Ingo Molnar , Dave Hansen , "H. Peter Anvin" , Andrew Morton , "Aneesh Kumar K.V" , Sebastian Andrzej Siewior , "Kirill A. Shutemov" , Pasha Tatashin Subject: [PATCH 2/7] x86/entry: Move PTI_USER_* to arch/x86/include/asm/processor-flags.h Date: Wed, 11 May 2022 15:27:42 +0800 Message-Id: <20220511072747.3960-3-jiangshanlai@gmail.com> X-Mailer: git-send-email 2.19.1.6.gb485710b In-Reply-To: <20220511072747.3960-1-jiangshanlai@gmail.com> References: <20220511072747.3960-1-jiangshanlai@gmail.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" From: Lai Jiangshan These constants will be also used in C file. Move them to arch/x86/include/asm/processor-flags.h which already has a kin X86_CR3_PTI_PCID_USER_BIT defined in it. Remove PTI_PGTABLE_SWITCH_BIT and replace it with PTI_USER_PGTABLE_BIT since they are the same in meaning and value. Remove kernel_to_user_p4dp() and user_to_kernel_p4dp() since they are using the removed PTI_PGTABLE_SWITCH_BIT and they have no caller. Signed-off-by: Lai Jiangshan --- arch/x86/entry/calling.h | 10 ---------- arch/x86/include/asm/pgtable.h | 23 +++-------------------- arch/x86/include/asm/processor-flags.h | 15 +++++++++++++++ 3 files changed, 18 insertions(+), 30 deletions(-) diff --git a/arch/x86/entry/calling.h b/arch/x86/entry/calling.h index a97cc78ecb92..f0f60810aee7 100644 --- a/arch/x86/entry/calling.h +++ b/arch/x86/entry/calling.h @@ -142,16 +142,6 @@ For 32-bit we have the following conventions - kernel = is built with =20 #ifdef CONFIG_PAGE_TABLE_ISOLATION =20 -/* - * PAGE_TABLE_ISOLATION PGDs are 8k. Flip bit 12 to switch between the two - * halves: - */ -#define PTI_USER_PGTABLE_BIT PAGE_SHIFT -#define PTI_USER_PGTABLE_MASK (1 << PTI_USER_PGTABLE_BIT) -#define PTI_USER_PCID_BIT X86_CR3_PTI_PCID_USER_BIT -#define PTI_USER_PCID_MASK (1 << PTI_USER_PCID_BIT) -#define PTI_USER_PGTABLE_AND_PCID_MASK (PTI_USER_PCID_MASK | PTI_USER_PGT= ABLE_MASK) - .macro SET_NOFLUSH_BIT reg:req bts $X86_CR3_PCID_NOFLUSH_BIT, \reg .endm diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h index 62ab07e24aef..19095f7a0840 100644 --- a/arch/x86/include/asm/pgtable.h +++ b/arch/x86/include/asm/pgtable.h @@ -5,6 +5,7 @@ #include #include #include +#include =20 /* * Macro to mark a page protection value as UC- @@ -1191,14 +1192,6 @@ static inline bool pgdp_maps_userspace(void *__ptr) static inline int pgd_large(pgd_t pgd) { return 0; } =20 #ifdef CONFIG_PAGE_TABLE_ISOLATION -/* - * All top-level PAGE_TABLE_ISOLATION page tables are order-1 pages - * (8k-aligned and 8k in size). The kernel one is at the beginning 4k and - * the user one is in the last 4k. To switch between them, you - * just need to flip the 12th bit in their addresses. - */ -#define PTI_PGTABLE_SWITCH_BIT PAGE_SHIFT - /* * This generates better code than the inline assembly in * __set_bit(). @@ -1220,22 +1213,12 @@ static inline void *ptr_clear_bit(void *ptr, int bi= t) =20 static inline pgd_t *kernel_to_user_pgdp(pgd_t *pgdp) { - return ptr_set_bit(pgdp, PTI_PGTABLE_SWITCH_BIT); + return ptr_set_bit(pgdp, PTI_USER_PGTABLE_BIT); } =20 static inline pgd_t *user_to_kernel_pgdp(pgd_t *pgdp) { - return ptr_clear_bit(pgdp, PTI_PGTABLE_SWITCH_BIT); -} - -static inline p4d_t *kernel_to_user_p4dp(p4d_t *p4dp) -{ - return ptr_set_bit(p4dp, PTI_PGTABLE_SWITCH_BIT); -} - -static inline p4d_t *user_to_kernel_p4dp(p4d_t *p4dp) -{ - return ptr_clear_bit(p4dp, PTI_PGTABLE_SWITCH_BIT); + return ptr_clear_bit(pgdp, PTI_USER_PGTABLE_BIT); } #endif /* CONFIG_PAGE_TABLE_ISOLATION */ =20 diff --git a/arch/x86/include/asm/processor-flags.h b/arch/x86/include/asm/= processor-flags.h index 02c2cbda4a74..4dd2fbbc861a 100644 --- a/arch/x86/include/asm/processor-flags.h +++ b/arch/x86/include/asm/processor-flags.h @@ -4,6 +4,7 @@ =20 #include #include +#include =20 #ifdef CONFIG_VM86 #define X86_VM_MASK X86_EFLAGS_VM @@ -50,7 +51,21 @@ #endif =20 #ifdef CONFIG_PAGE_TABLE_ISOLATION + # define X86_CR3_PTI_PCID_USER_BIT 11 + +#ifdef CONFIG_X86_64 +/* + * PAGE_TABLE_ISOLATION PGDs are 8k. Flip bit 12 to switch between the two + * halves: + */ +#define PTI_USER_PGTABLE_BIT PAGE_SHIFT +#define PTI_USER_PGTABLE_MASK (1 << PTI_USER_PGTABLE_BIT) +#define PTI_USER_PCID_BIT X86_CR3_PTI_PCID_USER_BIT +#define PTI_USER_PCID_MASK (1 << PTI_USER_PCID_BIT) +#define PTI_USER_PGTABLE_AND_PCID_MASK (PTI_USER_PCID_MASK | PTI_USER_PGT= ABLE_MASK) +#endif + #endif =20 #endif /* _ASM_X86_PROCESSOR_FLAGS_H */ --=20 2.19.1.6.gb485710b From nobody Sun Jun 14 21:38:11 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 389BCC433EF for ; Wed, 11 May 2022 07:27:40 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S242973AbiEKH1d (ORCPT ); Wed, 11 May 2022 03:27:33 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:53198 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S242956AbiEKH1W (ORCPT ); Wed, 11 May 2022 03:27:22 -0400 Received: from mail-pf1-x42a.google.com (mail-pf1-x42a.google.com [IPv6:2607:f8b0:4864:20::42a]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id BE1623DA42 for ; Wed, 11 May 2022 00:27:16 -0700 (PDT) Received: by mail-pf1-x42a.google.com with SMTP id bo5so1257067pfb.4 for ; Wed, 11 May 2022 00:27:16 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20210112; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=QPVmCWUy/pAelwn42xkgA5+/zRNQTu28RGMt2EDlrN0=; b=PYKMhrwupiX1DKVFHB67nL6xi45Gejxb9M3oWtYgWDpMTbNtlZhy3M9WBRoEc+gFU+ y9swxYtgp6J09vJYhWRcCAviWIa6waO2YY94BGwT73H0BNP4XDHzroirDB44Fy9c7kXg mMXIMrDnYLuq8t2U3f9KuAFPmy7fOA362VFVUk18+JpAna3cjgn3lrrAVH3vvraENPpb tr9zIk0WGFBenjThfNM0wy8/ZGmtNVRgTKaaYfVrSjuTI0WD7TMqFFecdA2jVnlF2K7c ZdiFVif8AaVTB92CAq46ZTicaV94F5HiwO2W2CJGMXukbWl4hcgOOhDiSqSTaIMcZkt2 IYoQ== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=QPVmCWUy/pAelwn42xkgA5+/zRNQTu28RGMt2EDlrN0=; b=XkuEHWIK1NN6KQXkVp8T5vnuVSZ3Gs/Vtr6zc0OyBe0Bj4U4CUuq+JCQ6Ifn/iDelU 7F/tLMd41zvr47gYzur25QjD054DuDSKsmLREjI2Nljybw0QHupDq03XhUxGMK2L3iJ0 /GuJ1VHR506jXSZzGTUKwo5KmHN7d731RMn21r4GFL4YXpWA7b+nh+m2Hku3kUUe97sE IbZM00sEFlFbhJ+ZVwmB0rROFlielZywgClDs5M6gLR02eCYdK0FhD5NHnBkj3+nPno7 o/NlMahfKTn/MKXBa/2wVkiZ3x3EO3XH2+DCmV4+H4nCh29ssaR6oTrIh28d1KRucUqW wCqg== X-Gm-Message-State: AOAM531xs2l4b6LmCgfLqCJIUBjcuKjcI+RGUlK8paouYb+XVKlEQA90 cQTq4Qcir4VGd/ctRXNsKeSBYuhVmGw= X-Google-Smtp-Source: ABdhPJzN3XtJyFsowFOS2kEc3mCV/5GAixyocAPpmL7YzPDA3RBzWStn46ECX90OlktJImbSkeEu/g== X-Received: by 2002:a05:6a00:14ce:b0:50f:ac00:2a8b with SMTP id w14-20020a056a0014ce00b0050fac002a8bmr23519267pfu.36.1652254036178; Wed, 11 May 2022 00:27:16 -0700 (PDT) Received: from localhost ([47.251.4.198]) by smtp.gmail.com with ESMTPSA id s125-20020a637783000000b003c14af5062dsm932739pgc.69.2022.05.11.00.27.15 (version=TLS1_2 cipher=ECDHE-ECDSA-AES128-GCM-SHA256 bits=128/128); Wed, 11 May 2022 00:27:15 -0700 (PDT) From: Lai Jiangshan To: linux-kernel@vger.kernel.org Cc: Borislav Petkov , Peter Zijlstra , Josh Poimboeuf , Andy Lutomirski , Thomas Gleixner , x86@kernel.org, Lai Jiangshan , Ingo Molnar , Dave Hansen , "H. Peter Anvin" , Kees Cook , Brian Gerst Subject: [PATCH 3/7] x86: Mark __native_read_cr3() & native_write_cr3() as __always_inline Date: Wed, 11 May 2022 15:27:43 +0800 Message-Id: <20220511072747.3960-4-jiangshanlai@gmail.com> X-Mailer: git-send-email 2.19.1.6.gb485710b In-Reply-To: <20220511072747.3960-1-jiangshanlai@gmail.com> References: <20220511072747.3960-1-jiangshanlai@gmail.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" From: Lai Jiangshan Mark __native_read_cr3() & native_write_cr3() as __always_inline to ensure they are not instrumentable and in the .entry.text section if the caller is not instrumentable and in the .entry.text section. It prepares for __native_read_cr3() and native_write_cr3() to be used in the C entry code for handling KPTI. Signed-off-by: Lai Jiangshan --- arch/x86/include/asm/special_insns.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/include/asm/special_insns.h b/arch/x86/include/asm/sp= ecial_insns.h index 45b18eb94fa1..dbaee50abb3c 100644 --- a/arch/x86/include/asm/special_insns.h +++ b/arch/x86/include/asm/special_insns.h @@ -42,14 +42,14 @@ static __always_inline void native_write_cr2(unsigned l= ong val) asm volatile("mov %0,%%cr2": : "r" (val) : "memory"); } =20 -static inline unsigned long __native_read_cr3(void) +static __always_inline unsigned long __native_read_cr3(void) { unsigned long val; asm volatile("mov %%cr3,%0\n\t" : "=3Dr" (val) : __FORCE_ORDER); return val; } =20 -static inline void native_write_cr3(unsigned long val) +static __always_inline void native_write_cr3(unsigned long val) { asm volatile("mov %0,%%cr3": : "r" (val) : "memory"); } --=20 2.19.1.6.gb485710b From nobody Sun Jun 14 21:38:11 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id CC8F9C433F5 for ; Wed, 11 May 2022 07:27:58 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S242922AbiEKH14 (ORCPT ); Wed, 11 May 2022 03:27:56 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:55346 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S232073AbiEKH1w (ORCPT ); Wed, 11 May 2022 03:27:52 -0400 Received: from mail-pg1-x52e.google.com (mail-pg1-x52e.google.com [IPv6:2607:f8b0:4864:20::52e]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 911473AA7E for ; Wed, 11 May 2022 00:27:50 -0700 (PDT) Received: by mail-pg1-x52e.google.com with SMTP id 31so1050255pgp.8 for ; Wed, 11 May 2022 00:27:50 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20210112; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=TdTLtLMgpiAme8Im7kZEk/uGXhOybjozcq+A7lmpHwg=; b=pvxV6K/xSiTVa9ouO64Cucp0uIhtHSTkom5BcnoiAqnJbB6ZJM9EK6bsCZya40wO9u yIyF6ffIgXHGT8WhsRTBuHt1Q+wl6TRs+G84OsLjn+NYynhjPwYTShUNp4caTwUPfEX2 J2VnIsA/OIpu4Xx9ncex06m0trvedxBr3Eh3xjpHqmyDsVFFaAJoZxBiq5N6w4ttoEpB Z5KVmG1nQnJ4L/XmIMnHTQSywz9rVsCbio+A9JF98Je/VWiknZZCnYbt1FKExyikMmhZ 5en1vydKM8h4FAXvfU2qO7luR05jrQ/JModUp4yPOwXTDQXDnkO16XkgqFkdyQGKCdrq XoQQ== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=TdTLtLMgpiAme8Im7kZEk/uGXhOybjozcq+A7lmpHwg=; b=M4L7wc/VbReRj4+EZXq+JSanhM9GCW2xdq45wFWpmay115ZOUItbISU701d2D+1xSV +5nHT6V2YhAlhv1i8d2yocgLjeQNzqiLSVT2UnVQsuukplu6lr1NktUL0iwOVbg21z6X sjJR88EjEHJ/NaQljUTlUQWZf1X4m2CaAvb8Ju5HGuNBkx12Do8bnTkJfvgH+DBFdhPM MitiN2h9WpiVncZW0TS4M8x1vnmzPDR+Mb1gt0w7YN7+wqNxQha0kdKYLM5ZRfsc/LdZ cjQ2MFMA78ONLqWrFgNycEEcKTkTmoIdTBcKNe7qlsnvdqP6bV77+Qd1zKpHm9N2XNR/ 5ZJA== X-Gm-Message-State: AOAM531IV37LdZFC+DX+Cy8cx7YBiPvQvNpfZkldErn/2eUgo98P7+CD EcAJ1MoyqBYvJiAGJe9AG0Juf4RlCtg= X-Google-Smtp-Source: ABdhPJxhuWB4xSn8IVNoHnxsY9u1CPBfaXXhRAQrRzX+MginwFsfHYkQKLdClr1S8jOscXPq0b9Qzw== X-Received: by 2002:a65:6e47:0:b0:3c6:7d47:ddc8 with SMTP id be7-20020a656e47000000b003c67d47ddc8mr15078467pgb.157.1652254069888; Wed, 11 May 2022 00:27:49 -0700 (PDT) Received: from localhost ([47.251.4.198]) by smtp.gmail.com with ESMTPSA id h17-20020a635311000000b003c14af505f8sm982033pgb.16.2022.05.11.00.27.49 (version=TLS1_2 cipher=ECDHE-ECDSA-AES128-GCM-SHA256 bits=128/128); Wed, 11 May 2022 00:27:49 -0700 (PDT) From: Lai Jiangshan To: linux-kernel@vger.kernel.org Cc: Borislav Petkov , Peter Zijlstra , Josh Poimboeuf , Andy Lutomirski , Thomas Gleixner , x86@kernel.org, Lai Jiangshan , Ingo Molnar , Dave Hansen , "H. Peter Anvin" Subject: [PATCH 4/7] x86/entry: Add arch/x86/entry/entry64.c for C entry code Date: Wed, 11 May 2022 15:27:44 +0800 Message-Id: <20220511072747.3960-5-jiangshanlai@gmail.com> X-Mailer: git-send-email 2.19.1.6.gb485710b In-Reply-To: <20220511072747.3960-1-jiangshanlai@gmail.com> References: <20220511072747.3960-1-jiangshanlai@gmail.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" From: Lai Jiangshan Add a C file "entry64.c" to deposit C entry code for traps and faults which will be as the same logic as the existing ASM code in entry_64.S. The file is as low level as entry_64.S and its code can be running in the environments that the GS base is a user controlled value, or the CR3 is the KPTI user CR3 or both. All the code in this file should not be instrumentable. Many instrument facilities can be disabled by per-function attributes which are included in __noinstr_section. But stack-protector can not be disabled function- granularly by some compliers. So stack-protector is disabled for the whole file in Makefile. Suggested-by: Joerg Roedel Signed-off-by: Lai Jiangshan --- arch/x86/entry/Makefile | 3 ++- arch/x86/entry/entry64.c | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 arch/x86/entry/entry64.c diff --git a/arch/x86/entry/Makefile b/arch/x86/entry/Makefile index 7fec5dcf6438..792f7009ff32 100644 --- a/arch/x86/entry/Makefile +++ b/arch/x86/entry/Makefile @@ -10,13 +10,14 @@ KCOV_INSTRUMENT :=3D n CFLAGS_REMOVE_common.o =3D $(CC_FLAGS_FTRACE) =20 CFLAGS_common.o +=3D -fno-stack-protector +CFLAGS_entry64.o +=3D -fno-stack-protector =20 obj-y :=3D entry_$(BITS).o thunk_$(BITS).o syscall_$(BITS).o obj-y +=3D common.o +obj-$(CONFIG_X86_64) +=3D entry64.o =20 obj-y +=3D vdso/ obj-y +=3D vsyscall/ =20 obj-$(CONFIG_IA32_EMULATION) +=3D entry_64_compat.o syscall_32.o obj-$(CONFIG_X86_X32_ABI) +=3D syscall_x32.o - diff --git a/arch/x86/entry/entry64.c b/arch/x86/entry/entry64.c new file mode 100644 index 000000000000..ace73861c2a0 --- /dev/null +++ b/arch/x86/entry/entry64.c @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 1991, 1992 Linus Torvalds + * Copyright (C) 2000, 2001, 2002 Andi Kleen SuSE Labs + * Copyright (C) 2000 Pavel Machek + * Copyright (C) 2022 Lai Jiangshan, Ant Group + * + * Handle entries and exits for hardware traps and faults. + * + * It is as low level as entry_64.S and its code can be running in the + * environments that the GS base is a user controlled value, or the CR3 + * is the PTI user CR3 or both. + */ +#include --=20 2.19.1.6.gb485710b From nobody Sun Jun 14 21:38:11 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id C91A5C433F5 for ; Wed, 11 May 2022 07:28:08 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S242924AbiEKH2F (ORCPT ); Wed, 11 May 2022 03:28:05 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:55806 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S242921AbiEKH15 (ORCPT ); Wed, 11 May 2022 03:27:57 -0400 Received: from mail-pl1-x62d.google.com (mail-pl1-x62d.google.com [IPv6:2607:f8b0:4864:20::62d]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id E82723AA63 for ; Wed, 11 May 2022 00:27:56 -0700 (PDT) Received: by mail-pl1-x62d.google.com with SMTP id n8so1080282plh.1 for ; Wed, 11 May 2022 00:27:56 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20210112; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=QCT2LbXITXaXg1IP5G0BZIj7t3pQnzQLCDE67J//+m0=; b=WCh8jTEBtYga4RnW8R0dDgoAWfGSj1X9VsFDHHrAs6qu9ulzVaT1ISO6sVdqMIfmV3 ICH6fLUghmDrbvY3uNAjOBavJR2SM6RQ80chgPBLXXgVQuMOBKzFD1CXxiOT+JwaRxKr qflg/IBXD5QDI4tQsUethlRaBJkDGGafck0wyU3UhCFDhT83W/4/jlwlfTsXxSPRLYsn gASe50r/wpsC1I2KBfonRvghziY6U7W3h0+bB0FA6o8OqbnJuKLMNmnfa/wgFmkVxJwj Y1z7VEa8pmZ9KTa7EcdbFsEaAqQIiHk+7w+0FsmvT71nDSi5GrymkptDoddXk9+/cSjW mfkw== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=QCT2LbXITXaXg1IP5G0BZIj7t3pQnzQLCDE67J//+m0=; b=4dIvKfS38KvoBwY9obqaERy1XQoQS6kL/nlIY2z6evFvZoSa5grZedtP/Gcz88Twjk 1iNykT02SOD/1a24vFWtuZM9Gld+OWw+LQvqgcGN8cqVoGn2EHOLlHKlBiyOso/1Gxpu vSHwqAKKpg84t2qqAmTQtlHdoFoLFAcZmtu9fgTOGCAY0i8iEGT+WpiSS6r7iskv695R 9I5xOr/QntKdngSeiBX6vbSNYIW1BVN5zvGlolL5z89v9fg1aslJDBAvdPB0ysz23e4E gCTrqASNmnJdzpDpeeBjEzJILAaQJu9AFbrMIksN1rnbAm7V1RXytp3hhyDUxlV8wZHi SnXQ== X-Gm-Message-State: AOAM533eOtbldLEr3XZp9/+u9bThy09ZxWD5JAPgxtWWSiQ0UOzyv3Mt bfscV3VmH1Nl8b9GPdh0ZLi7qIHBrfc= X-Google-Smtp-Source: ABdhPJzQX7ce+QdzR67oV2Ue6B/1loxjFuPDBDFPk/rF8o9MvuobsIpNt+mmLM851ou9V6d6EaJekA== X-Received: by 2002:a17:90a:ea18:b0:1da:4630:513d with SMTP id w24-20020a17090aea1800b001da4630513dmr3881405pjy.237.1652254076219; Wed, 11 May 2022 00:27:56 -0700 (PDT) Received: from localhost ([47.251.4.198]) by smtp.gmail.com with ESMTPSA id z5-20020aa785c5000000b0050dc76281ddsm837343pfn.183.2022.05.11.00.27.55 (version=TLS1_2 cipher=ECDHE-ECDSA-AES128-GCM-SHA256 bits=128/128); Wed, 11 May 2022 00:27:55 -0700 (PDT) From: Lai Jiangshan To: linux-kernel@vger.kernel.org Cc: Borislav Petkov , Peter Zijlstra , Josh Poimboeuf , Andy Lutomirski , Thomas Gleixner , x86@kernel.org, Lai Jiangshan , Ingo Molnar , Dave Hansen , "H. Peter Anvin" Subject: [PATCH 5/7] x86/entry: Add the C verion of SWITCH_TO_KERNEL_CR3 as switch_to_kernel_cr3() Date: Wed, 11 May 2022 15:27:45 +0800 Message-Id: <20220511072747.3960-6-jiangshanlai@gmail.com> X-Mailer: git-send-email 2.19.1.6.gb485710b In-Reply-To: <20220511072747.3960-1-jiangshanlai@gmail.com> References: <20220511072747.3960-1-jiangshanlai@gmail.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" From: Lai Jiangshan Add the C version switch_to_kernel_cr3() which implements the macro SWITCH_TO_KERNEL_CR3() in arch/x86/entry/calling.h. No functional difference intended. Signed-off-by: Lai Jiangshan --- arch/x86/entry/entry64.c | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/arch/x86/entry/entry64.c b/arch/x86/entry/entry64.c index ace73861c2a0..bd77cc8373ce 100644 --- a/arch/x86/entry/entry64.c +++ b/arch/x86/entry/entry64.c @@ -12,3 +12,27 @@ * is the PTI user CR3 or both. */ #include + +#ifdef CONFIG_PAGE_TABLE_ISOLATION +static __always_inline void pti_switch_to_kernel_cr3(unsigned long user_cr= 3) +{ + /* + * Clear PCID and "PAGE_TABLE_ISOLATION bit", point CR3 + * at kernel pagetables: + */ + unsigned long cr3 =3D user_cr3 & ~PTI_USER_PGTABLE_AND_PCID_MASK; + + if (static_cpu_has(X86_FEATURE_PCID)) + cr3 |=3D X86_CR3_PCID_NOFLUSH; + + native_write_cr3(cr3); +} + +static __always_inline void switch_to_kernel_cr3(void) +{ + if (static_cpu_has(X86_FEATURE_PTI)) + pti_switch_to_kernel_cr3(__native_read_cr3()); +} +#else +static __always_inline void switch_to_kernel_cr3(void) {} +#endif --=20 2.19.1.6.gb485710b From nobody Sun Jun 14 21:38:11 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id B96D5C433FE for ; Wed, 11 May 2022 07:28:16 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S243036AbiEKH2O (ORCPT ); Wed, 11 May 2022 03:28:14 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:56682 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S242956AbiEKH2H (ORCPT ); Wed, 11 May 2022 03:28:07 -0400 Received: from mail-pl1-x636.google.com (mail-pl1-x636.google.com [IPv6:2607:f8b0:4864:20::636]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id CADAF3D483 for ; Wed, 11 May 2022 00:28:00 -0700 (PDT) Received: by mail-pl1-x636.google.com with SMTP id i1so1052736plg.7 for ; Wed, 11 May 2022 00:28:00 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20210112; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=gkGEhy3VfxaC6VD1+0Znm8PQRe5Rv1gv7lzNV+9U6sU=; b=hOL2SmGEKUuNpF8//yrIHyFyW9QpHj5EcEyqSlL2j5Ss2Kk637E+sdDr1ipujizSp6 mBs3ISgZQuqdAdu6OMZ75YBdnojiygEvrhyLm+fc2iZzRtk4RIIOEbmWSMxqSmzEJ6iu 9mpCShLthHzj8HAPNuuxT5t9pJU5mrD9R70FEwFrqI+jpZbyrbhS/F9XNccIhoi7zerA qQwXXC5zyn6Q42wm1x+zxTMSLFzjR4NIBGUmXDMwlT7zDPQcWsWqPmllg0KrINWZE7Hh V3iBAJED+kr+y055flPWxxW2ORJHzR62PfxEpXLilu7+dZkR2JTPP+fr0ooi5+Up7Czy P7gA== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=gkGEhy3VfxaC6VD1+0Znm8PQRe5Rv1gv7lzNV+9U6sU=; b=Eys4qDuBve7Qtgo/wCyWHnNzLu56oqPeegFAisMbeN62YUA8wbiho7astHTK21NiPs 7UrRmdIc3aVQo5DT8BBDUC08STEpuppiiAoQ4nRHHfWjkrpaXQPSPQUXnSdOAleIKcXD 5aU084/FuQ1P2fFdoO3t4d1Sp9gLrZ+RRq4sqsaYAcMhx8NO39hugvq8JnvY9/2n6bJ6 Kw8moap5DFiAb5eyfA/FC/nysdbuMylIeNyFo/o63vcuohpKQeSICoLUShCcYLLApsvA BHeZ4ILftsIJ042N9O/DpYBmL4wpKUUgm+YBdyfpvkn9SxprebS2OXcuodI/XlBNLR9F VfQg== X-Gm-Message-State: AOAM530jJBuFsqFtSvrgNgxqE0cGRY6Mu7/CIJwpAr+9Mjxivlo50EE8 mhz7Dy/YmznRyway2kGxwz0rpGUH/To= X-Google-Smtp-Source: ABdhPJyiupD//7dRtmU1nOj1/+hIdK289hdKppc2VOXLd8W9IyHJRVmAo9TXo3zE35ecQWZRa1cRVA== X-Received: by 2002:a17:90a:ec0e:b0:1d9:7f4d:23be with SMTP id l14-20020a17090aec0e00b001d97f4d23bemr3883921pjy.181.1652254080230; Wed, 11 May 2022 00:28:00 -0700 (PDT) Received: from localhost ([47.251.4.198]) by smtp.gmail.com with ESMTPSA id be12-20020a170902aa0c00b0015e8d4eb1e4sm954269plb.46.2022.05.11.00.27.59 (version=TLS1_2 cipher=ECDHE-ECDSA-AES128-GCM-SHA256 bits=128/128); Wed, 11 May 2022 00:27:59 -0700 (PDT) From: Lai Jiangshan To: linux-kernel@vger.kernel.org Cc: Borislav Petkov , Peter Zijlstra , Josh Poimboeuf , Andy Lutomirski , Thomas Gleixner , x86@kernel.org, Lai Jiangshan , Ingo Molnar , Dave Hansen , "H. Peter Anvin" Subject: [PATCH 6/7] x86/traps: Add fence_swapgs_{user,kernel}_entry() and user_entry_swapgs_and_fence() Date: Wed, 11 May 2022 15:27:46 +0800 Message-Id: <20220511072747.3960-7-jiangshanlai@gmail.com> X-Mailer: git-send-email 2.19.1.6.gb485710b In-Reply-To: <20220511072747.3960-1-jiangshanlai@gmail.com> References: <20220511072747.3960-1-jiangshanlai@gmail.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" From: Lai Jiangshan Add the C version fence_swapgs_{user,kernel}_entry() in entry64.c which are the same as the ASM macro FENCE_SWAPGS_{USER,KERNEL}_ENTRY. fence_swapgs_user_entry is used in the user entry swapgs code path, to prevent a speculative swapgs when coming from kernel space. fence_swapgs_kernel_entry is used in the kernel entry code path, to prevent the swapgs from getting speculatively skipped when coming from user space. Add the C user_entry_swapgs_and_fence() which implements the ASM code: swapgs FENCE_SWAPGS_USER_ENTRY It will be used in the user entry swapgs code path, doing the swapgs and lfence to prevent a speculative swapgs when coming from kernel space. Cc: Josh Poimboeuf Suggested-by: Peter Zijlstra Signed-off-by: Lai Jiangshan --- arch/x86/entry/entry64.c | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/arch/x86/entry/entry64.c b/arch/x86/entry/entry64.c index bd77cc8373ce..f7f23800cee4 100644 --- a/arch/x86/entry/entry64.c +++ b/arch/x86/entry/entry64.c @@ -36,3 +36,33 @@ static __always_inline void switch_to_kernel_cr3(void) #else static __always_inline void switch_to_kernel_cr3(void) {} #endif + +/* + * Mitigate Spectre v1 for conditional swapgs code paths. + * + * fence_swapgs_user_entry is used in the user entry swapgs code path, to + * prevent a speculative swapgs when coming from kernel space. It must be + * used with switch_to_kernel_cr3() in the same path. + * + * fence_swapgs_kernel_entry is used in the kernel entry code path without + * CR3 write or with conditinal CR3 write only, to prevent the swapgs from + * getting speculatively skipped when coming from user space. + * + * user_entry_swapgs_and_fence is a wrapper of swapgs and fence for user e= ntry + * code path. + */ +static __always_inline void fence_swapgs_user_entry(void) +{ + alternative("", "lfence", X86_FEATURE_FENCE_SWAPGS_USER); +} + +static __always_inline void fence_swapgs_kernel_entry(void) +{ + alternative("", "lfence", X86_FEATURE_FENCE_SWAPGS_KERNEL); +} + +static __always_inline void user_entry_swapgs_and_fence(void) +{ + native_swapgs(); + fence_swapgs_user_entry(); +} --=20 2.19.1.6.gb485710b From nobody Sun Jun 14 21:38:11 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id B9F00C433EF for ; Wed, 11 May 2022 07:28:24 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S242995AbiEKH2W (ORCPT ); Wed, 11 May 2022 03:28:22 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:56786 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S236652AbiEKH2O (ORCPT ); Wed, 11 May 2022 03:28:14 -0400 Received: from mail-pl1-x62e.google.com (mail-pl1-x62e.google.com [IPv6:2607:f8b0:4864:20::62e]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id D1E843D1D6 for ; Wed, 11 May 2022 00:28:08 -0700 (PDT) Received: by mail-pl1-x62e.google.com with SMTP id q18so1036387pln.12 for ; Wed, 11 May 2022 00:28:08 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20210112; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=fHFue09oINs+4LzVLRKmbmRrup/wwfhBquOsUuiiMnU=; b=f/rDUu0tPk4KejNi66lkkTJPUcBlNH56UeIjuFAsi+OH+iZxeubS3jYeHroGNS8z6c PiQ6TeKU0Iu3of5PQA0GmQOVTofq+nnAaqwj6h/QFWNjhksSs2y80Q3pUS6PXLOTQN1j tvqOuxoy2vVuaWULcfYV62wuCkzmBkvDujN+3D20KuP/uZC7N1Mq0Naf71Br5mtfK1UW /IvG5TNsjiQu3meH8ceTeyzC7Ga4HS4bEyFtAzUrbFKpXLvtWqOv3er2SEyIsZhgE69J iE7TmBoRRn10S6eLScrOuGUpq6QrNuLZi3PCYwsFiJ0y6NBuVnoD5zySCfGWD2drkn3h sL/Q== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=fHFue09oINs+4LzVLRKmbmRrup/wwfhBquOsUuiiMnU=; b=QMa1k/s/fi5Mnb2CDFd7yq5vZ+eobgri2i0wZUOMmJwCctmAyL626Wi3TYS4tSPx7P P6EmrZIOkmao0G76+n08QfZCvCMIzNN3qrsICPaU9UIIl0DCTIIk5ZbunO62nI6ZfvbD gvFZIgW/POGqFaL7e2b0l7BISUxayf6gagRswK9lTefb/YkE0/xJ7C0EpejvDQe2VskF 4QzUQYuFM6tkt2NK1GP4McuQKTrJeGs/grcQDkwMajajTSnicCluoQeSOaktEpO1QW01 p+e7sS91Z1r1DoO5N8V3NbMpJidwT4/gepREorzuE4x4jmpaiRwkPPiEucw+jQxvCuTC 8dxQ== X-Gm-Message-State: AOAM533+tiaaawVeT4B2/A7l5G95RlER+TsjjlEzknZLYBDtD7d8F1LS dqkpU0zOwKSm0RRcfI+DEWHXADXzZPU= X-Google-Smtp-Source: ABdhPJwuVwnK3M4Qo4WYSourzS04CVOJP9/isTTnVIlJWEfmLeABaS5kh7+J3rPc45Of3qnolpy63Q== X-Received: by 2002:a17:903:1c6:b0:15e:c623:b543 with SMTP id e6-20020a17090301c600b0015ec623b543mr24372321plh.147.1652254088030; Wed, 11 May 2022 00:28:08 -0700 (PDT) Received: from localhost ([47.251.4.198]) by smtp.gmail.com with ESMTPSA id bb9-20020a170902bc8900b0015f2d549b46sm930309plb.237.2022.05.11.00.28.07 (version=TLS1_2 cipher=ECDHE-ECDSA-AES128-GCM-SHA256 bits=128/128); Wed, 11 May 2022 00:28:07 -0700 (PDT) From: Lai Jiangshan To: linux-kernel@vger.kernel.org Cc: Borislav Petkov , Peter Zijlstra , Josh Poimboeuf , Andy Lutomirski , Thomas Gleixner , x86@kernel.org, Lai Jiangshan , Ingo Molnar , Dave Hansen , "H. Peter Anvin" , "Chang S. Bae" , Kees Cook , "Kirill A. Shutemov" , Fenghua Yu Subject: [PATCH 7/7] x86/entry: Implement the whole error_entry() as C code Date: Wed, 11 May 2022 15:27:47 +0800 Message-Id: <20220511072747.3960-8-jiangshanlai@gmail.com> X-Mailer: git-send-email 2.19.1.6.gb485710b In-Reply-To: <20220511072747.3960-1-jiangshanlai@gmail.com> References: <20220511072747.3960-1-jiangshanlai@gmail.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" From: Lai Jiangshan Implement error_entry() as C code in arch/x86/entry/entry64.c and replace the ASM version of error_entry(). The code might be in the user CR3 and user GS base at the start of the function so it calls __always_inline C function only until the GS and CR3 is switched. No functional change intended and comments are also copied. The C version generally has better readability and easier to be updated/improved. Signed-off-by: Lai Jiangshan --- arch/x86/entry/entry64.c | 69 +++++++++++++++++++++++++++++ arch/x86/entry/entry_64.S | 85 ++---------------------------------- arch/x86/include/asm/proto.h | 1 + arch/x86/include/asm/traps.h | 1 + arch/x86/kernel/traps.c | 2 - 5 files changed, 74 insertions(+), 84 deletions(-) diff --git a/arch/x86/entry/entry64.c b/arch/x86/entry/entry64.c index f7f23800cee4..bd047c329622 100644 --- a/arch/x86/entry/entry64.c +++ b/arch/x86/entry/entry64.c @@ -13,6 +13,8 @@ */ #include =20 +extern unsigned char asm_load_gs_index_gs_change[]; + #ifdef CONFIG_PAGE_TABLE_ISOLATION static __always_inline void pti_switch_to_kernel_cr3(unsigned long user_cr= 3) { @@ -66,3 +68,70 @@ static __always_inline void user_entry_swapgs_and_fence(= void) native_swapgs(); fence_swapgs_user_entry(); } + +/* + * Put pt_regs onto the task stack and switch GS and CR3 if needed. + * The actual stack switch is done in entry_64.S. + * + * Be careful, it might be in the user CR3 and user GS base at the start + * of the function. + */ +asmlinkage __visible __entry_text +struct pt_regs *error_entry(struct pt_regs *eregs) +{ + unsigned long iret_ip =3D (unsigned long)native_irq_return_iret; + + if (user_mode(eregs)) { + /* + * We entered from user mode. + * Switch to kernel gsbase and CR3. + */ + user_entry_swapgs_and_fence(); + switch_to_kernel_cr3(); + + /* Put pt_regs onto the task stack. */ + return sync_regs(eregs); + } + + /* + * There are two places in the kernel that can potentially fault with + * usergs. Handle them here. B stepping K8s sometimes report a + * truncated RIP for IRET exceptions returning to compat mode. Check + * for these here too. + */ + if ((eregs->ip =3D=3D iret_ip) || (eregs->ip =3D=3D (unsigned int)iret_ip= )) { + eregs->ip =3D iret_ip; /* Fix truncated RIP */ + + /* + * We came from an IRET to user mode, so we have user + * gsbase and CR3. Switch to kernel gsbase and CR3: + */ + user_entry_swapgs_and_fence(); + switch_to_kernel_cr3(); + + /* + * Pretend that the exception came from user mode: set up + * pt_regs as if we faulted immediately after IRET and then + * put pt_regs onto the real task stack. + */ + return sync_regs(fixup_bad_iret(eregs)); + } + + /* + * Hack: asm_load_gs_index_gs_change can fail with user gsbase. + * If this happens, fix up gsbase and proceed. We'll fix up the + * exception and land in asm_load_gs_index_gs_change's error + * handler with kernel gsbase. + */ + if (eregs->ip =3D=3D (unsigned long)asm_load_gs_index_gs_change) + native_swapgs(); + + /* + * Issue an LFENCE to prevent GS speculation, regardless of whether + * it is a kernel or user gsbase. + */ + fence_swapgs_kernel_entry(); + + /* Enter from kernel, don't move pt_regs */ + return eregs; +} diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index 3a1e3f215617..b678189b029e 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -343,7 +343,7 @@ SYM_CODE_END(push_and_clear_regs) * own pvops for IRET and load_gs_index(). And it doesn't need to * switch the CR3. So it can skip invoking error_entry(). */ - ALTERNATIVE "call error_entry; movq %rax, %rsp", \ + ALTERNATIVE "movq %rsp, %rdi; call error_entry; movq %rax, %rsp", \ "", X86_FEATURE_XENPV =20 ENCODE_FRAME_POINTER @@ -778,7 +778,7 @@ _ASM_NOKPROBE(common_interrupt_return) SYM_FUNC_START(asm_load_gs_index) FRAME_BEGIN swapgs -.Lgs_change: +SYM_INNER_LABEL(asm_load_gs_index_gs_change, SYM_L_GLOBAL) ANNOTATE_NOENDBR // error_entry movl %edi, %gs 2: ALTERNATIVE "", "mfence", X86_BUG_SWAPGS_FENCE @@ -799,7 +799,7 @@ SYM_FUNC_START(asm_load_gs_index) movl %eax, %gs jmp 2b =20 - _ASM_EXTABLE(.Lgs_change, .Lbad_gs) + _ASM_EXTABLE(asm_load_gs_index_gs_change, .Lbad_gs) =20 SYM_FUNC_END(asm_load_gs_index) EXPORT_SYMBOL(asm_load_gs_index) @@ -1006,85 +1006,6 @@ SYM_CODE_START_LOCAL(paranoid_exit) jmp restore_regs_and_return_to_kernel SYM_CODE_END(paranoid_exit) =20 -/* - * Switch GS and CR3 if needed. - */ -SYM_CODE_START_LOCAL(error_entry) - UNWIND_HINT_FUNC - testb $3, CS+8(%rsp) - jz .Lerror_kernelspace - - /* - * We entered from user mode or we're pretending to have entered - * from user mode due to an IRET fault. - */ - swapgs - FENCE_SWAPGS_USER_ENTRY - /* We have user CR3. Change to kernel CR3. */ - SWITCH_TO_KERNEL_CR3 scratch_reg=3D%rax - - leaq 8(%rsp), %rdi /* arg0 =3D pt_regs pointer */ -.Lerror_entry_from_usermode_after_swapgs: - /* Put us onto the real thread stack. */ - call sync_regs - RET - - /* - * There are two places in the kernel that can potentially fault with - * usergs. Handle them here. B stepping K8s sometimes report a - * truncated RIP for IRET exceptions returning to compat mode. Check - * for these here too. - */ -.Lerror_kernelspace: - leaq native_irq_return_iret(%rip), %rcx - cmpq %rcx, RIP+8(%rsp) - je .Lerror_bad_iret - movl %ecx, %eax /* zero extend */ - cmpq %rax, RIP+8(%rsp) - je .Lbstep_iret - cmpq $.Lgs_change, RIP+8(%rsp) - jne .Lerror_entry_done_lfence - - /* - * hack: .Lgs_change can fail with user gsbase. If this happens, fix up - * gsbase and proceed. We'll fix up the exception and land in - * .Lgs_change's error handler with kernel gsbase. - */ - swapgs - - /* - * Issue an LFENCE to prevent GS speculation, regardless of whether it is= a - * kernel or user gsbase. - */ -.Lerror_entry_done_lfence: - FENCE_SWAPGS_KERNEL_ENTRY - leaq 8(%rsp), %rax /* return pt_regs pointer */ - RET - -.Lbstep_iret: - /* Fix truncated RIP */ - movq %rcx, RIP+8(%rsp) - /* fall through */ - -.Lerror_bad_iret: - /* - * We came from an IRET to user mode, so we have user - * gsbase and CR3. Switch to kernel gsbase and CR3: - */ - swapgs - FENCE_SWAPGS_USER_ENTRY - SWITCH_TO_KERNEL_CR3 scratch_reg=3D%rax - - /* - * Pretend that the exception came from user mode: set up pt_regs - * as if we faulted immediately after IRET. - */ - leaq 8(%rsp), %rdi /* arg0 =3D pt_regs pointer */ - call fixup_bad_iret - mov %rax, %rdi - jmp .Lerror_entry_from_usermode_after_swapgs -SYM_CODE_END(error_entry) - SYM_CODE_START_LOCAL(error_return) UNWIND_HINT_REGS DEBUG_ENTRY_ASSERT_IRQS_OFF diff --git a/arch/x86/include/asm/proto.h b/arch/x86/include/asm/proto.h index 0f899c8d7a4e..95d6d3a53cd5 100644 --- a/arch/x86/include/asm/proto.h +++ b/arch/x86/include/asm/proto.h @@ -13,6 +13,7 @@ void syscall_init(void); #ifdef CONFIG_X86_64 void entry_SYSCALL_64(void); void entry_SYSCALL_64_safe_stack(void); +extern unsigned char native_irq_return_iret[]; long do_arch_prctl_64(struct task_struct *task, int option, unsigned long = arg2); #endif =20 diff --git a/arch/x86/include/asm/traps.h b/arch/x86/include/asm/traps.h index 47ecfff2c83d..2d00100d3e03 100644 --- a/arch/x86/include/asm/traps.h +++ b/arch/x86/include/asm/traps.h @@ -14,6 +14,7 @@ asmlinkage __visible notrace struct pt_regs *sync_regs(struct pt_regs *ere= gs); asmlinkage __visible notrace struct pt_regs *fixup_bad_iret(struct pt_regs *bad_regs); +asmlinkage __visible notrace struct pt_regs *error_entry(struct pt_regs *e= regs); void __init trap_init(void); asmlinkage __visible noinstr struct pt_regs *vc_switch_off_ist(struct pt_r= egs *eregs); #endif diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c index d62b2cb85cea..f76a15f654c5 100644 --- a/arch/x86/kernel/traps.c +++ b/arch/x86/kernel/traps.c @@ -436,8 +436,6 @@ DEFINE_IDTENTRY_DF(exc_double_fault) #endif =20 #ifdef CONFIG_X86_ESPFIX64 - extern unsigned char native_irq_return_iret[]; - /* * If IRET takes a non-IST fault on the espfix64 stack, then we * end up promoting it to a doublefault. In that case, take --=20 2.19.1.6.gb485710b