From nobody Tue Apr 28 05:13:09 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 4490EC433EF for ; Mon, 6 Jun 2022 14:44:39 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S239906AbiFFOoi (ORCPT ); Mon, 6 Jun 2022 10:44:38 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:50750 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S239823AbiFFOo3 (ORCPT ); Mon, 6 Jun 2022 10:44:29 -0400 Received: from mail-pj1-x102c.google.com (mail-pj1-x102c.google.com [IPv6:2607:f8b0:4864:20::102c]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 9BC235000A for ; Mon, 6 Jun 2022 07:44:28 -0700 (PDT) Received: by mail-pj1-x102c.google.com with SMTP id w13-20020a17090a780d00b001e8961b355dso853770pjk.5 for ; Mon, 06 Jun 2022 07:44:28 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20210112; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=J0AjupS423IcVxPOF8+Ias71Zs7GJnxkL456RKcGoI4=; b=N96+7PHU+aEx81sVHh6CFec/XmlIA4EYXB8GPuNvzQ3gFS/MAzcwvGBdtU9oazZzf1 lccNpukP/evxop0gpJssxdVvPC9+FW3LTlYWGsH0ZPzP4xGQiXDrpiLlqt5sZPtK9mJk r+w//DfnS8SM51tMbLkhxRwk8EhRSh+VRG1O9xhKZTW9x2qn94N0BV6ITq9sez5z/lXi D/HPH7EYhHxDwrEJKezy3M74vUGaidaXI2jrhgRQsnCcYLYG9riapBkGIOY7vVb5R2t1 zDeszG5J/d4BvkgNfviauw+gb5lqRwwimAwuG3WEpYeNHRVmitVkhiurdWJbqqmemRRw utcQ== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=J0AjupS423IcVxPOF8+Ias71Zs7GJnxkL456RKcGoI4=; b=T3qd5RkctF9zHaRVjrvThSOkmqRfRTJJ7l+DLDAPcIu6p/jDFgNGbX/KYt+QucBG7z Q6MNHxkR5ZJiWWqWQOWRBqMdKWGBi0c6hXc7EoF4JmpchYii/LKggYOIZKrdz/HYA8Zf qtuc0+5bCCaQe8bL5ohsOYSWmM1JcGPQlulypvtmUEwXy6JG0tTHghTXM2QQ+a1lYJcV WQZ9iYcbcYHu/aT5VCyXjESNqNwZGfYc7mdW4jz9AT3J7pKayTjZbxqsgX2aigwWM7Pa Cf68GH0ssJhrFQzaq85BP6BUJ8Q9wdN4EoF3LGA+nGwgPlc5KLCmmd2ToZ+Q0Q7ClmcD Wbvg== X-Gm-Message-State: AOAM530rWY/BYPCYq8JZlpIJUnfACmXG4iBaVoZVjKML2POBzeIlLngA eaLLIqEIc2WP0XJlgMpEdWKvqVHe3Pw= X-Google-Smtp-Source: ABdhPJz/kDVaaM5WXkqM1kL/NfpKznIoYnd07CGUfIgO5QKGW1t3iSzuXEPbI6rpwgXQTiSC4PlQ/A== X-Received: by 2002:a17:90a:9f04:b0:1e3:2d77:3eae with SMTP id n4-20020a17090a9f0400b001e32d773eaemr38454049pjp.243.1654526667932; Mon, 06 Jun 2022 07:44:27 -0700 (PDT) Received: from localhost ([198.11.178.15]) by smtp.gmail.com with ESMTPSA id q14-20020a170902eb8e00b001615f64aaabsm6200271plg.244.2022.06.06.07.44.26 (version=TLS1_2 cipher=ECDHE-ECDSA-AES128-GCM-SHA256 bits=128/128); Mon, 06 Jun 2022 07:44:27 -0700 (PDT) From: Lai Jiangshan To: linux-kernel@vger.kernel.org Cc: Borislav Petkov , Peter Zijlstra , Josh Poimboeuf , Andy Lutomirski , Thomas Gleixner , x86@kernel.org, Lai Jiangshan , Ingo Molnar , Dave Hansen , "H. Peter Anvin" , Sean Christopherson , Joerg Roedel , "Kirill A. Shutemov" , Miguel Ojeda , Kees Cook , Nathan Chancellor , Andrew Morton , Alexei Starovoitov , Marco Elver , Hao Luo , Nick Desaulniers , Rasmus Villemoes Subject: [PATCH V3 1/7] x86/entry: Introduce __entry_text for entry code written in C Date: Mon, 6 Jun 2022 22:45:03 +0800 Message-Id: <20220606144509.617611-2-jiangshanlai@gmail.com> X-Mailer: git-send-email 2.19.1.6.gb485710b In-Reply-To: <20220606144509.617611-1-jiangshanlai@gmail.com> References: <20220606144509.617611-1-jiangshanlai@gmail.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" From: Lai Jiangshan Some entry code will be implemented in C files. Introduce __entry_text to set them in .entry.text section. The new __entry_text disables instrumentation like noinstr, so __noinstr_section() is added for noinstr and the new __entry_text. Note, entry code can not access to %gs before the %gs base is switched to kernel %gs base, so stack protector can not be used on the C entry code. But __entry_text doesn't disable stack protector since some compilers might not support function level granular attribute to disable stack protector. It will be disabled in C file level. Cc: Borislav Petkov Reviewed-by: Miguel Ojeda Reviewed-by: Kees Cook Suggested-by: Nick Desaulniers Suggested-by: Peter Zijlstra Signed-off-by: Lai Jiangshan --- arch/x86/include/asm/idtentry.h | 3 +++ include/linux/compiler_types.h | 8 +++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/arch/x86/include/asm/idtentry.h b/arch/x86/include/asm/idtentr= y.h index 72184b0b2219..acc4c99f801c 100644 --- a/arch/x86/include/asm/idtentry.h +++ b/arch/x86/include/asm/idtentry.h @@ -13,6 +13,9 @@ =20 #include =20 +/* Entry code written in C. */ +#define __entry_text __noinstr_section(".entry.text") + /** * DECLARE_IDTENTRY - Declare functions for simple IDT entry points * No error code pushed by hardware diff --git a/include/linux/compiler_types.h b/include/linux/compiler_types.h index d08dfcb0ac68..bd9d9d19dc9b 100644 --- a/include/linux/compiler_types.h +++ b/include/linux/compiler_types.h @@ -225,9 +225,11 @@ struct ftrace_likely_data { #endif =20 /* Section for code which can't be instrumented at all */ -#define noinstr \ - noinline notrace __attribute((__section__(".noinstr.text"))) \ - __no_kcsan __no_sanitize_address __no_profile __no_sanitize_coverage +#define __noinstr_section(section) \ + noinline notrace __section(section) __no_kcsan \ + __no_sanitize_address __no_profile __no_sanitize_coverage + +#define noinstr __noinstr_section(".noinstr.text") =20 #endif /* __KERNEL__ */ =20 --=20 2.19.1.6.gb485710b From nobody Tue Apr 28 05:13:09 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 31929C43334 for ; Mon, 6 Jun 2022 14:44:42 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S239935AbiFFOoj (ORCPT ); Mon, 6 Jun 2022 10:44:39 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:51274 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S239910AbiFFOof (ORCPT ); Mon, 6 Jun 2022 10:44:35 -0400 Received: from mail-pf1-x42b.google.com (mail-pf1-x42b.google.com [IPv6:2607:f8b0:4864:20::42b]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 0232C50074 for ; Mon, 6 Jun 2022 07:44:34 -0700 (PDT) Received: by mail-pf1-x42b.google.com with SMTP id b135so12852471pfb.12 for ; Mon, 06 Jun 2022 07:44:33 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20210112; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=OkYs0lk3nHyAZKJDGCn/29siMwWnrfKmu5gV1SS4NfQ=; b=M8NSMNiAVGRL+nc2mCIZaZ52gNNlHv8ElkKTH8jMWhW6xYbpoJqgWNTveOmWj+1VvN 5Pxw/X7rDUCCWrK1Lwtx+GnHB4eRtniUw6zQyhdkyFaOYKRknod5kLfZ+gglaM1Jqkg5 FSmzdu/P7mLn7MDh8T3Bma2ZSKZySHLfNnoJgHM/WCQHuIokrSAR+VlZ67ouch65D4B/ 5kfFTTUK7MEXauaF7F71rJUaVFpwnHYP1XJ4Z7bg2zcgWDcY7GccWB7cOvKbySi+pkQP Zut6wSD1y0/SOXFnEVM5ZZaBfmzs+UUGCCpXYJb1muBe/pAGDHdBMmWy+RQGMgsdRbgk xx2Q== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=OkYs0lk3nHyAZKJDGCn/29siMwWnrfKmu5gV1SS4NfQ=; b=MawyNES1m55mgst1rIGJ4vnpGhr3083aFepyJ8iZ20AKeJcSKjEQkgLUJmSS3ou4Wy ljSxJnY801z8X/UkjMTsq9IRncMAa2Lfrd/sBm33bTO9z4nkj3fO4F43Vo7/VOM3zIZn BfHjzAhxDTXs3wsvquNuRzRw7oVLCmHQv6biw1hW+dSTsriU5fG35ZP1emfT6p8x7fPS 7lQ+Rm32tUGXHRKXAggR2jhaYIHiMmtHz4IC6zTev2wRNpycci3bQ/PPsHcLIj7T4VfW Sz2xMeIkFzFv2HiuapDHDMAoJKKqN7v0Cq98h224yvPP0Ok2dfMI/o0MRLVYtW7yCPG5 CspA== X-Gm-Message-State: AOAM531HaaWNkJf5R9iqXvbrHqtQHhEixq0vKBuVYlJn9OdiMYtz86TA FZmkLl0oddDmg4WghLCjUGAfrcigy5Q= X-Google-Smtp-Source: ABdhPJyRBekBNDNM20UyEIByz6XPcXR4ORScgmrN+JyQ3HZtrYfwDUB208Oyc/OPFI3sE8yVlbazDA== X-Received: by 2002:a65:6d87:0:b0:3fd:a62e:dc0a with SMTP id bc7-20020a656d87000000b003fda62edc0amr6427969pgb.286.1654526673290; Mon, 06 Jun 2022 07:44:33 -0700 (PDT) Received: from localhost ([198.11.178.15]) by smtp.gmail.com with ESMTPSA id iw22-20020a170903045600b0015e8d4eb2afsm10666584plb.249.2022.06.06.07.44.32 (version=TLS1_2 cipher=ECDHE-ECDSA-AES128-GCM-SHA256 bits=128/128); Mon, 06 Jun 2022 07:44:32 -0700 (PDT) From: Lai Jiangshan To: linux-kernel@vger.kernel.org Cc: Borislav Petkov , Peter Zijlstra , Josh Poimboeuf , Andy Lutomirski , Thomas Gleixner , x86@kernel.org, Lai Jiangshan , Ingo Molnar , Dave Hansen , "H. Peter Anvin" Subject: [PATCH V3 2/7] x86/entry: Move PTI_USER_* to arch/x86/include/asm/processor-flags.h Date: Mon, 6 Jun 2022 22:45:04 +0800 Message-Id: <20220606144509.617611-3-jiangshanlai@gmail.com> X-Mailer: git-send-email 2.19.1.6.gb485710b In-Reply-To: <20220606144509.617611-1-jiangshanlai@gmail.com> References: <20220606144509.617611-1-jiangshanlai@gmail.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" From: Lai Jiangshan These constants will be also used in C file. Move them to arch/x86/include/asm/processor-flags.h which already has a kin X86_CR3_PTI_PCID_USER_BIT defined. Signed-off-by: Lai Jiangshan --- arch/x86/entry/calling.h | 10 ---------- arch/x86/include/asm/processor-flags.h | 15 +++++++++++++++ 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/arch/x86/entry/calling.h b/arch/x86/entry/calling.h index 29b36e9e4e74..331a44994cc0 100644 --- a/arch/x86/entry/calling.h +++ b/arch/x86/entry/calling.h @@ -142,16 +142,6 @@ For 32-bit we have the following conventions - kernel = is built with =20 #ifdef CONFIG_PAGE_TABLE_ISOLATION =20 -/* - * PAGE_TABLE_ISOLATION PGDs are 8k. Flip bit 12 to switch between the two - * halves: - */ -#define PTI_USER_PGTABLE_BIT PAGE_SHIFT -#define PTI_USER_PGTABLE_MASK (1 << PTI_USER_PGTABLE_BIT) -#define PTI_USER_PCID_BIT X86_CR3_PTI_PCID_USER_BIT -#define PTI_USER_PCID_MASK (1 << PTI_USER_PCID_BIT) -#define PTI_USER_PGTABLE_AND_PCID_MASK (PTI_USER_PCID_MASK | PTI_USER_PGT= ABLE_MASK) - .macro SET_NOFLUSH_BIT reg:req bts $X86_CR3_PCID_NOFLUSH_BIT, \reg .endm diff --git a/arch/x86/include/asm/processor-flags.h b/arch/x86/include/asm/= processor-flags.h index 02c2cbda4a74..4dd2fbbc861a 100644 --- a/arch/x86/include/asm/processor-flags.h +++ b/arch/x86/include/asm/processor-flags.h @@ -4,6 +4,7 @@ =20 #include #include +#include =20 #ifdef CONFIG_VM86 #define X86_VM_MASK X86_EFLAGS_VM @@ -50,7 +51,21 @@ #endif =20 #ifdef CONFIG_PAGE_TABLE_ISOLATION + # define X86_CR3_PTI_PCID_USER_BIT 11 + +#ifdef CONFIG_X86_64 +/* + * PAGE_TABLE_ISOLATION PGDs are 8k. Flip bit 12 to switch between the two + * halves: + */ +#define PTI_USER_PGTABLE_BIT PAGE_SHIFT +#define PTI_USER_PGTABLE_MASK (1 << PTI_USER_PGTABLE_BIT) +#define PTI_USER_PCID_BIT X86_CR3_PTI_PCID_USER_BIT +#define PTI_USER_PCID_MASK (1 << PTI_USER_PCID_BIT) +#define PTI_USER_PGTABLE_AND_PCID_MASK (PTI_USER_PCID_MASK | PTI_USER_PGT= ABLE_MASK) +#endif + #endif =20 #endif /* _ASM_X86_PROCESSOR_FLAGS_H */ --=20 2.19.1.6.gb485710b From nobody Tue Apr 28 05:13:09 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id CA4DFC433EF for ; Mon, 6 Jun 2022 14:44:45 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S239967AbiFFOoo (ORCPT ); Mon, 6 Jun 2022 10:44:44 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:51806 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S239938AbiFFOok (ORCPT ); Mon, 6 Jun 2022 10:44:40 -0400 Received: from mail-pj1-x1030.google.com (mail-pj1-x1030.google.com [IPv6:2607:f8b0:4864:20::1030]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id F1B5D5007C for ; Mon, 6 Jun 2022 07:44:38 -0700 (PDT) Received: by mail-pj1-x1030.google.com with SMTP id gc3-20020a17090b310300b001e33092c737so12731150pjb.3 for ; Mon, 06 Jun 2022 07:44:38 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20210112; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=QPVmCWUy/pAelwn42xkgA5+/zRNQTu28RGMt2EDlrN0=; b=EzHN8U2F/StCuXhufFA3sJ4pReqdTjATzKpSNQpYATgle7+Xx5SwhvaduKMju9mWbW Q2rrzMoiOmo5KAkaW9v2Ci3cFVvNopWOcGj1FSWcw3prj7l9T5cD3y7uoFAdIRlBJhFZ hB3Tjj6Xgo9KDIRlYoW5torY4HWJLyeOVpmm7U89YGMleBnndBjZ9bZwpPGbF6N7MMhX fqkOpA7bJfIpHitAcQ6cQbGpSgddMTjJCeUgr61gipobhiH4d7gMlJMYfDi/PhDv1VYY GQ769Wj5w1zo5sMC5xqy/sIrFyLGIoTXAYvf+nt+Il+uS0frxrJJhv4hMaBzjZrTQFbK GoPA== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=QPVmCWUy/pAelwn42xkgA5+/zRNQTu28RGMt2EDlrN0=; b=JpFm9Yxrg8eBvkAyD2nZfXkTkjR0gwUKkLufs9/7GzFqkaq2e3J7cIMgQoAoF+JWJ5 OKxlAPEe02vT/om5SOgeAhmnnN54iQUKdQ1iXqHkOyEXwE0slKPYnHf49Gf//W5S/SRE qyBOXo2nxbBgKGQ/sfRUHN//qsbz7CIiIibOpgawgiLQFWH2n/Ug5DiR5PNdv97LvY8+ cJgtsJ11xIPe8xqyJAg+dpNhqSa22YYdVSQ+qh/Fnt0Y4UJ8hul17Yu7NFVfxI4Ql4D8 os4TgYahyzl27kKgeLSjuR8eCP2sIJkeuJWRAC6GV2/f5j+thTnDBZ+TGCH9s+jJF8dT nBuw== X-Gm-Message-State: AOAM5330/kiaX/wPBXKT4lTXjM6WCbIj3RWrPZNOrIjx4h56uFVG0iVJ ynMYkdevFM+543kUq4h9HO/jMBbr/UU= X-Google-Smtp-Source: ABdhPJxqzFYGQCX5s0AGPOo9uaQDZnY9Ixd1ApPz41KU0QnpUFmS5yShfSvUde4QtlcWBdxbi9a+sg== X-Received: by 2002:a17:90b:947:b0:1e8:9bd0:5420 with SMTP id dw7-20020a17090b094700b001e89bd05420mr217080pjb.32.1654526678292; Mon, 06 Jun 2022 07:44:38 -0700 (PDT) Received: from localhost ([198.11.178.15]) by smtp.gmail.com with ESMTPSA id iy9-20020a170903130900b0015e8d4eb243sm7228775plb.141.2022.06.06.07.44.37 (version=TLS1_2 cipher=ECDHE-ECDSA-AES128-GCM-SHA256 bits=128/128); Mon, 06 Jun 2022 07:44:37 -0700 (PDT) From: Lai Jiangshan To: linux-kernel@vger.kernel.org Cc: Borislav Petkov , Peter Zijlstra , Josh Poimboeuf , Andy Lutomirski , Thomas Gleixner , x86@kernel.org, Lai Jiangshan , Ingo Molnar , Dave Hansen , "H. Peter Anvin" , Kees Cook , Brian Gerst Subject: [PATCH V3 3/7] x86: Mark __native_read_cr3() & native_write_cr3() as __always_inline Date: Mon, 6 Jun 2022 22:45:05 +0800 Message-Id: <20220606144509.617611-4-jiangshanlai@gmail.com> X-Mailer: git-send-email 2.19.1.6.gb485710b In-Reply-To: <20220606144509.617611-1-jiangshanlai@gmail.com> References: <20220606144509.617611-1-jiangshanlai@gmail.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" From: Lai Jiangshan Mark __native_read_cr3() & native_write_cr3() as __always_inline to ensure they are not instrumentable and in the .entry.text section if the caller is not instrumentable and in the .entry.text section. It prepares for __native_read_cr3() and native_write_cr3() to be used in the C entry code for handling KPTI. Signed-off-by: Lai Jiangshan --- arch/x86/include/asm/special_insns.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/include/asm/special_insns.h b/arch/x86/include/asm/sp= ecial_insns.h index 45b18eb94fa1..dbaee50abb3c 100644 --- a/arch/x86/include/asm/special_insns.h +++ b/arch/x86/include/asm/special_insns.h @@ -42,14 +42,14 @@ static __always_inline void native_write_cr2(unsigned l= ong val) asm volatile("mov %0,%%cr2": : "r" (val) : "memory"); } =20 -static inline unsigned long __native_read_cr3(void) +static __always_inline unsigned long __native_read_cr3(void) { unsigned long val; asm volatile("mov %%cr3,%0\n\t" : "=3Dr" (val) : __FORCE_ORDER); return val; } =20 -static inline void native_write_cr3(unsigned long val) +static __always_inline void native_write_cr3(unsigned long val) { asm volatile("mov %0,%%cr3": : "r" (val) : "memory"); } --=20 2.19.1.6.gb485710b From nobody Tue Apr 28 05:13:09 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 54F08C43334 for ; Mon, 6 Jun 2022 14:44:50 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S239993AbiFFOot (ORCPT ); Mon, 6 Jun 2022 10:44:49 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:52332 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S239973AbiFFOop (ORCPT ); Mon, 6 Jun 2022 10:44:45 -0400 Received: from mail-pj1-x1030.google.com (mail-pj1-x1030.google.com [IPv6:2607:f8b0:4864:20::1030]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id E646750455 for ; Mon, 6 Jun 2022 07:44:43 -0700 (PDT) Received: by mail-pj1-x1030.google.com with SMTP id a10so12983162pju.3 for ; Mon, 06 Jun 2022 07:44:43 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20210112; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=4unt6PUqjlwmajoQtsGt030sfPAmI7QPMB4lAchbRI4=; b=NTfisVuYf6vYip3A6oX/MtlyiOevTjc09L/PPyG1XEp9uKlhU2KdJP3HX7Z9A7j2kU Axic04KC7h61W3hKt9ubEiMliDseLMGQX9su3bO3oO12PbhU8u9n/KQxY5TMDEMJfPPV pZskzj9G1d75GXBTlQx/2LKlBRUs9ycTX3pDERNcrz3zs3kFwWM3vKqEb0eRfnPKcjZb CxcVrKkRXdZ9OSesmTiJtMM41eA6wZ7wRRUnpqqR55FCd9K8IOhBjG+yM8dOQTgTj6rn 7DN/bPNAmSc2oanW8TKkOKur5wQXG2M+CgUZJCSI8wdZcRyETB3GWLlqSDINvwfitYu2 NK7A== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=4unt6PUqjlwmajoQtsGt030sfPAmI7QPMB4lAchbRI4=; b=T0sWWtNIpb7k+QerVSYrV6FpZGAQxz4ONNISKYUKWQ2d5VTQQPtDf7Bj0+PW4TspPi ocMqSmOadf1gRHChDXIoEmFxN/YEvGpxfCVHhnNIfnOjsYPHrIFIdhkokfdzdDudQPa4 I+kMkgwwv/BbrUkjlaSwazw0eJuVaSyLQcyhoUphwelnH23aq3yKR9hB0vTXsgoTw7KO JxVp1RuoJxbf7DrHTCkbTwpA6bC8WMTWvLRxF7UWSPcglHr63p4Yd5Lj0OIBkJNdGici kwbMXN0tDt97MRJqpiXUUsC5zeJmHiZTdJfIXdw3Ny2+pxEuruIE4XQ2D8HmjSOH4nYS 1OAw== X-Gm-Message-State: AOAM533putVJAb+3im4EO7O5LlSynTFVxDMwYfgO+Xqe1NMQAZda5YDz 5sUjhuaVVs8x0Tn9Oei7wG+HTgHupMI= X-Google-Smtp-Source: ABdhPJxA1hqQGmk44f5+1g/UktQNwXxazNTajiocBnQlcpf7qSvFY3fXZhU05jEne4jYVkuUdbUC/Q== X-Received: by 2002:a17:90a:6747:b0:1e6:6a5b:f040 with SMTP id c7-20020a17090a674700b001e66a5bf040mr29353109pjm.134.1654526683263; Mon, 06 Jun 2022 07:44:43 -0700 (PDT) Received: from localhost ([198.11.178.15]) by smtp.gmail.com with ESMTPSA id g196-20020a6252cd000000b0051b9e224623sm11394200pfb.141.2022.06.06.07.44.42 (version=TLS1_2 cipher=ECDHE-ECDSA-AES128-GCM-SHA256 bits=128/128); Mon, 06 Jun 2022 07:44:42 -0700 (PDT) From: Lai Jiangshan To: linux-kernel@vger.kernel.org Cc: Borislav Petkov , Peter Zijlstra , Josh Poimboeuf , Andy Lutomirski , Thomas Gleixner , x86@kernel.org, Lai Jiangshan , Ingo Molnar , Dave Hansen , "H. Peter Anvin" Subject: [PATCH V3 4/7] x86/entry: Add arch/x86/entry/entry64.c for C entry code Date: Mon, 6 Jun 2022 22:45:06 +0800 Message-Id: <20220606144509.617611-5-jiangshanlai@gmail.com> X-Mailer: git-send-email 2.19.1.6.gb485710b In-Reply-To: <20220606144509.617611-1-jiangshanlai@gmail.com> References: <20220606144509.617611-1-jiangshanlai@gmail.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" From: Lai Jiangshan Add a C file "entry64.c" to deposit C entry code for traps and faults which will be as the same logic as the existing ASM code in entry_64.S. The file is as low level as entry_64.S and its code can be running in the environments that the GS base is a user controlled value, or the CR3 is the KPTI user CR3 or both. All the code in this file should not be instrumentable. Many instrument facilities can be disabled by per-function attributes which are included in the macro __noinstr_section. But stack-protector can not be disabled function-granularly by some compliers. So stack-protector is disabled for the whole file in Makefile. Suggested-by: Joerg Roedel Signed-off-by: Lai Jiangshan --- arch/x86/entry/Makefile | 3 ++- arch/x86/entry/entry64.c | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 arch/x86/entry/entry64.c diff --git a/arch/x86/entry/Makefile b/arch/x86/entry/Makefile index 7fec5dcf6438..792f7009ff32 100644 --- a/arch/x86/entry/Makefile +++ b/arch/x86/entry/Makefile @@ -10,13 +10,14 @@ KCOV_INSTRUMENT :=3D n CFLAGS_REMOVE_common.o =3D $(CC_FLAGS_FTRACE) =20 CFLAGS_common.o +=3D -fno-stack-protector +CFLAGS_entry64.o +=3D -fno-stack-protector =20 obj-y :=3D entry_$(BITS).o thunk_$(BITS).o syscall_$(BITS).o obj-y +=3D common.o +obj-$(CONFIG_X86_64) +=3D entry64.o =20 obj-y +=3D vdso/ obj-y +=3D vsyscall/ =20 obj-$(CONFIG_IA32_EMULATION) +=3D entry_64_compat.o syscall_32.o obj-$(CONFIG_X86_X32_ABI) +=3D syscall_x32.o - diff --git a/arch/x86/entry/entry64.c b/arch/x86/entry/entry64.c new file mode 100644 index 000000000000..ace73861c2a0 --- /dev/null +++ b/arch/x86/entry/entry64.c @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 1991, 1992 Linus Torvalds + * Copyright (C) 2000, 2001, 2002 Andi Kleen SuSE Labs + * Copyright (C) 2000 Pavel Machek + * Copyright (C) 2022 Lai Jiangshan, Ant Group + * + * Handle entries and exits for hardware traps and faults. + * + * It is as low level as entry_64.S and its code can be running in the + * environments that the GS base is a user controlled value, or the CR3 + * is the PTI user CR3 or both. + */ +#include --=20 2.19.1.6.gb485710b From nobody Tue Apr 28 05:13:09 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 92884C433EF for ; Mon, 6 Jun 2022 14:45:11 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S240017AbiFFOow (ORCPT ); Mon, 6 Jun 2022 10:44:52 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:52732 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S239996AbiFFOot (ORCPT ); Mon, 6 Jun 2022 10:44:49 -0400 Received: from mail-pg1-x52e.google.com (mail-pg1-x52e.google.com [IPv6:2607:f8b0:4864:20::52e]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 95CF050E1E for ; Mon, 6 Jun 2022 07:44:48 -0700 (PDT) Received: by mail-pg1-x52e.google.com with SMTP id 7so12281320pga.12 for ; Mon, 06 Jun 2022 07:44:48 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20210112; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=FSeMlanFh2svXxaNRkwpD5s2Jt4O19S77leAuNHNOlo=; b=WyES++Ws5th4dbIFtc9TyfPnVGRjm9A3RdjviwsyL5TUR4voWiPOI22jrHtE09OkBU DB+COkkoaryb+TiQZ8xREwMtrGSURLJrFyTgtXUQNFZbZEfpjsjkbjPa7D+NWdMxrza/ q2pkY393zj9G0k1PphbRqD9U+mf+3uki0YsQb5R7CEi/eJI80836tIn1Kxg75f2WpIpv WyzJ1dSJac/oaOmhRLav/5soBFC4jFYdf0CeMxQiIJMUAStPMaDy91HC527UWwZ0HU1J QKpTvnc7W4mtU/KWP8N23cUBky5PJ7zCEIMhwYdA42u1+Dh5du2Oct1oi2XEIvyWxO8R ofRA== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=FSeMlanFh2svXxaNRkwpD5s2Jt4O19S77leAuNHNOlo=; b=hAXLJoN6hOjiAZDhY5YJdJIqWv0nYBprnppsTLSRsD2K5ALDAa7nNRyVS8AW9mT7K8 Ii28Bt6SJly4W5Xo/uE1naGqukqU7ZqJr23ftpl/hqwcxNtPahYP8hFOyjAkkkWZaoPo iideA0rQ2kRhr1T01l44qKxhNFFpEn01YbZp+O5NQPCMRrrUqRymlB803duOexDn4nZ0 WPIv6ih+EA17in0Ya5RHSnIYol+JaPpFL4RxB2t5hnbOJVaMF2EGXkXfvxtwok49JnxT vFNAKxGERKn+3ibN/1UvzgjXVOPxqTFv9rzqrsEjEXbFumaetvRh6QNSHLY4Up6pWuM5 Md1g== X-Gm-Message-State: AOAM532/okrTXmj11rr4DXFwfRaS1cnnEOSlsORWNafPzpNXkojm8Us1 A26C4YRnr7jdGcVNDpqR6lr81RQN+Mo= X-Google-Smtp-Source: ABdhPJyhjn8JL5F/rkrJLtDcxn01B4M7DGyI+z+JnIQ91v9wV1926Nh2e1nU3mcEEbgLJbfF5Sg9MA== X-Received: by 2002:a05:6a00:1488:b0:51b:e7a2:9181 with SMTP id v8-20020a056a00148800b0051be7a29181mr16324623pfu.31.1654526687214; Mon, 06 Jun 2022 07:44:47 -0700 (PDT) Received: from localhost ([198.11.178.15]) by smtp.gmail.com with ESMTPSA id r10-20020a170902ea4a00b00163e459be9asm3624550plg.136.2022.06.06.07.44.46 (version=TLS1_2 cipher=ECDHE-ECDSA-AES128-GCM-SHA256 bits=128/128); Mon, 06 Jun 2022 07:44:46 -0700 (PDT) From: Lai Jiangshan To: linux-kernel@vger.kernel.org Cc: Borislav Petkov , Peter Zijlstra , Josh Poimboeuf , Andy Lutomirski , Thomas Gleixner , x86@kernel.org, Lai Jiangshan , Ingo Molnar , Dave Hansen , "H. Peter Anvin" Subject: [PATCH V3 5/7] x86/entry: Add the C verion of SWITCH_TO_KERNEL_CR3 as switch_to_kernel_cr3() Date: Mon, 6 Jun 2022 22:45:07 +0800 Message-Id: <20220606144509.617611-6-jiangshanlai@gmail.com> X-Mailer: git-send-email 2.19.1.6.gb485710b In-Reply-To: <20220606144509.617611-1-jiangshanlai@gmail.com> References: <20220606144509.617611-1-jiangshanlai@gmail.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" From: Lai Jiangshan Add the C version switch_to_kernel_cr3() which implements the macro SWITCH_TO_KERNEL_CR3() in arch/x86/entry/calling.h. No functional difference intended. Note: The compiler generates "AND $0xe7,%ah" (3 bytes) for the code "cr3 =3D user_cr3 & ~PTI_USER_PGTABLE_AND_PCID_MASK" while the ASM code in SWITCH_TO_KERNEL_CR3() results "AND $0xffffffffffffe7ff,%rax" (6 bytes). The compiler generates lengthier code for "cr3 |=3D X86_CR3_PCID_NOFLUSH" because it uses "MOVABS+OR" (13 bytes) rather than a single "BTS" (5 bytes). Signed-off-by: Lai Jiangshan --- arch/x86/entry/entry64.c | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/arch/x86/entry/entry64.c b/arch/x86/entry/entry64.c index ace73861c2a0..bd77cc8373ce 100644 --- a/arch/x86/entry/entry64.c +++ b/arch/x86/entry/entry64.c @@ -12,3 +12,27 @@ * is the PTI user CR3 or both. */ #include + +#ifdef CONFIG_PAGE_TABLE_ISOLATION +static __always_inline void pti_switch_to_kernel_cr3(unsigned long user_cr= 3) +{ + /* + * Clear PCID and "PAGE_TABLE_ISOLATION bit", point CR3 + * at kernel pagetables: + */ + unsigned long cr3 =3D user_cr3 & ~PTI_USER_PGTABLE_AND_PCID_MASK; + + if (static_cpu_has(X86_FEATURE_PCID)) + cr3 |=3D X86_CR3_PCID_NOFLUSH; + + native_write_cr3(cr3); +} + +static __always_inline void switch_to_kernel_cr3(void) +{ + if (static_cpu_has(X86_FEATURE_PTI)) + pti_switch_to_kernel_cr3(__native_read_cr3()); +} +#else +static __always_inline void switch_to_kernel_cr3(void) {} +#endif --=20 2.19.1.6.gb485710b From nobody Tue Apr 28 05:13:09 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 89CB5C433EF for ; Mon, 6 Jun 2022 14:45:20 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S240025AbiFFOpS (ORCPT ); Mon, 6 Jun 2022 10:45:18 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:53172 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S240031AbiFFOoz (ORCPT ); Mon, 6 Jun 2022 10:44:55 -0400 Received: from mail-pg1-x52b.google.com (mail-pg1-x52b.google.com [IPv6:2607:f8b0:4864:20::52b]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id BC2AD26543 for ; Mon, 6 Jun 2022 07:44:51 -0700 (PDT) Received: by mail-pg1-x52b.google.com with SMTP id y187so13090525pgd.3 for ; Mon, 06 Jun 2022 07:44:51 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20210112; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=gkGEhy3VfxaC6VD1+0Znm8PQRe5Rv1gv7lzNV+9U6sU=; b=XJm7qKD2bOT58RQ97zCn/zWbGHjdyy77P5xi96N2i7a9i8tTiEn3+2Panyl6bkL0K2 xda6xnU78lpVlW2JjeD3Vn/7rouIulCAg4Y9duaPAhKbDdqe3FSwZnjuNZ8T5Fu4aGHG 0TLfDpYoeKYfB2D9JSICOzPOdue57TP6Dbf0iM8FPbdQ7qGKqhmTPOwBPheewKwpdN8e rJgsLR3GpJuL5WOrsi3uJrQ72lkyiyzlXEOyuFawixOKPgiqVRWHm6UFLbTb0RpHYs/7 bshgQln3hBIa2J2CM3dRE1XjwKOLOxgoZSIsuReh0RjdRmDIxYHs2VxcLR12FcQBXEcT KEhA== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=gkGEhy3VfxaC6VD1+0Znm8PQRe5Rv1gv7lzNV+9U6sU=; b=Y+AUCXyktJn8jkcs7sIZF7z+jN1njwKr1jZ53MObRpS65elNDlY8PRyWKKpylC8eAP WCmEdSEEP4RozvJfr+Isr1/n/Kgcg8ew1zi25ZfkXo3z/wUws2zOrRxFVYUFbQH9NRta +ULBZ4TKMkrobkfnu6vlGC3NuLid87sURTGFFuhrc2TqX1nIxrqN2Znh+9lSDZXZA4b5 +HoUG2DHdKh4k8cTlv3L41fyqn9xwiCNuE/6OhHGps/b7kZVQNJg1IVu3B8LWNs4Hu7j aMOpNxt0HAVMKTlKnGvn4NxfK+5mUmXbWHKFiCf70HQKcBwRBIky4wh6OnMn91ZoFiea zCdg== X-Gm-Message-State: AOAM532iKzZOCahq52cGrcjm2UR4rbl7CWqj2reMCct/hX4W2TlaqxcN Ae4oerKekH3YZFNulsp94nNn8PDTcdw= X-Google-Smtp-Source: ABdhPJwHtzCg5kFnUiWNadbeSw965fhqP0DNjV0hekDkRGPaONCqQVXORVETWVV98temea8+zvNgdw== X-Received: by 2002:a63:5a58:0:b0:3fb:b455:ed15 with SMTP id k24-20020a635a58000000b003fbb455ed15mr21332437pgm.228.1654526691026; Mon, 06 Jun 2022 07:44:51 -0700 (PDT) Received: from localhost ([198.11.178.15]) by smtp.gmail.com with ESMTPSA id y125-20020a626483000000b0050dc76281d3sm7606451pfb.173.2022.06.06.07.44.50 (version=TLS1_2 cipher=ECDHE-ECDSA-AES128-GCM-SHA256 bits=128/128); Mon, 06 Jun 2022 07:44:50 -0700 (PDT) From: Lai Jiangshan To: linux-kernel@vger.kernel.org Cc: Borislav Petkov , Peter Zijlstra , Josh Poimboeuf , Andy Lutomirski , Thomas Gleixner , x86@kernel.org, Lai Jiangshan , Ingo Molnar , Dave Hansen , "H. Peter Anvin" Subject: [PATCH V3 6/7] x86/traps: Add fence_swapgs_{user,kernel}_entry() and user_entry_swapgs_and_fence() Date: Mon, 6 Jun 2022 22:45:08 +0800 Message-Id: <20220606144509.617611-7-jiangshanlai@gmail.com> X-Mailer: git-send-email 2.19.1.6.gb485710b In-Reply-To: <20220606144509.617611-1-jiangshanlai@gmail.com> References: <20220606144509.617611-1-jiangshanlai@gmail.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" From: Lai Jiangshan Add the C version fence_swapgs_{user,kernel}_entry() in entry64.c which are the same as the ASM macro FENCE_SWAPGS_{USER,KERNEL}_ENTRY. fence_swapgs_user_entry is used in the user entry swapgs code path, to prevent a speculative swapgs when coming from kernel space. fence_swapgs_kernel_entry is used in the kernel entry code path, to prevent the swapgs from getting speculatively skipped when coming from user space. Add the C user_entry_swapgs_and_fence() which implements the ASM code: swapgs FENCE_SWAPGS_USER_ENTRY It will be used in the user entry swapgs code path, doing the swapgs and lfence to prevent a speculative swapgs when coming from kernel space. Cc: Josh Poimboeuf Suggested-by: Peter Zijlstra Signed-off-by: Lai Jiangshan --- arch/x86/entry/entry64.c | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/arch/x86/entry/entry64.c b/arch/x86/entry/entry64.c index bd77cc8373ce..f7f23800cee4 100644 --- a/arch/x86/entry/entry64.c +++ b/arch/x86/entry/entry64.c @@ -36,3 +36,33 @@ static __always_inline void switch_to_kernel_cr3(void) #else static __always_inline void switch_to_kernel_cr3(void) {} #endif + +/* + * Mitigate Spectre v1 for conditional swapgs code paths. + * + * fence_swapgs_user_entry is used in the user entry swapgs code path, to + * prevent a speculative swapgs when coming from kernel space. It must be + * used with switch_to_kernel_cr3() in the same path. + * + * fence_swapgs_kernel_entry is used in the kernel entry code path without + * CR3 write or with conditinal CR3 write only, to prevent the swapgs from + * getting speculatively skipped when coming from user space. + * + * user_entry_swapgs_and_fence is a wrapper of swapgs and fence for user e= ntry + * code path. + */ +static __always_inline void fence_swapgs_user_entry(void) +{ + alternative("", "lfence", X86_FEATURE_FENCE_SWAPGS_USER); +} + +static __always_inline void fence_swapgs_kernel_entry(void) +{ + alternative("", "lfence", X86_FEATURE_FENCE_SWAPGS_KERNEL); +} + +static __always_inline void user_entry_swapgs_and_fence(void) +{ + native_swapgs(); + fence_swapgs_user_entry(); +} --=20 2.19.1.6.gb485710b From nobody Tue Apr 28 05:13:09 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 27577C433EF for ; Mon, 6 Jun 2022 14:45:24 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S239926AbiFFOpX (ORCPT ); Mon, 6 Jun 2022 10:45:23 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:53432 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S240058AbiFFOpA (ORCPT ); Mon, 6 Jun 2022 10:45:00 -0400 Received: from mail-pj1-x102f.google.com (mail-pj1-x102f.google.com [IPv6:2607:f8b0:4864:20::102f]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 92F8227FF1 for ; Mon, 6 Jun 2022 07:44:59 -0700 (PDT) Received: by mail-pj1-x102f.google.com with SMTP id gc3-20020a17090b310300b001e33092c737so12731886pjb.3 for ; Mon, 06 Jun 2022 07:44:59 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20210112; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=p+0ZvBcKjmOSwrSdkGloKpQ9H3nxamEQCaD2KXxqLfY=; b=FvqOZnH0mRX3uHYAPbiYuJBlNkp/2eOON9LX0IK2ME2E+X+SEuBu6vIe140tN4r16V Gid/QTm+E1i9Z95akrYb3IeO4lo3JNbePJA0mewIlcqvIJQJxkB29tIEx4BLF0/vgQv7 UseTdzeWgpXUJ2iQ7bnsx4bg6tOIC6j/1Ma1q24YANtTbDPeuvSKNTcAtd4yVJkD3qRf IcoSxerQGm35TP7X6isqpGkgNGTHt7Uf4dwrOtW/jgG3JUiVthrJQFLu2T9OOBhTN/iR wFwEj/+bkhX89kXAP/SDvSaIkxGOTuByy3Wlwx+vEbPbURaJ03br06r2zWVxX5jbhyPC zzzA== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=p+0ZvBcKjmOSwrSdkGloKpQ9H3nxamEQCaD2KXxqLfY=; b=i5rKeWK7o5McKledMmTwvLatpcqJuRETUxEHnrZl0tQ5hLDPECbV67OBnEGastaacV qoBC05nZOKwjuVCegDDoU+rIFEeUOfxaKpYaa68DC3GYmj24kbbXHxJlZAqfAJ45kJ8y Y3FmY2/P3xWqo+9OW/Wte6iDhsxfkS/pvmZEHT140YZdF2b5uUYTYPFi/dnZSdovVQi5 OFsJFynKrLBGp9SeYppGuTKKL/sDdFM25nmU6dt0crRd/fXyu9OHomvP347H0FQYYr6U WHUQFXmOs67vI4AncntILrcTeHUqSOepb8S8v0hbRjLkG7OEv8j30tiNV6pcU+yBT5vd jz7g== X-Gm-Message-State: AOAM533hMfHxLCUWZyTdltJjv1LnQG4B/Iw54iiFYUZBiL+zUEwqw2kn aq7vVsTaUoLbTQIO0VqbCM2Anij8UaE= X-Google-Smtp-Source: ABdhPJyBBuXBiPw9Hc0HUpfuJK9Ow56dHwfaER9Skz2FkBXnYXEyjaTX/rEzMZuSeOT+hSRSnuRS0Q== X-Received: by 2002:a17:90b:3b86:b0:1e8:6b89:2bb2 with SMTP id pc6-20020a17090b3b8600b001e86b892bb2mr10368784pjb.15.1654526698723; Mon, 06 Jun 2022 07:44:58 -0700 (PDT) Received: from localhost ([198.11.178.15]) by smtp.gmail.com with ESMTPSA id e5-20020a170902784500b00163f8eb7eb3sm10945646pln.196.2022.06.06.07.44.57 (version=TLS1_2 cipher=ECDHE-ECDSA-AES128-GCM-SHA256 bits=128/128); Mon, 06 Jun 2022 07:44:58 -0700 (PDT) From: Lai Jiangshan To: linux-kernel@vger.kernel.org Cc: Borislav Petkov , Peter Zijlstra , Josh Poimboeuf , Andy Lutomirski , Thomas Gleixner , x86@kernel.org, Lai Jiangshan , Ingo Molnar , Dave Hansen , "H. Peter Anvin" , "Chang S. Bae" , Kees Cook , "Kirill A. Shutemov" , Fenghua Yu Subject: [PATCH V3 7/7] x86/entry: Implement the whole error_entry() as C code Date: Mon, 6 Jun 2022 22:45:09 +0800 Message-Id: <20220606144509.617611-8-jiangshanlai@gmail.com> X-Mailer: git-send-email 2.19.1.6.gb485710b In-Reply-To: <20220606144509.617611-1-jiangshanlai@gmail.com> References: <20220606144509.617611-1-jiangshanlai@gmail.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" From: Lai Jiangshan Implement error_entry() as C code in arch/x86/entry/entry64.c and replace the ASM version of error_entry(). The code might be in the user CR3 and user GS base at the start of the function so it calls __always_inline C function only until the GS and CR3 is switched. No functional change intended and comments are also copied. The C version generally has better readability and easier to be updated/improved. Note: To avoid using goto, the C code has two call sites of sync_regs(). It calls sync_regs() directly after fixup_bad_iret() returns while the ASM code uses JMP instruction to jump to the start of the first call site. The complier uses tail-call-optimization for calling sync_regs(). It uses "JMP sync_regs" while the ASM code uses "CALL+RET". Signed-off-by: Lai Jiangshan --- arch/x86/entry/entry64.c | 69 +++++++++++++++++++++++++++++ arch/x86/entry/entry_64.S | 85 ++---------------------------------- arch/x86/include/asm/proto.h | 1 + arch/x86/include/asm/traps.h | 1 + arch/x86/kernel/traps.c | 2 - 5 files changed, 74 insertions(+), 84 deletions(-) diff --git a/arch/x86/entry/entry64.c b/arch/x86/entry/entry64.c index f7f23800cee4..bd047c329622 100644 --- a/arch/x86/entry/entry64.c +++ b/arch/x86/entry/entry64.c @@ -13,6 +13,8 @@ */ #include =20 +extern unsigned char asm_load_gs_index_gs_change[]; + #ifdef CONFIG_PAGE_TABLE_ISOLATION static __always_inline void pti_switch_to_kernel_cr3(unsigned long user_cr= 3) { @@ -66,3 +68,70 @@ static __always_inline void user_entry_swapgs_and_fence(= void) native_swapgs(); fence_swapgs_user_entry(); } + +/* + * Put pt_regs onto the task stack and switch GS and CR3 if needed. + * The actual stack switch is done in entry_64.S. + * + * Be careful, it might be in the user CR3 and user GS base at the start + * of the function. + */ +asmlinkage __visible __entry_text +struct pt_regs *error_entry(struct pt_regs *eregs) +{ + unsigned long iret_ip =3D (unsigned long)native_irq_return_iret; + + if (user_mode(eregs)) { + /* + * We entered from user mode. + * Switch to kernel gsbase and CR3. + */ + user_entry_swapgs_and_fence(); + switch_to_kernel_cr3(); + + /* Put pt_regs onto the task stack. */ + return sync_regs(eregs); + } + + /* + * There are two places in the kernel that can potentially fault with + * usergs. Handle them here. B stepping K8s sometimes report a + * truncated RIP for IRET exceptions returning to compat mode. Check + * for these here too. + */ + if ((eregs->ip =3D=3D iret_ip) || (eregs->ip =3D=3D (unsigned int)iret_ip= )) { + eregs->ip =3D iret_ip; /* Fix truncated RIP */ + + /* + * We came from an IRET to user mode, so we have user + * gsbase and CR3. Switch to kernel gsbase and CR3: + */ + user_entry_swapgs_and_fence(); + switch_to_kernel_cr3(); + + /* + * Pretend that the exception came from user mode: set up + * pt_regs as if we faulted immediately after IRET and then + * put pt_regs onto the real task stack. + */ + return sync_regs(fixup_bad_iret(eregs)); + } + + /* + * Hack: asm_load_gs_index_gs_change can fail with user gsbase. + * If this happens, fix up gsbase and proceed. We'll fix up the + * exception and land in asm_load_gs_index_gs_change's error + * handler with kernel gsbase. + */ + if (eregs->ip =3D=3D (unsigned long)asm_load_gs_index_gs_change) + native_swapgs(); + + /* + * Issue an LFENCE to prevent GS speculation, regardless of whether + * it is a kernel or user gsbase. + */ + fence_swapgs_kernel_entry(); + + /* Enter from kernel, don't move pt_regs */ + return eregs; +} diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index 4300ba49b5ee..f8322398fe1c 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -348,7 +348,7 @@ SYM_CODE_END(push_and_clear_regs) * own pvops for IRET and load_gs_index(). And it doesn't need to * switch the CR3. So it can skip invoking error_entry(). */ - ALTERNATIVE "call error_entry; movq %rax, %rsp", \ + ALTERNATIVE "movq %rsp, %rdi; call error_entry; movq %rax, %rsp", \ "", X86_FEATURE_XENPV =20 ENCODE_FRAME_POINTER @@ -784,7 +784,7 @@ _ASM_NOKPROBE(common_interrupt_return) SYM_FUNC_START(asm_load_gs_index) FRAME_BEGIN swapgs -.Lgs_change: +SYM_INNER_LABEL(asm_load_gs_index_gs_change, SYM_L_GLOBAL) ANNOTATE_NOENDBR // error_entry movl %edi, %gs 2: ALTERNATIVE "", "mfence", X86_BUG_SWAPGS_FENCE @@ -805,7 +805,7 @@ SYM_FUNC_START(asm_load_gs_index) movl %eax, %gs jmp 2b =20 - _ASM_EXTABLE(.Lgs_change, .Lbad_gs) + _ASM_EXTABLE(asm_load_gs_index_gs_change, .Lbad_gs) =20 SYM_FUNC_END(asm_load_gs_index) EXPORT_SYMBOL(asm_load_gs_index) @@ -1012,85 +1012,6 @@ SYM_CODE_START_LOCAL(paranoid_exit) jmp restore_regs_and_return_to_kernel SYM_CODE_END(paranoid_exit) =20 -/* - * Switch GS and CR3 if needed. - */ -SYM_CODE_START_LOCAL(error_entry) - UNWIND_HINT_FUNC - testb $3, CS+8(%rsp) - jz .Lerror_kernelspace - - /* - * We entered from user mode or we're pretending to have entered - * from user mode due to an IRET fault. - */ - swapgs - FENCE_SWAPGS_USER_ENTRY - /* We have user CR3. Change to kernel CR3. */ - SWITCH_TO_KERNEL_CR3 scratch_reg=3D%rax - - leaq 8(%rsp), %rdi /* arg0 =3D pt_regs pointer */ -.Lerror_entry_from_usermode_after_swapgs: - /* Put us onto the real thread stack. */ - call sync_regs - RET - - /* - * There are two places in the kernel that can potentially fault with - * usergs. Handle them here. B stepping K8s sometimes report a - * truncated RIP for IRET exceptions returning to compat mode. Check - * for these here too. - */ -.Lerror_kernelspace: - leaq native_irq_return_iret(%rip), %rcx - cmpq %rcx, RIP+8(%rsp) - je .Lerror_bad_iret - movl %ecx, %eax /* zero extend */ - cmpq %rax, RIP+8(%rsp) - je .Lbstep_iret - cmpq $.Lgs_change, RIP+8(%rsp) - jne .Lerror_entry_done_lfence - - /* - * hack: .Lgs_change can fail with user gsbase. If this happens, fix up - * gsbase and proceed. We'll fix up the exception and land in - * .Lgs_change's error handler with kernel gsbase. - */ - swapgs - - /* - * Issue an LFENCE to prevent GS speculation, regardless of whether it is= a - * kernel or user gsbase. - */ -.Lerror_entry_done_lfence: - FENCE_SWAPGS_KERNEL_ENTRY - leaq 8(%rsp), %rax /* return pt_regs pointer */ - RET - -.Lbstep_iret: - /* Fix truncated RIP */ - movq %rcx, RIP+8(%rsp) - /* fall through */ - -.Lerror_bad_iret: - /* - * We came from an IRET to user mode, so we have user - * gsbase and CR3. Switch to kernel gsbase and CR3: - */ - swapgs - FENCE_SWAPGS_USER_ENTRY - SWITCH_TO_KERNEL_CR3 scratch_reg=3D%rax - - /* - * Pretend that the exception came from user mode: set up pt_regs - * as if we faulted immediately after IRET. - */ - leaq 8(%rsp), %rdi /* arg0 =3D pt_regs pointer */ - call fixup_bad_iret - mov %rax, %rdi - jmp .Lerror_entry_from_usermode_after_swapgs -SYM_CODE_END(error_entry) - SYM_CODE_START_LOCAL(error_return) UNWIND_HINT_REGS DEBUG_ENTRY_ASSERT_IRQS_OFF diff --git a/arch/x86/include/asm/proto.h b/arch/x86/include/asm/proto.h index 12ef86b19910..199d27fbf903 100644 --- a/arch/x86/include/asm/proto.h +++ b/arch/x86/include/asm/proto.h @@ -15,6 +15,7 @@ void entry_SYSCALL_64(void); void entry_SYSCALL_64_safe_stack(void); void entry_SYSRETQ_unsafe_stack(void); void entry_SYSRETQ_end(void); +extern unsigned char native_irq_return_iret[]; long do_arch_prctl_64(struct task_struct *task, int option, unsigned long = arg2); #endif =20 diff --git a/arch/x86/include/asm/traps.h b/arch/x86/include/asm/traps.h index 47ecfff2c83d..2d00100d3e03 100644 --- a/arch/x86/include/asm/traps.h +++ b/arch/x86/include/asm/traps.h @@ -14,6 +14,7 @@ asmlinkage __visible notrace struct pt_regs *sync_regs(struct pt_regs *ere= gs); asmlinkage __visible notrace struct pt_regs *fixup_bad_iret(struct pt_regs *bad_regs); +asmlinkage __visible notrace struct pt_regs *error_entry(struct pt_regs *e= regs); void __init trap_init(void); asmlinkage __visible noinstr struct pt_regs *vc_switch_off_ist(struct pt_r= egs *eregs); #endif diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c index d62b2cb85cea..f76a15f654c5 100644 --- a/arch/x86/kernel/traps.c +++ b/arch/x86/kernel/traps.c @@ -436,8 +436,6 @@ DEFINE_IDTENTRY_DF(exc_double_fault) #endif =20 #ifdef CONFIG_X86_ESPFIX64 - extern unsigned char native_irq_return_iret[]; - /* * If IRET takes a non-IST fault on the espfix64 stack, then we * end up promoting it to a doublefault. In that case, take --=20 2.19.1.6.gb485710b