From nobody Sun May 10 14:12:37 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 378DFC433F5 for ; Tue, 3 May 2022 03:20:48 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S230327AbiECDYQ (ORCPT ); Mon, 2 May 2022 23:24:16 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:60200 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S230267AbiECDYK (ORCPT ); Mon, 2 May 2022 23:24:10 -0400 Received: from mail-pg1-x52e.google.com (mail-pg1-x52e.google.com [IPv6:2607:f8b0:4864:20::52e]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 46C0829804 for ; Mon, 2 May 2022 20:20:39 -0700 (PDT) Received: by mail-pg1-x52e.google.com with SMTP id q76so10032828pgq.10 for ; Mon, 02 May 2022 20:20:39 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20210112; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=vhqdV4bppKv2NJnLqEsRygbCN8xm6iT8adOuIjwb7P4=; b=hdWE9vWA3cARnrc1tOi5iCsSFHSQFQlN+HOy8FZ5QgW9l4vDv2ycAjuqNCwfWJl2Xh MopVuBcO3BChOZZ5ZVZIf0NEBUlORQmrnZmsC5UiHwUZNRCcghcykVfYeZ8nC3ehn6jl NYWCrN938I/L+otm47htYDvb4PX97qvEMKNOhKhF7ZCUTYZZME4CsX5a5quh9NqTjg2N ZQlR6MkhIkOevdWVJ0Oqyj1ygiLawuu/0smBBEvsls8fB7/oCbsjZDV8nhA10qwZOQSm /D1kG3a2xyvJLIuy7ykeNpPh4tSDinmByGvVmDcumAwhrYGO1Y/V3C5WuzFLhPGwErj/ RI+A== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=vhqdV4bppKv2NJnLqEsRygbCN8xm6iT8adOuIjwb7P4=; b=wMLDS1AxOXim9AMUn9RjceNaTN8UqCKV8jBk4u5YZR+39QJoxLTP3VUNkSJBaneE8M jdHiFfk03ilkzKU/XHZ1YGPC3YNmS9pLfr2qFMoH9CJgy3GKspbIjxJgf5Aj+BxmG4Sn lV5ENOTEjt2QS2hGpWHVFLg6P2vpKRFknfHOVQlOjSgZAhE08d4Bb8SkApT/DfMoF2eW T6U+39Q/06fmm3zIZB27PJxB3aNWskx56MBfcwA7uh5+L+7SWPQgSqDEj2zh2iaICDPx CKMNbnP8V0iVnwFhRXGj5XInjC+rDppBhLhFmZFH3/fSigKk37Qsw86+bao/US/3LI5G W5Rw== X-Gm-Message-State: AOAM533BbbJj30hdHs1mc8i6YaCOmlnRPXSXagT6rqsB/3nrJX4Tgrqh tiMnqKSuh9aywUO9lS6c+AdZBob1XLk= X-Google-Smtp-Source: ABdhPJwzjaJ09Og9yjQOqDMoVOoS0NJ/yoVNZ/p7AqRR5GJ+iJa7is7ictXnEAmbz+Tk9B5Y6x77ZA== X-Received: by 2002:a05:6a00:ad0:b0:4e1:2d96:2ab0 with SMTP id c16-20020a056a000ad000b004e12d962ab0mr13936845pfl.3.1651548038640; Mon, 02 May 2022 20:20:38 -0700 (PDT) Received: from localhost ([47.251.4.198]) by smtp.gmail.com with ESMTPSA id m7-20020a170902f64700b0015e8d4eb1fdsm5378948plg.71.2022.05.02.20.20.37 (version=TLS1_2 cipher=ECDHE-ECDSA-AES128-GCM-SHA256 bits=128/128); Mon, 02 May 2022 20:20:38 -0700 (PDT) From: Lai Jiangshan To: linux-kernel@vger.kernel.org Cc: Borislav Petkov , Peter Zijlstra , Josh Poimboeuf , Andy Lutomirski , Thomas Gleixner , Juergen Gross , x86@kernel.org, Lai Jiangshan , Ingo Molnar , Dave Hansen , "H. Peter Anvin" , "Kirill A. Shutemov" , Fenghua Yu , "Chang S. Bae" Subject: [PATCH V7 1/6] x86/traps: Move pt_regs only in fixup_bad_iret() Date: Tue, 3 May 2022 11:21:02 +0800 Message-Id: <20220503032107.680190-2-jiangshanlai@gmail.com> X-Mailer: git-send-email 2.19.1.6.gb485710b In-Reply-To: <20220503032107.680190-1-jiangshanlai@gmail.com> References: <20220503032107.680190-1-jiangshanlai@gmail.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" From: Lai Jiangshan Always stash the address error_entry() is going to return to, in %r12 and get rid of the void *error_entry_ret; slot in struct bad_iret_stack which was supposed to account for it and pt_regs pushed on the stack. After this, both fixup_bad_iret() and sync_regs() can work on a struct pt_regs pointer directly. Signed-off-by: Lai Jiangshan --- arch/x86/entry/entry_64.S | 5 ++++- arch/x86/include/asm/traps.h | 2 +- arch/x86/kernel/traps.c | 18 ++++++------------ 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index 73d958522b6a..ecbfca3cc18c 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -1061,9 +1061,12 @@ SYM_CODE_START_LOCAL(error_entry) * Pretend that the exception came from user mode: set up pt_regs * as if we faulted immediately after IRET. */ - mov %rsp, %rdi + popq %r12 /* save return addr in %12 */ + movq %rsp, %rdi /* arg0 =3D pt_regs pointer */ call fixup_bad_iret mov %rax, %rsp + ENCODE_FRAME_POINTER + pushq %r12 jmp .Lerror_entry_from_usermode_after_swapgs SYM_CODE_END(error_entry) =20 diff --git a/arch/x86/include/asm/traps.h b/arch/x86/include/asm/traps.h index 35317c5c551d..47ecfff2c83d 100644 --- a/arch/x86/include/asm/traps.h +++ b/arch/x86/include/asm/traps.h @@ -13,7 +13,7 @@ #ifdef CONFIG_X86_64 asmlinkage __visible notrace struct pt_regs *sync_regs(struct pt_regs *ere= gs); asmlinkage __visible notrace -struct bad_iret_stack *fixup_bad_iret(struct bad_iret_stack *s); +struct pt_regs *fixup_bad_iret(struct pt_regs *bad_regs); void __init trap_init(void); asmlinkage __visible noinstr struct pt_regs *vc_switch_off_ist(struct pt_r= egs *eregs); #endif diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c index a4e2efde5d1f..111b18d57a54 100644 --- a/arch/x86/kernel/traps.c +++ b/arch/x86/kernel/traps.c @@ -898,13 +898,7 @@ asmlinkage __visible noinstr struct pt_regs *vc_switch= _off_ist(struct pt_regs *r } #endif =20 -struct bad_iret_stack { - void *error_entry_ret; - struct pt_regs regs; -}; - -asmlinkage __visible noinstr -struct bad_iret_stack *fixup_bad_iret(struct bad_iret_stack *s) +asmlinkage __visible noinstr struct pt_regs *fixup_bad_iret(struct pt_regs= *bad_regs) { /* * This is called from entry_64.S early in handling a fault @@ -914,19 +908,19 @@ struct bad_iret_stack *fixup_bad_iret(struct bad_iret= _stack *s) * just below the IRET frame) and we want to pretend that the * exception came from the IRET target. */ - struct bad_iret_stack tmp, *new_stack =3D - (struct bad_iret_stack *)__this_cpu_read(cpu_tss_rw.x86_tss.sp0) - 1; + struct pt_regs tmp, *new_stack =3D + (struct pt_regs *)__this_cpu_read(cpu_tss_rw.x86_tss.sp0) - 1; =20 /* Copy the IRET target to the temporary storage. */ - __memcpy(&tmp.regs.ip, (void *)s->regs.sp, 5*8); + __memcpy(&tmp.ip, (void *)bad_regs->sp, 5*8); =20 /* Copy the remainder of the stack from the current stack. */ - __memcpy(&tmp, s, offsetof(struct bad_iret_stack, regs.ip)); + __memcpy(&tmp, bad_regs, offsetof(struct pt_regs, ip)); =20 /* Update the entry stack */ __memcpy(new_stack, &tmp, sizeof(tmp)); =20 - BUG_ON(!user_mode(&new_stack->regs)); + BUG_ON(!user_mode(new_stack)); return new_stack; } #endif --=20 2.19.1.6.gb485710b From nobody Sun May 10 14:12:37 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 15622C433F5 for ; Tue, 3 May 2022 03:20:55 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S230331AbiECDYW (ORCPT ); Mon, 2 May 2022 23:24:22 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:60492 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S230329AbiECDYQ (ORCPT ); Mon, 2 May 2022 23:24:16 -0400 Received: from mail-pf1-x432.google.com (mail-pf1-x432.google.com [IPv6:2607:f8b0:4864:20::432]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id C6D7331222 for ; Mon, 2 May 2022 20:20:45 -0700 (PDT) Received: by mail-pf1-x432.google.com with SMTP id a11so13769498pff.1 for ; Mon, 02 May 2022 20:20:45 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20210112; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=iWDAJ1Qy8kTssfxxRg0WHjshQinm3EhF18pt1sufO9Y=; b=TAiC/zyrxBX9SqMU1DDy9Sxt5L4RS6IA7H2pvjursJmu3PEHSSd56aFVd+t86iGmQa N6XZwDIvfh1L0g6HcPLcXnsH/pu3o1aBSudWxGcQsgdSK7hoK8H7+SHad2QSnwnv/Ory aNmUC9J6G9Ejnf+HKD+YqB7UW3iXTOmjnjSCWhvoXYbuFx4g9JeRR4gEZBEDpwKeCCMu wVa+uSEOF4L87F0FSKkP/Kjh8XU/jUA8yBy1SZaFWZTyx26/qjxPdc4wdJKO52WKb2lj XzTrI3BE7evc4oxnB6l8cVjv9QODG4bl5ONcR8CA1Ze1Y1s0g9w0C0PDKPkLdCYzxmKC VUUA== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=iWDAJ1Qy8kTssfxxRg0WHjshQinm3EhF18pt1sufO9Y=; b=IBFePCwCnG5vCsjgiYAcyHpg6qr0k6Yudl3+3rihViqWlIbc+5TDfsX0e3MenqO1CS z0RduGIBCgkYzyEnYeRngOs8eawVbTONreWIGHG0c9JHmnkoJrfM6GANxmncHlmC2hAU w2c5GP0qSX3CvJYSHElGqMHOdd5twvByieOZR+KD8i7wX2O/rYv4WGqtDb20sbUHLXQg DH3GYmfwcD8FoMimG+mT/QnNADzHtH2AqG/UwdwZO9qG1UpFyFgyE4rbGIzTlol0IKfP wGjMrSgUUtRBfb1Ze0kQ6QZ2tEY1hoxg6VOzzOJqpmi4djdm3FvS0zB1u4Blq2zTrkQ3 u2Dg== X-Gm-Message-State: AOAM531YNUKm2O2mhdnpCweS1jKTDnQwZvRlyGrPxokDILr/x73NAwbN pt6Y2Csa2+T1wJ+BCxUTehKhrU+2GWM= X-Google-Smtp-Source: ABdhPJzn2FwJe7lQZYHbqR8cUASuvRxAE09Hn9KskTR7Rk4WW2OZ5y/YN10qgZcW8du284vM7/NffQ== X-Received: by 2002:a65:6955:0:b0:380:64fd:a2dd with SMTP id w21-20020a656955000000b0038064fda2ddmr11930352pgq.383.1651548045134; Mon, 02 May 2022 20:20:45 -0700 (PDT) Received: from localhost ([47.251.4.198]) by smtp.gmail.com with ESMTPSA id q3-20020a170902edc300b0015e8d4eb269sm5374266plk.179.2022.05.02.20.20.43 (version=TLS1_2 cipher=ECDHE-ECDSA-AES128-GCM-SHA256 bits=128/128); Mon, 02 May 2022 20:20:44 -0700 (PDT) From: Lai Jiangshan To: linux-kernel@vger.kernel.org Cc: Borislav Petkov , Peter Zijlstra , Josh Poimboeuf , Andy Lutomirski , Thomas Gleixner , Juergen Gross , x86@kernel.org, Lai Jiangshan , Ingo Molnar , Dave Hansen , "H. Peter Anvin" Subject: [PATCH V7 2/6] x86/entry: Switch the stack after error_entry() returns Date: Tue, 3 May 2022 11:21:03 +0800 Message-Id: <20220503032107.680190-3-jiangshanlai@gmail.com> X-Mailer: git-send-email 2.19.1.6.gb485710b In-Reply-To: <20220503032107.680190-1-jiangshanlai@gmail.com> References: <20220503032107.680190-1-jiangshanlai@gmail.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" From: Lai Jiangshan error_entry() calls sync_regs(), and fixup_bad_iret() before sync_regs() if it is a fault from bad IRET, to copy the pt_regs to the kernel stack and switches the kernel stack directly after sync_regs(). But error_entry() itself is also a function call, so the code has to stash the address error_entry() is going to return to, in %r12 and makes the work complicated. Move the code of switching stack after error_entry() and get rid of the need to handle the return address. Signed-off-by: Lai Jiangshan --- arch/x86/entry/entry_64.S | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index ecbfca3cc18c..ca3e99e08a44 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -326,6 +326,8 @@ SYM_CODE_END(ret_from_fork) .macro idtentry_body cfunc has_error_code:req =20 call error_entry + movq %rax, %rsp /* switch to the task stack if from userspace */ + ENCODE_FRAME_POINTER UNWIND_HINT_REGS =20 movq %rsp, %rdi /* pt_regs pointer into 1st argument*/ @@ -1002,14 +1004,10 @@ SYM_CODE_START_LOCAL(error_entry) /* We have user CR3. Change to kernel CR3. */ SWITCH_TO_KERNEL_CR3 scratch_reg=3D%rax =20 + leaq 8(%rsp), %rdi /* arg0 =3D pt_regs pointer */ .Lerror_entry_from_usermode_after_swapgs: /* Put us onto the real thread stack. */ - popq %r12 /* save return addr in %12 */ - movq %rsp, %rdi /* arg0 =3D pt_regs pointer */ call sync_regs - movq %rax, %rsp /* switch stack */ - ENCODE_FRAME_POINTER - pushq %r12 RET =20 /* @@ -1041,6 +1039,7 @@ SYM_CODE_START_LOCAL(error_entry) */ .Lerror_entry_done_lfence: FENCE_SWAPGS_KERNEL_ENTRY + leaq 8(%rsp), %rax /* return pt_regs pointer */ RET =20 .Lbstep_iret: @@ -1061,12 +1060,9 @@ SYM_CODE_START_LOCAL(error_entry) * Pretend that the exception came from user mode: set up pt_regs * as if we faulted immediately after IRET. */ - popq %r12 /* save return addr in %12 */ - movq %rsp, %rdi /* arg0 =3D pt_regs pointer */ + leaq 8(%rsp), %rdi /* arg0 =3D pt_regs pointer */ call fixup_bad_iret - mov %rax, %rsp - ENCODE_FRAME_POINTER - pushq %r12 + mov %rax, %rdi jmp .Lerror_entry_from_usermode_after_swapgs SYM_CODE_END(error_entry) =20 --=20 2.19.1.6.gb485710b From nobody Sun May 10 14:12:37 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id C6AD3C433EF for ; Tue, 3 May 2022 03:21:04 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S230347AbiECDYc (ORCPT ); Mon, 2 May 2022 23:24:32 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:60866 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S230337AbiECDYX (ORCPT ); Mon, 2 May 2022 23:24:23 -0400 Received: from mail-pf1-x42f.google.com (mail-pf1-x42f.google.com [IPv6:2607:f8b0:4864:20::42f]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 7B2E533E0A for ; Mon, 2 May 2022 20:20:52 -0700 (PDT) Received: by mail-pf1-x42f.google.com with SMTP id h1so13739739pfv.12 for ; Mon, 02 May 2022 20:20:52 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20210112; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=x61TjLKwnlPP/1Q/z1UHB85PFlxeFJXks6RYgmVkwuc=; b=iXYClFs+HWZ4g95as2fyx9dN6umKDVfdPfMOgeeXq1YD+Crki+bpt7fhpQ4rcCe/VX mADF6JsQl3FLiqepNL/BA7AB0vXrUxTtOGj9Z/sSEqtKqbyMifwxk47RG0ZTnWxSzHJe bZ4XgjrHfcNMlq/Y+Ao2ao8wNz1pb8PWjdFtbDbnYxwKRdHn6u6BbZ4cqr7XOSh4Jj1B KpxrPqAnTsSP2aUHBAlewzK2SwgqHEItUjSD3FT4vjsHkU1DZOWb3sGomgCatb76wBrn Bh4YC2ryd9RNUBhjXRJXelmWBk8ct2Mhbc3Llxzmhb4s4RN+8gIDRpZXXXxyb3nNNUBw Ex6A== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=x61TjLKwnlPP/1Q/z1UHB85PFlxeFJXks6RYgmVkwuc=; b=aSHhkWdovUVGoW4t5qQFH3BqfmRLfDOXbtSk6R78ZQvAkMF4MHVRCX/2ZRgpQYiWyX jiaXREEuBCxS6dsjJsSSoHhIacId9f2j3MyhPCBt2susRPB13ZacqOG4TYZUCD2s6Ael P4NRHsfvlJlJx8S1mP0xiPfSEptT6SgxBJskJ4Co8RC+0SxgoU8VxxEAEaJoZ0ZYy7QG Taikgtr1r9t18fJXf6w9zS1fdECAOCu77pRYk2FWlZlnVWn0K4zy9eQCdi8rb5IlRiaZ zWe2ID8PEZ9aqIxwmiAjmL/ZojpkerfBPdIE8VIeI7UFApbpgwTapGx1gPe5FvRwbgcK uxPQ== X-Gm-Message-State: AOAM5313kMKI/vOK5dqGxPSxF9M+/DiTqkCTku4X3mvQNLIqDZ5wMEad VnfA+8SCIt+HWQ3ccNSKoT3x4hTVfkw= X-Google-Smtp-Source: ABdhPJwIJaYiXyDx8XCMWi6AiESvXSZzHhrbf5VnJ2C4/dkUn4HNJwq/+9PI8RUCXpaA6TSn/z+jPw== X-Received: by 2002:a05:6a00:1352:b0:50d:ecff:f06c with SMTP id k18-20020a056a00135200b0050decfff06cmr7592097pfu.6.1651548051873; Mon, 02 May 2022 20:20:51 -0700 (PDT) Received: from localhost ([47.251.4.198]) by smtp.gmail.com with ESMTPSA id z21-20020aa791d5000000b0050dc76281ddsm5395704pfa.183.2022.05.02.20.20.50 (version=TLS1_2 cipher=ECDHE-ECDSA-AES128-GCM-SHA256 bits=128/128); Mon, 02 May 2022 20:20:51 -0700 (PDT) From: Lai Jiangshan To: linux-kernel@vger.kernel.org Cc: Borislav Petkov , Peter Zijlstra , Josh Poimboeuf , Andy Lutomirski , Thomas Gleixner , Juergen Gross , x86@kernel.org, Lai Jiangshan , Ingo Molnar , Dave Hansen , "H. Peter Anvin" Subject: [PATCH V7 3/6] x86/entry: Move PUSH_AND_CLEAR_REGS out of error_entry() Date: Tue, 3 May 2022 11:21:04 +0800 Message-Id: <20220503032107.680190-4-jiangshanlai@gmail.com> X-Mailer: git-send-email 2.19.1.6.gb485710b In-Reply-To: <20220503032107.680190-1-jiangshanlai@gmail.com> References: <20220503032107.680190-1-jiangshanlai@gmail.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" From: Lai Jiangshan The macro idtentry calls error_entry() unconditionally even on XENPV. But the code XENPV needs in error_entry() is PUSH_AND_CLEAR_REGS only. And PUSH_AND_CLEAR_REGS in error_entry() makes the stack not return to its original place when the function returns, which means it is not possible to convert it to a C function. Move PUSH_AND_CLEAR_REGS out of error_entry(), add a function to wrap PUSH_AND_CLEAR_REGS and call it before error_entry(). It will allow for error_entry() to be not called on XENPV and for error_entry() to be converted to C code. Cc: Juergen Gross Signed-off-by: Lai Jiangshan Reviewed-by: Juergen Gross --- arch/x86/entry/entry_64.S | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index ca3e99e08a44..b1cef3b0a7ab 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -318,6 +318,14 @@ SYM_CODE_END(ret_from_fork) #endif .endm =20 +/* Save all registers in pt_regs */ +SYM_CODE_START_LOCAL(push_and_clear_regs) + UNWIND_HINT_FUNC + PUSH_AND_CLEAR_REGS save_ret=3D1 + ENCODE_FRAME_POINTER 8 + RET +SYM_CODE_END(push_and_clear_regs) + /** * idtentry_body - Macro to emit code calling the C function * @cfunc: C function to be called @@ -325,6 +333,9 @@ SYM_CODE_END(ret_from_fork) */ .macro idtentry_body cfunc has_error_code:req =20 + call push_and_clear_regs + UNWIND_HINT_REGS + call error_entry movq %rax, %rsp /* switch to the task stack if from userspace */ ENCODE_FRAME_POINTER @@ -985,13 +996,11 @@ SYM_CODE_START_LOCAL(paranoid_exit) SYM_CODE_END(paranoid_exit) =20 /* - * Save all registers in pt_regs, and switch GS if needed. + * Switch GS and CR3 if needed. */ SYM_CODE_START_LOCAL(error_entry) UNWIND_HINT_FUNC cld - PUSH_AND_CLEAR_REGS save_ret=3D1 - ENCODE_FRAME_POINTER 8 testb $3, CS+8(%rsp) jz .Lerror_kernelspace =20 --=20 2.19.1.6.gb485710b From nobody Sun May 10 14:12:37 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 8519FC433F5 for ; Tue, 3 May 2022 03:21:08 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S230353AbiECDYg (ORCPT ); Mon, 2 May 2022 23:24:36 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:33074 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S230338AbiECDY3 (ORCPT ); Mon, 2 May 2022 23:24:29 -0400 Received: from mail-pj1-x1036.google.com (mail-pj1-x1036.google.com [IPv6:2607:f8b0:4864:20::1036]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id B18AF35255 for ; Mon, 2 May 2022 20:20:58 -0700 (PDT) Received: by mail-pj1-x1036.google.com with SMTP id j8-20020a17090a060800b001cd4fb60dccso974162pjj.2 for ; Mon, 02 May 2022 20:20:58 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20210112; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=xotWSQoi2wlC3vMI8lQ8RpO96nR6JNjKt6GYA46wjrE=; b=j2TkGrSUFai8FQ8ertrRrE8Ty6ZPlVq1zCXtVp+22pyjlvLpM7dE7qX6I+rdhB8I32 Z4T+adbpQ3dtQJMCuFGFf4Oig+WmhCOI//Se09Q4L4fmiTimRII7dclbz0/haNaYHlsE NMHlx1+8ORo0wu/APD+tMPNoVXaaAqJh+cXCSdcAOLiTAp5OoY76FJ8f4jjljHuDB/ML yjHJ8ZuEYlLhvaEeVwDkrWyp9TnGmcBQMOgRv1FQGGxUit7cGzXx+QsqgDAyaBHHAJ36 Ic3ubzDoUpyX8c1mjUV13OUdwMj4jfC5Hr4ZckXWAl6kyZ0QyIsqRrUyhXGJsv2KHAiW 1XCw== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=xotWSQoi2wlC3vMI8lQ8RpO96nR6JNjKt6GYA46wjrE=; b=nL/7bp4Wd62cLEyQ9NGfjWhmxqX0jVrZa58JA6I3sxknVREOuh/Wlr45IMSXeV/jXO DOdTBOx8f2jUBKeOn0k/us1krZyidsfLFOFrhgTzPHWAYZ13bP+QnoOdNI7+PAbvdokt E8FGMHoNuLdeXWl958CGc4wKWT7bS9KLmvYlHZWsrrlxu89M59V781geVekhwehJVogt 5NRnvxujLUlcW8xd75WwPUCsKpixEDO4L+ONGGtIT1+FNN+dRDkyM/VyfjIDXX1jMc4m r6fpPG7llje9ZP0dKvALSDmVfBPp0nNNzYUZT8X4slOgd2N7WjmUSfiFO9FEzNWGmxGA AcAA== X-Gm-Message-State: AOAM530i1nLhNnjEQzEMI94muhVkJdiYqQPBJr/5BjmH6GlcDp6mHFkn /NjUawmJdthmLCfVRrirWLN/RcoFTdo= X-Google-Smtp-Source: ABdhPJzmp6UBaLfASoGK81unwVdXeaJ+Sk4uU9HjWwtSBuFgFdJH66RGjL9tXeLPF7LkY2H8y7aMkw== X-Received: by 2002:a17:902:f684:b0:15e:8c4a:c54b with SMTP id l4-20020a170902f68400b0015e8c4ac54bmr14463764plg.21.1651548058099; Mon, 02 May 2022 20:20:58 -0700 (PDT) Received: from localhost ([47.251.4.198]) by smtp.gmail.com with ESMTPSA id n5-20020a170902f60500b0015e8d4eb206sm5392698plg.80.2022.05.02.20.20.56 (version=TLS1_2 cipher=ECDHE-ECDSA-AES128-GCM-SHA256 bits=128/128); Mon, 02 May 2022 20:20:57 -0700 (PDT) From: Lai Jiangshan To: linux-kernel@vger.kernel.org Cc: Borislav Petkov , Peter Zijlstra , Josh Poimboeuf , Andy Lutomirski , Thomas Gleixner , Juergen Gross , x86@kernel.org, Lai Jiangshan , Ingo Molnar , Dave Hansen , "H. Peter Anvin" Subject: [PATCH V7 4/6] x86/entry: Move cld to the start of idtentry macro Date: Tue, 3 May 2022 11:21:05 +0800 Message-Id: <20220503032107.680190-5-jiangshanlai@gmail.com> X-Mailer: git-send-email 2.19.1.6.gb485710b In-Reply-To: <20220503032107.680190-1-jiangshanlai@gmail.com> References: <20220503032107.680190-1-jiangshanlai@gmail.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" From: Lai Jiangshan Make it next to CLAC Suggested-by: Peter Zijlstra Signed-off-by: Lai Jiangshan --- arch/x86/entry/entry_64.S | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index b1cef3b0a7ab..ab6ab6d3dab5 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -371,6 +371,7 @@ SYM_CODE_START(\asmsym) UNWIND_HINT_IRET_REGS offset=3D\has_error_code*8 ENDBR ASM_CLAC + cld =20 .if \has_error_code =3D=3D 0 pushq $-1 /* ORIG_RAX: no syscall to restart */ @@ -439,6 +440,7 @@ SYM_CODE_START(\asmsym) UNWIND_HINT_IRET_REGS ENDBR ASM_CLAC + cld =20 pushq $-1 /* ORIG_RAX: no syscall to restart */ =20 @@ -495,6 +497,7 @@ SYM_CODE_START(\asmsym) UNWIND_HINT_IRET_REGS ENDBR ASM_CLAC + cld =20 /* * If the entry is from userspace, switch stacks and treat it as @@ -557,6 +560,7 @@ SYM_CODE_START(\asmsym) UNWIND_HINT_IRET_REGS offset=3D8 ENDBR ASM_CLAC + cld =20 /* paranoid_entry returns GS information for paranoid_exit in EBX. */ call paranoid_entry @@ -882,7 +886,6 @@ SYM_CODE_END(xen_failsafe_callback) */ SYM_CODE_START_LOCAL(paranoid_entry) UNWIND_HINT_FUNC - cld PUSH_AND_CLEAR_REGS save_ret=3D1 ENCODE_FRAME_POINTER 8 =20 @@ -1000,7 +1003,6 @@ SYM_CODE_END(paranoid_exit) */ SYM_CODE_START_LOCAL(error_entry) UNWIND_HINT_FUNC - cld testb $3, CS+8(%rsp) jz .Lerror_kernelspace =20 @@ -1134,6 +1136,7 @@ SYM_CODE_START(asm_exc_nmi) */ =20 ASM_CLAC + cld =20 /* Use %rdx as our temp variable throughout */ pushq %rdx @@ -1153,7 +1156,6 @@ SYM_CODE_START(asm_exc_nmi) */ =20 swapgs - cld FENCE_SWAPGS_USER_ENTRY SWITCH_TO_KERNEL_CR3 scratch_reg=3D%rdx movq %rsp, %rdx --=20 2.19.1.6.gb485710b From nobody Sun May 10 14:12:37 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 5DAC8C433F5 for ; Tue, 3 May 2022 03:21:18 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S230384AbiECDYq (ORCPT ); Mon, 2 May 2022 23:24:46 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:33586 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S230355AbiECDYg (ORCPT ); Mon, 2 May 2022 23:24:36 -0400 Received: from mail-pl1-x62c.google.com (mail-pl1-x62c.google.com [IPv6:2607:f8b0:4864:20::62c]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 689E03B283 for ; Mon, 2 May 2022 20:21:05 -0700 (PDT) Received: by mail-pl1-x62c.google.com with SMTP id s14so13970935plk.8 for ; Mon, 02 May 2022 20:21:05 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20210112; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=VgGgMxYprzA3l/xL3EqCp11c0RZBUqA2IVp2y5zwqwA=; b=S+Zz45MWXks4nx4ynNsiBuY3M56T6z4EiXW0WpmvNvAkBcDnPi8B/Nxm2oX3Z21Nlh o9DHs790/56cgU/3G6nNFMfyqo94VMk0tay8JMcYI3+N971ZUh3amcaL689ltG6XmDuY IK2GaVJXyrqdi7LJ+/F3TB5iEgSQiq7w5IUtDss4ocWgJxoLIwITm2xPzG8440Bo0dDT yBXkbyKCNdx8kCSIqKduqeO3rZB/PqYVuTCEx5wvgUwqMP5HX6nW8rFauA1sRdEDHGQ7 oPQPH2ApTrqLUkj+keH1EdMrx4W7pEAFHOyQzYwcTHI8fptOHu6Ny+CLiOemC2rKRET6 KkDA== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=VgGgMxYprzA3l/xL3EqCp11c0RZBUqA2IVp2y5zwqwA=; b=BJnx5WT/kNBZ4FR7nLW7S7t3v/fsXQ/AMF9/nh7DfG5dNXBfe8yvSUbW4uGchtRBWZ p9UAvK/B7oseieUVEOvum3ZxPKu1mgclD2CZHB2jstnHpRuC8MmYe56/xoDJe6ZxJAL2 z3pwTZDeJtryZVL8WH6hBTlQwG8KiqHgdif86nMCHiWRmaZy0B8tEuJ8cSeOQciCEkuZ PiaOMr9Tg9P6F+6+29P1BVWsyVJ/h6CbonbVXVB5koYfaB4y69lb7SgFFl9tIXDImpFu T1Wjr1zZcaZsNNDSDsX6UQMt/obKhI7JEbVIvpx82geDqhZEVM81u6MTO7DHJy+u/Srz ofmA== X-Gm-Message-State: AOAM530wUoIWTo7CHeJRH0oFjtG3DU5iYYa9/0MLDwnwK6POkGsqmVnf 4YtnnrYO8FXE2VlRR+qrjOcNrEz4nGo= X-Google-Smtp-Source: ABdhPJzCceSsy+gBasYlsMUGbi/7N68x5v0EIJSpy2AZTAAtMRoJOfWLebTQyavueQvCLGEv1Mc4Xw== X-Received: by 2002:a17:90a:488c:b0:1c7:b62e:8e8c with SMTP id b12-20020a17090a488c00b001c7b62e8e8cmr2477589pjh.157.1651548064578; Mon, 02 May 2022 20:21:04 -0700 (PDT) Received: from localhost ([47.251.4.198]) by smtp.gmail.com with ESMTPSA id a17-20020a631a51000000b003c14af5060fsm11896873pgm.39.2022.05.02.20.21.03 (version=TLS1_2 cipher=ECDHE-ECDSA-AES128-GCM-SHA256 bits=128/128); Mon, 02 May 2022 20:21:04 -0700 (PDT) From: Lai Jiangshan To: linux-kernel@vger.kernel.org Cc: Borislav Petkov , Peter Zijlstra , Josh Poimboeuf , Andy Lutomirski , Thomas Gleixner , Juergen Gross , x86@kernel.org, Lai Jiangshan , Ingo Molnar , Dave Hansen , "H. Peter Anvin" Subject: [PATCH V7 5/6] x86/entry: Don't call error_entry() for XENPV Date: Tue, 3 May 2022 11:21:06 +0800 Message-Id: <20220503032107.680190-6-jiangshanlai@gmail.com> X-Mailer: git-send-email 2.19.1.6.gb485710b In-Reply-To: <20220503032107.680190-1-jiangshanlai@gmail.com> References: <20220503032107.680190-1-jiangshanlai@gmail.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" From: Lai Jiangshan When in XENPV, it is already in the task stack, and it can't fault for native_iret() nor native_load_gs_index() since XENPV uses its own pvops for IRET and load_gs_index(). And it doesn't need to switch the CR3. So there is no reason to call error_entry() in XENPV. Cc: Juergen Gross Signed-off-by: Lai Jiangshan Reviewed-by: Juergen Gross --- arch/x86/entry/entry_64.S | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index ab6ab6d3dab5..062aa9d95961 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -336,8 +336,17 @@ SYM_CODE_END(push_and_clear_regs) call push_and_clear_regs UNWIND_HINT_REGS =20 - call error_entry - movq %rax, %rsp /* switch to the task stack if from userspace */ + /* + * Call error_entry() and switch to the task stack if from userspace. + * + * When in XENPV, it is already in the task stack, and it can't fault + * for native_iret() nor native_load_gs_index() since XENPV uses its + * own pvops for IRET and load_gs_index(). And it doesn't need to + * switch the CR3. So it can skip invoking error_entry(). + */ + ALTERNATIVE "call error_entry; movq %rax, %rsp", \ + "", X86_FEATURE_XENPV + ENCODE_FRAME_POINTER UNWIND_HINT_REGS =20 --=20 2.19.1.6.gb485710b From nobody Sun May 10 14:12:37 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 8A631C433EF for ; Tue, 3 May 2022 03:21:25 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S230388AbiECDYy (ORCPT ); Mon, 2 May 2022 23:24:54 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:34074 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S230375AbiECDYo (ORCPT ); Mon, 2 May 2022 23:24:44 -0400 Received: from mail-pj1-x1029.google.com (mail-pj1-x1029.google.com [IPv6:2607:f8b0:4864:20::1029]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 4A3863B55A for ; Mon, 2 May 2022 20:21:13 -0700 (PDT) Received: by mail-pj1-x1029.google.com with SMTP id t11-20020a17090ad50b00b001d95bf21996so1123918pju.2 for ; Mon, 02 May 2022 20:21:13 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20210112; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=lB48jYFG9Z5zUm2kKQoJTg5qAQTn/y96aJT2/yNcQUo=; b=bNwc2L9/9lr/9BjYkGR2Q/bEyjWI5kUABXvSyOEuHiYHzhi8mnh7RrrM0nDQoXJRSI VGDhsSxCpigoGwHMi1/p3+ncjKt+EEvzeqUlpnmbuCsLb2010lHLrLWNOvuwzZCMuntl +KRYINwRtFPi9n2O1pff7fsgmUUZHVWtOQm7UgswTykaN23k3KE8+bEgnLgOBqrlVh9a IXV3y93EL9xfBOlQsjMY9GzeswiVa1YxeSHMDcE/CU81N+KV2AMgCQFoyy9EREJS1/BI u//VJdn2ySPmuxQEOzN0mkxcPJ6P9eMWekFdDaIyf9CHwbpluE3rD2bVCRQMSUIGS5am wb1A== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=lB48jYFG9Z5zUm2kKQoJTg5qAQTn/y96aJT2/yNcQUo=; b=pQheTeV89fuFdLI6f3x9o9W8vwHoTFZiSdLyAAxdvBBojlGxeZS8QOzD4+zUspmrKI tp+1S2Xawcm5dh9AgzwS1k30xiTipyvuGeSn9TCQOPVTuvmTAZhwRa9O5aCKyeYVAg/s 0zjJtwI47/nsyc8miiAoqgStJRsf9fXqQNXYnbxAkTOWCf2I6raiXo+hgjDX5qoemNKP 9O4TbnGo8s1ZSsdxTnTDIUJVnOX5/ZJXgnJz0WE41v0AGsxM7LjIidRUuqrT0yONNMQK Dr9iXhau6FCEC8I/eVjYvV3Tf6EUMWnGGJs75zzPLGsMVqBasNNRpHQPo4UjVQY+lxVk c01A== X-Gm-Message-State: AOAM5335fbLoXTrLZPOeUma6yWz4kQvRjibFTEpKY2Eq3id8vpFRBZVO LIXrtckitv4nw19pqKb6UIJkAonxxSw= X-Google-Smtp-Source: ABdhPJxXP1AAmaNEXnDBeCM5tlwV8bGPaeAYGTaENQQgjK6iIb7xqWExvBqa3WUNtjWAIs73YFSHVg== X-Received: by 2002:a17:90b:4c85:b0:1dc:5778:5344 with SMTP id my5-20020a17090b4c8500b001dc57785344mr2579726pjb.8.1651548072270; Mon, 02 May 2022 20:21:12 -0700 (PDT) Received: from localhost ([47.251.4.198]) by smtp.gmail.com with ESMTPSA id p12-20020a634f4c000000b003c14af50620sm11959455pgl.56.2022.05.02.20.21.11 (version=TLS1_2 cipher=ECDHE-ECDSA-AES128-GCM-SHA256 bits=128/128); Mon, 02 May 2022 20:21:11 -0700 (PDT) From: Lai Jiangshan To: linux-kernel@vger.kernel.org Cc: Borislav Petkov , Peter Zijlstra , Josh Poimboeuf , Andy Lutomirski , Thomas Gleixner , Juergen Gross , x86@kernel.org, Lai Jiangshan , Ingo Molnar , Dave Hansen , "H. Peter Anvin" , "Kirill A. Shutemov" Subject: [PATCH V7 6/6] x86/entry: Convert SWAPGS to swapgs and remove the definition of SWAPGS Date: Tue, 3 May 2022 11:21:07 +0800 Message-Id: <20220503032107.680190-7-jiangshanlai@gmail.com> X-Mailer: git-send-email 2.19.1.6.gb485710b In-Reply-To: <20220503032107.680190-1-jiangshanlai@gmail.com> References: <20220503032107.680190-1-jiangshanlai@gmail.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" From: Lai Jiangshan XENPV doesn't use swapgs_restore_regs_and_return_to_usermode(), error_entry() and the code between entry_SYSENTER_compat() and entry_SYSENTER_compat_after_hwframe. Change the PV-compatible SWAPGS to the ASM instruction swapgs in these places. Also remove the definition of SWAPGS since there is no user of the SWAPGS anymore. Reviewed-by: Juergen Gross Signed-off-by: Lai Jiangshan --- arch/x86/entry/entry_64.S | 6 +++--- arch/x86/entry/entry_64_compat.S | 2 +- arch/x86/include/asm/irqflags.h | 8 -------- 3 files changed, 4 insertions(+), 12 deletions(-) diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index 062aa9d95961..312186612f4e 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -1019,7 +1019,7 @@ SYM_CODE_START_LOCAL(error_entry) * We entered from user mode or we're pretending to have entered * from user mode due to an IRET fault. */ - SWAPGS + swapgs FENCE_SWAPGS_USER_ENTRY /* We have user CR3. Change to kernel CR3. */ SWITCH_TO_KERNEL_CR3 scratch_reg=3D%rax @@ -1051,7 +1051,7 @@ SYM_CODE_START_LOCAL(error_entry) * gsbase and proceed. We'll fix up the exception and land in * .Lgs_change's error handler with kernel gsbase. */ - SWAPGS + swapgs =20 /* * Issue an LFENCE to prevent GS speculation, regardless of whether it is= a @@ -1072,7 +1072,7 @@ SYM_CODE_START_LOCAL(error_entry) * We came from an IRET to user mode, so we have user * gsbase and CR3. Switch to kernel gsbase and CR3: */ - SWAPGS + swapgs FENCE_SWAPGS_USER_ENTRY SWITCH_TO_KERNEL_CR3 scratch_reg=3D%rax =20 diff --git a/arch/x86/entry/entry_64_compat.S b/arch/x86/entry/entry_64_com= pat.S index 4fdb007cddbd..c5aeb0819707 100644 --- a/arch/x86/entry/entry_64_compat.S +++ b/arch/x86/entry/entry_64_compat.S @@ -50,7 +50,7 @@ SYM_CODE_START(entry_SYSENTER_compat) UNWIND_HINT_EMPTY ENDBR /* Interrupts are off on entry. */ - SWAPGS + swapgs =20 pushq %rax SWITCH_TO_KERNEL_CR3 scratch_reg=3D%rax diff --git a/arch/x86/include/asm/irqflags.h b/arch/x86/include/asm/irqflag= s.h index 111104d1c2cd..7793e52d6237 100644 --- a/arch/x86/include/asm/irqflags.h +++ b/arch/x86/include/asm/irqflags.h @@ -137,14 +137,6 @@ static __always_inline void arch_local_irq_restore(uns= igned long flags) if (!arch_irqs_disabled_flags(flags)) arch_local_irq_enable(); } -#else -#ifdef CONFIG_X86_64 -#ifdef CONFIG_XEN_PV -#define SWAPGS ALTERNATIVE "swapgs", "", X86_FEATURE_XENPV -#else -#define SWAPGS swapgs -#endif -#endif #endif /* !__ASSEMBLY__ */ =20 #endif --=20 2.19.1.6.gb485710b