From nobody Sat Feb 7 22:46:48 2026 Received: from mail.zytor.com (terminus.zytor.com [198.137.202.136]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 6AA3BA933 for ; Thu, 22 Aug 2024 07:40:09 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=198.137.202.136 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1724312411; cv=none; b=PCOyyd0Y97hZ7kRMIPGpe/JBFiI3xSIhkWMiOLiCRRe167ygxSDgNnDGY9KnUxMo6ig6Glg2wG4EbHnFgj9icL7VsGK0tP203pj7OMKEsOgfTZI8DD2yGt1UYflzOkXsmlOvjav+vbKDjGEn55acICrwnx5dRGq+bLPiHspDsyo= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1724312411; c=relaxed/simple; bh=XEMrGp/hJElB/rBq0I5kW6pqL+b6/Ysam0r70alkjtE=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=mbLivnKkdUjnPsxMjR9pgM8YfiE6kCsN3y/xe6cnsfGOUrpxgi15ARqjPz8a3hcKqE2D3OTku4JS2WzdXB197UcNOqLaNq0dk6yB5lTUqgm5HBgWkDvdMbaMyMYC1PwxpuiQm2onVdtqGmE01eYd1gYQvNLOqWcPGIS0H9SoOMI= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=zytor.com; spf=pass smtp.mailfrom=zytor.com; dkim=pass (2048-bit key) header.d=zytor.com header.i=@zytor.com header.b=uZwpleLd; arc=none smtp.client-ip=198.137.202.136 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=zytor.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=zytor.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=zytor.com header.i=@zytor.com header.b="uZwpleLd" Received: from terminus.zytor.com (terminus.zytor.com [IPv6:2607:7c80:54:3:0:0:0:136]) (authenticated bits=0) by mail.zytor.com (8.17.2/8.17.1) with ESMTPSA id 47M7d7s62176365 (version=TLSv1.3 cipher=TLS_AES_256_GCM_SHA384 bits=256 verify=NO); Thu, 22 Aug 2024 00:39:12 -0700 DKIM-Filter: OpenDKIM Filter v2.11.0 mail.zytor.com 47M7d7s62176365 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=zytor.com; s=2024081601; t=1724312352; bh=p2ysv/OVc0XonUsdqfQCe5lPlcqRKnShtxUnJ8YFWxA=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=uZwpleLdx2BxmefnaartaDug1Ne5hkEjo+epKjpwjxAsdVw1qDltDyfmUkV0Jdvtg RexcTlquj/29+8sdbYM6EJBrpEywPV0eAWOmp6DM7m5Udk0JTvFStb/xIZOg34Qanc oVifEAMCQ80RH47YCRLVAvxknWt1PIY4G+4GkTfbR6RxWMPI8z7tzhrxfDdVQgPyyl 0NtPETJHf6qNpA0iSWFFdZ0b27QHbRaT99rsOIGHkHvwm9tp8qxpIoaj0R4ZdHIrlh vFEDkm2FoYLZxUh+L7aWym5RJp+wg2BQMvNLLEMkYjhIGdnlkGnQfPY+d59Yzb69nh Cz9g/VEugqW9w== From: "Xin Li (Intel)" To: linux-kernel@vger.kernel.org Cc: tglx@linutronix.de, mingo@redhat.com, bp@alien8.de, dave.hansen@linux.intel.com, x86@kernel.org, hpa@zytor.com, peterz@infradead.org, andrew.cooper3@citrix.com, seanjc@google.com Subject: [PATCH v2 1/3] x86/entry: Test ti_work for zero before processing individual bits Date: Thu, 22 Aug 2024 00:39:04 -0700 Message-ID: <20240822073906.2176342-2-xin@zytor.com> X-Mailer: git-send-email 2.46.0 In-Reply-To: <20240822073906.2176342-1-xin@zytor.com> References: <20240822073906.2176342-1-xin@zytor.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" In most cases, ti_work values passed to arch_exit_to_user_mode_prepare() are zeros, e.g., 99% in kernel build tests. So an obvious optimization is to test ti_work for zero before processing individual bits in it. In addition, Intel 0day tests find no perf regression with this change. Suggested-by: H. Peter Anvin (Intel) Signed-off-by: Xin Li (Intel) --- Change since v1: * Leave fpregs_assert_state_consistent() unconditional and independent of ti_work. (Brian Gerst and Thomas Gleixner) * Add arch_exit_work() to spare an extra indentation level. (Thomas Gleixne= r) --- arch/x86/include/asm/entry-common.h | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/arch/x86/include/asm/entry-common.h b/arch/x86/include/asm/ent= ry-common.h index fb2809b20b0a..db970828f385 100644 --- a/arch/x86/include/asm/entry-common.h +++ b/arch/x86/include/asm/entry-common.h @@ -44,8 +44,7 @@ static __always_inline void arch_enter_from_user_mode(str= uct pt_regs *regs) } #define arch_enter_from_user_mode arch_enter_from_user_mode =20 -static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs, - unsigned long ti_work) +static inline void arch_exit_work(unsigned long ti_work) { if (ti_work & _TIF_USER_RETURN_NOTIFY) fire_user_return_notifiers(); @@ -56,6 +55,13 @@ static inline void arch_exit_to_user_mode_prepare(struct= pt_regs *regs, fpregs_assert_state_consistent(); if (unlikely(ti_work & _TIF_NEED_FPU_LOAD)) switch_fpu_return(); +} + +static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs, + unsigned long ti_work) +{ + if (IS_ENABLED(CONFIG_X86_DEBUG_FPU) || unlikely(ti_work)) + arch_exit_work(ti_work); =20 #ifdef CONFIG_COMPAT /* --=20 2.46.0 From nobody Sat Feb 7 22:46:48 2026 Received: from mail.zytor.com (terminus.zytor.com [198.137.202.136]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 6AA801531DD for ; Thu, 22 Aug 2024 07:40:09 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=198.137.202.136 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1724312411; cv=none; b=QSrwQbiMYRmVr0Ir8DZqNqax44K/wo1zTQy/gRoOl5fTWB3z7zyvhv1BFS4SHFjlzM227aSudCO5iEuYiEGj7Ztb0qxRHIwvjEV+6n6hTi6c4ydsuOY3V+QJfG5hsDShTRrfTTnUe4sv6EMDHPX8Z7tGQc7UT9OqoOA6NrZejrE= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1724312411; c=relaxed/simple; bh=qv5IMD9jfZqw9E3gyg3gYpHIvZkme5FZDalGfFBfbzE=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=cHqjeq17GYhDaEm/DZ1j27ZJeU+2+zDGn6iAthmKpcUVtYbOye3gQ6CY6D3If4g+1Rx3j80dKDE2BikxbNZbHLa9BXUR6ScaUbyZVDwt1BX0+a2CRofzgTyYIl1JTezd4sT3pGthiB+UOheCDoFIqOdt5mv8Gy6xcHTJpc983zc= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=zytor.com; spf=pass smtp.mailfrom=zytor.com; dkim=pass (2048-bit key) header.d=zytor.com header.i=@zytor.com header.b=ccHnrX5z; arc=none smtp.client-ip=198.137.202.136 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=zytor.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=zytor.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=zytor.com header.i=@zytor.com header.b="ccHnrX5z" Received: from terminus.zytor.com (terminus.zytor.com [IPv6:2607:7c80:54:3:0:0:0:136]) (authenticated bits=0) by mail.zytor.com (8.17.2/8.17.1) with ESMTPSA id 47M7d7s72176365 (version=TLSv1.3 cipher=TLS_AES_256_GCM_SHA384 bits=256 verify=NO); Thu, 22 Aug 2024 00:39:12 -0700 DKIM-Filter: OpenDKIM Filter v2.11.0 mail.zytor.com 47M7d7s72176365 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=zytor.com; s=2024081601; t=1724312353; bh=W8Bu9+mIy9ZYM1EaY/aDq6EBee6SbMv0QtR5ansMrtE=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=ccHnrX5zofHcDdfZ48vt/FBiftC+RiHYP7j0SdSoU01sqwQYWDBfDUp8uqMWHRl4p Q0N3NPEwZ4WAulr6gww0vwjDuupSfdc7bS+sdMjsFOFkYy8tB71YGkx6aB1h3P9va6 3oO1CBkFBH54o/SOsfj5AyhwI/PBEsrnhuuxfbKXwCUDiCOcG7wef2NdTQ9ehltjU1 3XrjZqDvkuXqVSCPnTBgZ8y/vDYL2N7xjN38U/UH25c5pzTf+yXVQ68R5GrH7VT7gS o8WoxIm7nVJKTACJ0MJHAj9NWhr0m4+sDdBzjgQfTHnblEOb5LuYpcJleoRNu5wf5a UvD3LzihLesPA== From: "Xin Li (Intel)" To: linux-kernel@vger.kernel.org Cc: tglx@linutronix.de, mingo@redhat.com, bp@alien8.de, dave.hansen@linux.intel.com, x86@kernel.org, hpa@zytor.com, peterz@infradead.org, andrew.cooper3@citrix.com, seanjc@google.com Subject: [PATCH v2 2/3] x86/msr: Switch between WRMSRNS and WRMSR with the alternatives mechanism Date: Thu, 22 Aug 2024 00:39:05 -0700 Message-ID: <20240822073906.2176342-3-xin@zytor.com> X-Mailer: git-send-email 2.46.0 In-Reply-To: <20240822073906.2176342-1-xin@zytor.com> References: <20240822073906.2176342-1-xin@zytor.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Andrew Cooper Per the discussion about FRED MSR writes with WRMSRNS instruction [1], use the alternatives mechanism to choose WRMSRNS when it's available, otherwise fallback to WRMSR. Remove the dependency on X86_FEATURE_WRMSRNS as WRMSRNS is no longer dependent on FRED. [1] https://lore.kernel.org/lkml/15f56e6a-6edd-43d0-8e83-bb6430096514@citri= x.com/ Note, keep as DS prefix unless we get information to the contrary. Signed-off-by: Andrew Cooper Signed-off-by: Xin Li (Intel) --- Changes since v1: * Remove the dependency on X86_FEATURE_WRMSRNS. (Thomas Gleixner) --- arch/x86/include/asm/msr.h | 25 +++++++++++-------------- arch/x86/include/asm/switch_to.h | 1 - arch/x86/kernel/cpu/cpuid-deps.c | 1 - 3 files changed, 11 insertions(+), 16 deletions(-) diff --git a/arch/x86/include/asm/msr.h b/arch/x86/include/asm/msr.h index d642037f9ed5..001853541f1e 100644 --- a/arch/x86/include/asm/msr.h +++ b/arch/x86/include/asm/msr.h @@ -99,19 +99,6 @@ static __always_inline void __wrmsr(unsigned int msr, u3= 2 low, u32 high) : : "c" (msr), "a"(low), "d" (high) : "memory"); } =20 -/* - * WRMSRNS behaves exactly like WRMSR with the only difference being - * that it is not a serializing instruction by default. - */ -static __always_inline void __wrmsrns(u32 msr, u32 low, u32 high) -{ - /* Instruction opcode for WRMSRNS; supported in binutils >=3D 2.40. */ - asm volatile("1: .byte 0x0f,0x01,0xc6\n" - "2:\n" - _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_WRMSR) - : : "c" (msr), "a"(low), "d" (high)); -} - #define native_rdmsr(msr, val1, val2) \ do { \ u64 __val =3D __rdmsr((msr)); \ @@ -312,9 +299,19 @@ do { \ =20 #endif /* !CONFIG_PARAVIRT_XXL */ =20 +/* Instruction opcode for WRMSRNS supported in binutils >=3D 2.40 */ +#define WRMSRNS _ASM_BYTES(0x0f,0x01,0xc6) + +/* Non-serializing WRMSR, when available. Falls back to a serializing WRM= SR. */ static __always_inline void wrmsrns(u32 msr, u64 val) { - __wrmsrns(msr, val, val >> 32); + /* + * WRMSR is 2 bytes. WRMSRNS is 3 bytes. Pad WRMSR with a redundant + * DS prefix to avoid a trailing NOP. + */ + asm volatile("1: " ALTERNATIVE("ds wrmsr", WRMSRNS, X86_FEATURE_WRMSRNS) + "2: " _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_WRMSR) + : : "c" (msr), "a" ((u32)val), "d" ((u32)(val >> 32))); } =20 /* diff --git a/arch/x86/include/asm/switch_to.h b/arch/x86/include/asm/switch= _to.h index c3bd0c0758c9..e9ded149a9e3 100644 --- a/arch/x86/include/asm/switch_to.h +++ b/arch/x86/include/asm/switch_to.h @@ -71,7 +71,6 @@ static inline void update_task_stack(struct task_struct *= task) this_cpu_write(cpu_tss_rw.x86_tss.sp1, task->thread.sp0); #else if (cpu_feature_enabled(X86_FEATURE_FRED)) { - /* WRMSRNS is a baseline feature for FRED. */ wrmsrns(MSR_IA32_FRED_RSP0, (unsigned long)task_stack_page(task) + THREA= D_SIZE); } else if (cpu_feature_enabled(X86_FEATURE_XENPV)) { /* Xen PV enters the kernel on the thread stack. */ diff --git a/arch/x86/kernel/cpu/cpuid-deps.c b/arch/x86/kernel/cpu/cpuid-d= eps.c index b7d9f530ae16..8bd84114c2d9 100644 --- a/arch/x86/kernel/cpu/cpuid-deps.c +++ b/arch/x86/kernel/cpu/cpuid-deps.c @@ -83,7 +83,6 @@ static const struct cpuid_dep cpuid_deps[] =3D { { X86_FEATURE_AMX_TILE, X86_FEATURE_XFD }, { X86_FEATURE_SHSTK, X86_FEATURE_XSAVES }, { X86_FEATURE_FRED, X86_FEATURE_LKGS }, - { X86_FEATURE_FRED, X86_FEATURE_WRMSRNS }, {} }; =20 --=20 2.46.0 From nobody Sat Feb 7 22:46:48 2026 Received: from mail.zytor.com (terminus.zytor.com [198.137.202.136]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 6AAD51531F8 for ; Thu, 22 Aug 2024 07:40:09 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=198.137.202.136 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1724312411; cv=none; b=mwDzDKPhmdkMg9CHYrD4C0vxAi9dJrzfxlWWsmnMYd4BDciWlSVKlU7AXtZ/Gecq4YHHDCrv4js8bGQuzSW0szU5plLlvFdUI6xf/ZU0StAwPU+M9pg+nHdR3dZYd1MLiYhJLVKI6a7nWcy9ceiF58VciWLeLA+ncXZFXjdUFws= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1724312411; c=relaxed/simple; bh=aNYnXJfAjoHaQ/QbdLgAYt+ETay54vFfF5z45r3L8a4=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=DKTiAK2+/WD6iURBLaKJRX8YbY+BK1bCW4S/zVdCfubXue1GrIH1bczWlB5s3T6EQQIwB5HaaCNJ6rGC2vAn9VI42mSl9cVoJlepeDorZ0K7WJOkVigSvqLj54OT/dNi/rPu/whxTqz53izW1ir+11O/M4p/yCi35tunlczOq9c= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=zytor.com; spf=pass smtp.mailfrom=zytor.com; dkim=pass (2048-bit key) header.d=zytor.com header.i=@zytor.com header.b=eocL3/ZF; arc=none smtp.client-ip=198.137.202.136 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=zytor.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=zytor.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=zytor.com header.i=@zytor.com header.b="eocL3/ZF" Received: from terminus.zytor.com (terminus.zytor.com [IPv6:2607:7c80:54:3:0:0:0:136]) (authenticated bits=0) by mail.zytor.com (8.17.2/8.17.1) with ESMTPSA id 47M7d7s82176365 (version=TLSv1.3 cipher=TLS_AES_256_GCM_SHA384 bits=256 verify=NO); Thu, 22 Aug 2024 00:39:13 -0700 DKIM-Filter: OpenDKIM Filter v2.11.0 mail.zytor.com 47M7d7s82176365 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=zytor.com; s=2024081601; t=1724312353; bh=IeFVXk/e0IdKRKA1IK5jS0OXRd7M07rrVaEv/CTf7Uk=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=eocL3/ZFw52FiPYTLHSyM4KV4lLsfwZZnQsoUuvs6jJ+evfyUQ10Wmx3WXiq9Cmnf pnl32OtZtjRO6hRwf8LEItDdtGkWn5fU6tGoXiIZM4sGv6sKLiNbWpXdcKUGa3RYRO 1Ws3j250ZhqjyEmD7F/UJbNQ3GI50uFGDHYjr3J7iCuOeJBme4+JTvHyCYQDWXOwmp tmAJHKG15ml2RS7Isr+8MkuxDX6uELP7Pf6blE6iASZuILeLzz0Tyym6dbH6TjcYKO 60i+E7eR9mcKz/yHlm/PagofJycb5xCTzW2gVNCRdQaxQ+eOA+tctFbGr9LeHWtlTF d7u5U5mkA49QA== From: "Xin Li (Intel)" To: linux-kernel@vger.kernel.org Cc: tglx@linutronix.de, mingo@redhat.com, bp@alien8.de, dave.hansen@linux.intel.com, x86@kernel.org, hpa@zytor.com, peterz@infradead.org, andrew.cooper3@citrix.com, seanjc@google.com Subject: [PATCH v2 3/3] x86/entry: Set FRED RSP0 on return to userspace instead of context switch Date: Thu, 22 Aug 2024 00:39:06 -0700 Message-ID: <20240822073906.2176342-4-xin@zytor.com> X-Mailer: git-send-email 2.46.0 In-Reply-To: <20240822073906.2176342-1-xin@zytor.com> References: <20240822073906.2176342-1-xin@zytor.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" FRED RSP0 is a per task constant pointing to top of its kernel stack for user level event delivery, and needs to be updated when a task is scheduled in. Keep FRED RSP0 in per CPU cache, and update FRED RSP0 and its copy in per CPU cache only when switching to a new task, i.e., current stack is changed. An API fred_sync_rsp0() is added for KVM to stuff the cache with the guest's FRED RSP0 when switching to host. Suggested-by: Sean Christopherson Suggested-by: Thomas Gleixner Signed-off-by: Xin Li (Intel) --- Change since v1: * Use per CPU cache, rather than a new TI flag. (Thomas Gleixner) --- arch/x86/include/asm/entry-common.h | 3 +++ arch/x86/include/asm/fred.h | 21 ++++++++++++++++++++- arch/x86/include/asm/switch_to.h | 5 +---- arch/x86/kernel/fred.c | 3 +++ 4 files changed, 27 insertions(+), 5 deletions(-) diff --git a/arch/x86/include/asm/entry-common.h b/arch/x86/include/asm/ent= ry-common.h index db970828f385..77d20555e04d 100644 --- a/arch/x86/include/asm/entry-common.h +++ b/arch/x86/include/asm/entry-common.h @@ -8,6 +8,7 @@ #include #include #include +#include =20 /* Check that the stack and regs on entry from user mode are sane. */ static __always_inline void arch_enter_from_user_mode(struct pt_regs *regs) @@ -63,6 +64,8 @@ static inline void arch_exit_to_user_mode_prepare(struct = pt_regs *regs, if (IS_ENABLED(CONFIG_X86_DEBUG_FPU) || unlikely(ti_work)) arch_exit_work(ti_work); =20 + fred_update_rsp0(); + #ifdef CONFIG_COMPAT /* * Compat syscalls set TS_COMPAT. Make sure we clear it before diff --git a/arch/x86/include/asm/fred.h b/arch/x86/include/asm/fred.h index 66d7dbe2d314..25ca00bd70e8 100644 --- a/arch/x86/include/asm/fred.h +++ b/arch/x86/include/asm/fred.h @@ -36,6 +36,7 @@ =20 #ifdef CONFIG_X86_FRED #include +#include =20 #include =20 @@ -87,12 +88,30 @@ void cpu_init_fred_exceptions(void); void cpu_init_fred_rsps(void); void fred_complete_exception_setup(void); =20 +DECLARE_PER_CPU(unsigned long, fred_rsp0); + +static __always_inline void fred_sync_rsp0(unsigned long rsp0) +{ + __this_cpu_write(fred_rsp0, rsp0); +} + +static __always_inline void fred_update_rsp0(void) +{ + unsigned long rsp0 =3D (unsigned long) task_stack_page(current) + THREAD_= SIZE; + + if (cpu_feature_enabled(X86_FEATURE_FRED) && (__this_cpu_read(fred_rsp0) = !=3D rsp0)) { + wrmsrns(MSR_IA32_FRED_RSP0, rsp0); + __this_cpu_write(fred_rsp0, rsp0); + } +} #else /* CONFIG_X86_FRED */ static __always_inline unsigned long fred_event_data(struct pt_regs *regs)= { return 0; } static inline void cpu_init_fred_exceptions(void) { } static inline void cpu_init_fred_rsps(void) { } static inline void fred_complete_exception_setup(void) { } -static __always_inline void fred_entry_from_kvm(unsigned int type, unsigne= d int vector) { } +static inline void fred_entry_from_kvm(unsigned int type, unsigned int vec= tor) { } +static inline void fred_sync_rsp0(unsigned long rsp0) { } +static inline void fred_update_rsp0(void) { } #endif /* CONFIG_X86_FRED */ #endif /* !__ASSEMBLY__ */ =20 diff --git a/arch/x86/include/asm/switch_to.h b/arch/x86/include/asm/switch= _to.h index e9ded149a9e3..75248546403d 100644 --- a/arch/x86/include/asm/switch_to.h +++ b/arch/x86/include/asm/switch_to.h @@ -70,12 +70,9 @@ static inline void update_task_stack(struct task_struct = *task) #ifdef CONFIG_X86_32 this_cpu_write(cpu_tss_rw.x86_tss.sp1, task->thread.sp0); #else - if (cpu_feature_enabled(X86_FEATURE_FRED)) { - wrmsrns(MSR_IA32_FRED_RSP0, (unsigned long)task_stack_page(task) + THREA= D_SIZE); - } else if (cpu_feature_enabled(X86_FEATURE_XENPV)) { + if (!cpu_feature_enabled(X86_FEATURE_FRED) && cpu_feature_enabled(X86_FEA= TURE_XENPV)) /* Xen PV enters the kernel on the thread stack. */ load_sp0(task_top_of_stack(task)); - } #endif } =20 diff --git a/arch/x86/kernel/fred.c b/arch/x86/kernel/fred.c index 99a134fcd5bf..036d5cd47f88 100644 --- a/arch/x86/kernel/fred.c +++ b/arch/x86/kernel/fred.c @@ -21,6 +21,9 @@ =20 #define FRED_STKLVL(vector, lvl) ((lvl) << (2 * (vector))) =20 +DEFINE_PER_CPU(unsigned long, fred_rsp0); +EXPORT_PER_CPU_SYMBOL(fred_rsp0); + void cpu_init_fred_exceptions(void) { /* When FRED is enabled by default, remove this log message */ --=20 2.46.0